From cb64bb687427cb757e8ebe77731455cba371b298 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 13 Jan 2011 14:58:53 -0800 Subject: [PATCH 001/964] Porting Matti's branch to git. No changes. --- aux/broctl | 2 +- policy/bro.init | 12 +- policy/icmp.bro | 144 +++++++++++- src/Analyzer.cc | 8 +- src/AnalyzerTags.h | 2 +- src/DPM.cc | 12 +- src/ICMP.cc | 555 +++++++++++++++++++++++++++++++++++++++------ src/ICMP.h | 77 ++----- src/Sessions.cc | 50 +++- src/Val.cc | 1 - src/Val.h | 8 +- src/event.bif | 18 +- src/net_util.cc | 59 ++++- src/net_util.h | 1 + 14 files changed, 795 insertions(+), 154 deletions(-) diff --git a/aux/broctl b/aux/broctl index a05be1242b..0d8b64252f 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit a05be1242b4e06dca1bb1a38ed871e7e2d78181b +Subproject commit 0d8b64252f00f147f31f5e8c02a6a710699b67d9 diff --git a/policy/bro.init b/policy/bro.init index 1ba8f59b4d..e812db0075 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -38,11 +38,12 @@ type icmp_context: record { id: conn_id; len: count; proto: count; - frag_offset: count; - bad_hdr_len: bool; - bad_checksum: bool; - MF: bool; - DF: bool; + frag_offset: count &optional; #no frag offset for IPv6 + bad_hdr_len: bool &optional; + bad_checksum: bool &optional; #no checksum in IPv6 header + MF: bool &optional; #no MF for IPv6 + DF: bool &optional; #no DF for IPv6 + ICMP6Flag: bool; }; type addr_set: set[addr]; @@ -534,6 +535,7 @@ const IPPROTO_IGMP = 2; # group mgmt protocol const IPPROTO_IPIP = 4; # IP encapsulation in IP const IPPROTO_TCP = 6; # TCP const IPPROTO_UDP = 17; # user datagram protocol +const IPPROTO_ICMPV6 = 58; # ICMP for IPv6 const IPPROTO_RAW = 255; # raw IP packet type ip_hdr: record { diff --git a/policy/icmp.bro b/policy/icmp.bro index c6c3c87d44..d75b2a3731 100644 --- a/policy/icmp.bro +++ b/policy/icmp.bro @@ -1,4 +1,5 @@ # $Id: icmp.bro 6883 2009-08-19 21:08:09Z vern $ +# While using this script, please notice that the last F/T value is the IPv6 Flag @load hot @load weird @@ -55,8 +56,16 @@ type flow_info: record { payload: string; }; +#Insert whitelisted routers here, Router advertisements from other +#routers will be logged as possible rogue router attacks +const routers_whitelist: table[string] of bool = { + #["fe80::260:97ff:fe07:69ea"] = T, #an example + } &redef &default = F; + const names: table[count] of string = { [0] = "echo_reply", + [1] = "unreach", #icmpv6 + [2] = "too_big", #icmpv6 [3] = "unreach", [4] = "quench", [5] = "redirect", @@ -71,6 +80,33 @@ const names: table[count] of string = { [16] = "info_reply", [17] = "mask_req", [18] = "mask_reply", + [128] = "echo_req", #icmpv6 + [129] = "echo_reply", #icmpv6 + [130] = "group_memb_query", #icmpv6 + [131] = "group_memb_report", #icmpv6 + [132] = "group_memb_reduct", #icmpv6 + [133] = "router_sol", #icmpv6 + [134] = "router_ad", #icmpv6 + [135] = "neighbor_sol", #icmpv6 + [136] = "neighbor_ad", #icmpv6 + [137] = "redirect", #icmpv6 + [138] = "router_renum", #icmpv6 + [139] = "node_info_query", #icmpv6 + [140] = "node_info_resp", #icmpv6 + [141] = "inv_neigh_disc_sol", #icmpv6 + [142] = "inv_neigh_disc_ad", #icmpv6 + [143] = "mul_lis_report", #icmpv6 + [144] = "home_agent_addr_req", #icmpv6 + [145] = "home_agent_addr_reply",#icmpv6 + [146] = "mobible_prefx_sol", #icmpv6 + [147] = "mobible_prefx_ad", #icmpv6 + [148] = "cert_path_sol", #icmpv6 + [149] = "cert_path_ad", #icmpv6 + [150] = "experimental", #icmpv6 + [151] = "mcast_router_ad", #icmpv6 + [152] = "mcast_router_sol", #icmpv6 + [153] = "mcast_router_term", #icmpv6 + [154] = "fmip", #icmpv6 } &default = function(n: count): string { return fmt("icmp-%d", n); }; @@ -80,7 +116,8 @@ const IP_proto_name: table[count] of string = { [2] = "IGMP", [6] = "TCP", [17] = "UDP", - [41] = "IPV6", + [41] = "IP6", + [58] = "ICMP6", } &default = function(n: count): string { return fmt("%s", n); } &redef; @@ -123,12 +160,13 @@ global flows: table[flow_id] of flow_info &read_expire = 45 sec &expire_func = flush_flow; -event icmp_sent(c: connection, icmp: icmp_conn) +event icmp_sent(c: connection, icmp: icmp_conn, ICMP6: bool) { - print icmp_file, fmt("%.6f %.6f %s %s %s %s %s %s %s %s %s", + + print icmp_file, fmt("%.6f %.6f %s %s %s %s %s %s %s %s %s %s", network_time(), 0.0, icmp$orig_h, icmp$resp_h, names[icmp$itype], icmp$itype, icmp$icode, "icmp", - icmp$len, "0", "SH"); + icmp$len, "0", "SH", ICMP6); } event flow_summary(flow: flow_id, last_time: time) @@ -173,13 +211,64 @@ function update_flow(icmp: icmp_conn, id: count, is_orig: bool, payload: string) schedule +30sec { flow_summary(fid, fi$last_time) }; } -event icmp_echo_request(c: connection, icmp: icmp_conn, id: count, seq: count, payload: string) + +event icmp_error_message(c: connection, icmp: icmp_conn, code: count, context: icmp_context) #for other but the unreach types, which is preserved + { + + if ( active_connection(context$id) ) + { + # This section allows Bro to act on ICMP error message packets + # that happen in the context of an active connection. It is + # not currently used. + local c2 = connection_record(context$id); + local os = c2$orig$state; + local rs = c2$resp$state; + local is_attempt = + is_tcp_port(c2$id$orig_p) ? + (os == TCP_SYN_SENT && rs == TCP_INACTIVE) : + (os == UDP_ACTIVE && rs == UDP_INACTIVE); + + # Insert action here. + } + + if ( log_details ) + { + # ICMP error message packets are logged here. + # Due to the connection data contained *within* + # them, each log line will contain two connections' worth + # of data. The initial ICMP connection info is the same + # as logged for connections. + print icmp_file, fmt("%.6f %.6f %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s", + network_time(), 0.0, icmp$orig_h, icmp$resp_h, + names[icmp$itype], icmp$itype, icmp$icode, "icmp", + icmp$len, "0", "EncapPkt:", + # This is the encapsulated packet: + context$id$orig_h, context$id$orig_p, + context$id$resp_h, context$id$resp_p, + context$len, IP_proto_name[context$proto], + context$len, context$bad_hdr_len, + context$bad_checksum, context$ICMP6Flag); + } + + } + + + +event icmp6_placeholder(c: connection, icmp: icmp_conn, ICMP6: bool) #just for testing + { + print "icmp6_placeholder triggered"; + } + + +event icmp_echo_request(c: connection, icmp: icmp_conn, id: count, seq: count, payload: string, ICMP6: bool) { update_flow(icmp, id, T, payload); + local orig = icmp$orig_h; local resp = icmp$resp_h; + # Simple ping scan detector. if ( detect_scans && (orig !in Scan::distinct_peers || @@ -231,7 +320,7 @@ event icmp_echo_request(c: connection, icmp: icmp_conn, id: count, seq: count, p } event icmp_echo_reply(c: connection, icmp: icmp_conn, id: count, - seq: count, payload: string) + seq: count, payload: string, ICMP6: bool) { # Check payload with the associated flow. @@ -240,6 +329,8 @@ event icmp_echo_reply(c: connection, icmp: icmp_conn, id: count, fid$resp_h = icmp$orig_h; # it's an echo reply. fid$id = id; + + if ( fid !in flows ) { # NOTICE([$note=ICMPUnpairedEchoReply, @@ -266,9 +357,12 @@ event icmp_echo_reply(c: connection, icmp: icmp_conn, id: count, update_flow(icmp, id, F, payload); } + + event icmp_unreachable(c: connection, icmp: icmp_conn, code: count, context: icmp_context) { + if ( active_connection(context$id) ) { # This section allows Bro to act on ICMP-unreachable packets @@ -292,7 +386,7 @@ event icmp_unreachable(c: connection, icmp: icmp_conn, code: count, # them, each log line will contain two connections' worth # of data. The initial ICMP connection info is the same # as logged for connections. - print icmp_file, fmt("%.6f %.6f %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s", + print icmp_file, fmt("%.6f %.6f %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s", network_time(), 0.0, icmp$orig_h, icmp$resp_h, names[icmp$itype], icmp$itype, icmp$icode, "icmp", icmp$len, "0", "EncapPkt:", @@ -301,6 +395,40 @@ event icmp_unreachable(c: connection, icmp: icmp_conn, code: count, context$id$resp_h, context$id$resp_p, context$len, IP_proto_name[context$proto], context$len, context$bad_hdr_len, - context$bad_checksum); + context$bad_checksum, context$ICMP6Flag); } } + + + event icmp_router_advertisement(c: connection, icmp: icmp_conn, ICMP6: bool) + { + if ( routers_whitelist[ fmt("%s",icmp$orig_h) ] ) + { + print icmp_file, fmt("%.6f %.6f %s %s %s %s %s %s %s %s %s %s", + network_time(), 0.0, icmp$orig_h, icmp$resp_h, + names[icmp$itype], icmp$itype, icmp$icode, "icmp", + icmp$len, "0", "SH", ICMP6); + } + else + { + print icmp_file, fmt("%.6f %.6f %s %s %s %s %s %s %s %s", + network_time(), 0.0, icmp$orig_h, icmp$resp_h, + names[icmp$itype], "Possible Rogue Router Detected", icmp$itype, icmp$icode, + icmp$len, ICMP6); + } + + } + + + + + + + + + + + + + + diff --git a/src/Analyzer.cc b/src/Analyzer.cc index c323f99e23..a3f6bbfc2c 100644 --- a/src/Analyzer.cc +++ b/src/Analyzer.cc @@ -48,7 +48,7 @@ const Analyzer::Config Analyzer::analyzer_configs[] = { { AnalyzerTag::ICMP, "ICMP", ICMP_Analyzer::InstantiateAnalyzer, ICMP_Analyzer::Available, 0, false }, - { AnalyzerTag::ICMP_TimeExceeded, "ICMP_TIMEEXCEEDED", + /*{ AnalyzerTag::ICMP_TimeExceeded, "ICMP_TIMEEXCEEDED", ICMP_TimeExceeded_Analyzer::InstantiateAnalyzer, ICMP_TimeExceeded_Analyzer::Available, 0, false }, { AnalyzerTag::ICMP_Unreachable, "ICMP_UNREACHABLE", @@ -56,7 +56,11 @@ const Analyzer::Config Analyzer::analyzer_configs[] = { ICMP_Unreachable_Analyzer::Available, 0, false }, { AnalyzerTag::ICMP_Echo, "ICMP_ECHO", ICMP_Echo_Analyzer::InstantiateAnalyzer, - ICMP_Echo_Analyzer::Available, 0, false }, + ICMP_Echo_Analyzer::Available, 0, false },*/ + + + + { AnalyzerTag::TCP, "TCP", TCP_Analyzer::InstantiateAnalyzer, TCP_Analyzer::Available, 0, false }, diff --git a/src/AnalyzerTags.h b/src/AnalyzerTags.h index 9bf3efbd3c..eb18a03a73 100644 --- a/src/AnalyzerTags.h +++ b/src/AnalyzerTags.h @@ -22,7 +22,7 @@ namespace AnalyzerTag { PIA_TCP, PIA_UDP, // Transport-layer analyzers. - ICMP, ICMP_TimeExceeded, ICMP_Unreachable, ICMP_Echo, TCP, UDP, + ICMP,/* ICMP_TimeExceeded, ICMP_Unreachable, ICMP_Echo*/ TCP, UDP, // Application-layer analyzers (hand-written). BitTorrent, BitTorrentTracker, diff --git a/src/DPM.cc b/src/DPM.cc index 35111a38fa..7be9376b9f 100644 --- a/src/DPM.cc +++ b/src/DPM.cc @@ -215,7 +215,10 @@ bool DPM::BuildInitialAnalyzerTree(TransportProto proto, Connection* conn, case TRANSPORT_ICMP: { const struct icmp* icmpp = (const struct icmp *) data; - switch ( icmpp->icmp_type ) { + + + //Old code, moving to having only one ICMP analyzer + /*switch ( icmpp->icmp_type ) { case ICMP_ECHO: case ICMP_ECHOREPLY: @@ -241,10 +244,11 @@ bool DPM::BuildInitialAnalyzerTree(TransportProto proto, Connection* conn, DBG_DPD(conn, "activated ICMP Time Exceeded analyzer"); } break; - } + }*/ + //if ( ! root ) - if ( ! root ) - root = new ICMP_Analyzer(conn); + root = new ICMP_Analyzer(conn); + DBG_DPD(conn, "activated ICMP analyzer"); analyzed = true; break; diff --git a/src/ICMP.cc b/src/ICMP.cc index d73a9a781e..b83cf76a40 100644 --- a/src/ICMP.cc +++ b/src/ICMP.cc @@ -9,12 +9,17 @@ #include "Event.h" #include "ICMP.h" +#include + + + ICMP_Analyzer::ICMP_Analyzer(Connection* c) : TransportLayerAnalyzer(AnalyzerTag::ICMP, c) { icmp_conn_val = 0; c->SetInactivityTimeout(icmp_inactivity_timeout); request_len = reply_len = -1; + } ICMP_Analyzer::ICMP_Analyzer(AnalyzerTag::Tag tag, Connection* c) @@ -45,16 +50,43 @@ void ICMP_Analyzer::DeliverPacket(int arg_len, const u_char* data, // Subtract off the common part of ICMP header. PacketContents(data + 8, min(len, caplen) - 8); + const struct icmp* icmpp = (const struct icmp*) data; len = arg_len; - if ( ! ignore_checksums && caplen >= len && - icmp_checksum(icmpp, len) != 0xffff ) + + //We need a separate calculation for ICMP6 checksums, pseudoheader is appended to the + //ICMP6 checksum calculation, which is different from ICMP4 +#ifdef BROv6 + + + if (ip->NextProto() == IPPROTO_ICMPV6 && ! ignore_checksums && + caplen >= len && icmp6_checksum(icmpp,ip->IP6_Hdr(),len )!= 0xffff ) + { + Weird("bad_ICMP6_checksum"); + return; + } + else if (ip->NextProto() != IPPROTO_ICMPV6 && ! ignore_checksums && + caplen >= len && icmp_checksum(icmpp, len) != 0xffff ) { Weird("bad_ICMP_checksum"); return; } + + +#else + + if ( ! ignore_checksums && caplen >= len && + icmp_checksum(icmpp, len) != 0xffff ) + { + Weird("bad_ICMP_checksum"); + return; + } +#endif + + + Conn()->SetLastTime(current_timestamp); if ( rule_matcher ) @@ -66,7 +98,7 @@ void ICMP_Analyzer::DeliverPacket(int arg_len, const u_char* data, type = icmpp->icmp_type; code = icmpp->icmp_code; - // Move past common portion of ICMP header. + // Move past common portion of ICMP header. //OK for ICMPv6? data += 8; caplen -= 8; len -= 8; @@ -77,33 +109,113 @@ void ICMP_Analyzer::DeliverPacket(int arg_len, const u_char* data, else len_stat += len; - NextICMP(current_timestamp, icmpp, len, caplen, data); + NextICMP(current_timestamp, icmpp, len, caplen, data, ip); if ( rule_matcher ) matcher_state.Match(Rule::PAYLOAD, data, len, is_orig, false, false, true); } -void ICMP_Analyzer::NextICMP(double /* t */, const struct icmp* /* icmpp */, - int /* len */, int /* caplen */, - const u_char*& /* data */) - { - ICMPEvent(icmp_sent); - } -void ICMP_Analyzer::ICMPEvent(EventHandlerPtr f) - { + +/********************Generic analyzer for all ICMP4/ICMP6******************************/ +void ICMP_Analyzer::NextICMP(double t , const struct icmp* icmpp , int len , int caplen, + const u_char*& data, const IP_Hdr* ip_hdr ) + { + int ICMP6Flag = 0; + + //printf("Executing: ICMP_Analyzer::NextICMP\n"); + //printf("New analyzer structure\n"); + + if ( ip_hdr->NextProto() == IPPROTO_ICMPV6 ) + { + //printf("ICMP6!\n"); + ICMP6Flag = 1; + + switch (type) //Add new ICMP6 functions here, you can also use codes to narrow the area of single functions. + { + //All the echo stuff here + case ICMP6_ECHO_REQUEST: + case ICMP6_ECHO_REPLY: + Echo(t, icmpp, len, caplen, data, ip_hdr); + break; + + + //Error messages all have the same structure for their context, and are handled by the same function. + case ICMP6_PARAM_PROB: + case ICMP6_TIME_EXCEEDED: + case ICMP6_PACKET_TOO_BIG: + case ICMP6_DST_UNREACH: + Context(t, icmpp, len, caplen, data, ip_hdr); + break; + + //All router related stuff should eventually be handled by the Router() + case ND_REDIRECT: + case ND_ROUTER_SOLICIT: + case ICMP6_ROUTER_RENUMBERING: + case ND_ROUTER_ADVERT: + Router(t, icmpp, len, caplen, data, ip_hdr); //currently only logs the router stuff for other than router_advert + break; + + /* listed for convenience + case ICMP6_PARAM_PROB: break; + case MLD_LISTENER_QUERY: break; + case MLD_LISTENER_REPORT: break; + case MLD_LISTENER_REDUCTION: break; + case ND_NEIGHBOR_SOLICIT: break; + case ND_NEIGHBOR_ADVERT: break; + case ND_REDIRECT: break; + case ICMP6_ROUTER_RENUMBERING: break; + case ND_NEIGHBOR_SOLICIT: break; + case ND_NEIGHBOR_ADVERT: break; + case ICMP6_TIME_EXCEEDED: break; + */ + + default: ICMPEvent(icmp_sent, ICMP6Flag); break; + } + } + else if ( ip_hdr->NextProto() == IPPROTO_ICMP ) + { + + switch (type) //Add new ICMP4 functions here + { + case ICMP_ECHO: + case ICMP_ECHOREPLY: + Echo(t, icmpp, len, caplen, data, ip_hdr); + break; + + case ICMP_UNREACH: + case ICMP_TIMXCEED: + Context(t, icmpp, len, caplen, data, ip_hdr); + break; + + default: ICMPEvent(icmp_sent, ICMP6Flag); break; + } + + + } + else + Weird("Malformed ip header"); + } + + +void ICMP_Analyzer::ICMPEvent(EventHandlerPtr f, int ICMP6Flag) + { if ( ! f ) - return; + return; + val_list* vl = new val_list; vl->append(BuildConnVal()); - vl->append(BuildICMPVal()); + vl->append(BuildICMPVal(ICMP6Flag)); + //if ( f == icmp_sent ) //for now, testing purposes + vl->append(new Val(ICMP6Flag, TYPE_BOOL)); ConnectionEvent(f, vl); } -RecordVal* ICMP_Analyzer::BuildICMPVal() + +RecordVal* ICMP_Analyzer::BuildICMPVal(int ICMP6Flag) { if ( ! icmp_conn_val ) { @@ -111,7 +223,13 @@ RecordVal* ICMP_Analyzer::BuildICMPVal() icmp_conn_val->Assign(0, new AddrVal(Conn()->OrigAddr())); icmp_conn_val->Assign(1, new AddrVal(Conn()->RespAddr())); - icmp_conn_val->Assign(2, new Val(type, TYPE_COUNT)); + + if ( ICMP6Flag == 1 ) + icmp_conn_val->Assign(2, new Val(Type6to4(type), TYPE_COUNT)); //to avoid errors in getting the message type *name* right on the scripting level, type number will be different from true ipv6 + else + icmp_conn_val->Assign(2, new Val(type, TYPE_COUNT)); + + icmp_conn_val->Assign(3, new Val(code, TYPE_COUNT)); icmp_conn_val->Assign(4, new Val(len, TYPE_COUNT)); } @@ -121,48 +239,170 @@ RecordVal* ICMP_Analyzer::BuildICMPVal() return icmp_conn_val; } -RecordVal* ICMP_Analyzer::ExtractICMPContext(int len, const u_char*& data) +RecordVal* ICMP_Analyzer::ExtractICMP4Context(int len, const u_char*& data) { - const struct ip* ip = (const struct ip *) data; - uint32 ip_hdr_len = ip->ip_hl * 4; + /** + * For use only with ICMP4, ICMPV6 context extraction is still non-functional + */ + + const IP_Hdr ip_hdr_data((const struct ip*) data); + const IP_Hdr* ip_hdr = &ip_hdr_data; + int ICMP6Flag = 0; + + uint32 ip_hdr_len = ip_hdr->HdrLen(); uint32 ip_len, frag_offset; TransportProto proto = TRANSPORT_UNKNOWN; int DF, MF, bad_hdr_len, bad_checksum; - uint32 src_addr, dst_addr; + uint32 src_addr, dst_addr,src_addr2, dst_addr2; uint32 src_port, dst_port; - if ( ip_hdr_len < sizeof(struct ip) || ip_hdr_len > uint32(len) ) - { // We don't have an entire IP header. + if ( ip_hdr_len < sizeof(struct ip) || ip_hdr_len > uint32(len) ) + { // We don't have an entire IP header. + bad_hdr_len = 1; + ip_len = frag_offset = 0; + DF = MF = bad_checksum = 0; + src_addr = dst_addr = 0; + src_port = dst_port = 0; + } + + else + { + bad_hdr_len = 0; + ip_len = ip_hdr->TotalLen(); + bad_checksum = ones_complement_checksum((void*) ip_hdr->IP4_Hdr(), ip_hdr_len, 0) != 0xffff; + + src_addr = ip_hdr->SrcAddr4(); + dst_addr = ip_hdr->DstAddr4(); + + switch ( ip_hdr->NextProto() ) { + case 1: proto = TRANSPORT_ICMP; break; + case 6: proto = TRANSPORT_TCP; break; + case 17: proto = TRANSPORT_UDP; break; + + // Default uses TRANSPORT_UNKNOWN, per initialization above. + } + + uint32 frag_field = ip_hdr->FragField(); + DF = ip_hdr->DF(); + MF = frag_field & 0x2000; + frag_offset = frag_field & /* IP_OFFMASK not portable */ 0x1fff; + + const u_char* transport_hdr = ((u_char *) ip_hdr->IP4_Hdr() + ip_hdr_len); + + if ( uint32(len) < ip_hdr_len + 4 ) //what is this value for ipv6? + { + // 4 above is the magic number meaning that both + // port numbers are included in the ICMP. + bad_hdr_len = 1; + src_port = dst_port = 0; + } + + switch ( proto ) { + case TRANSPORT_ICMP: + { + const struct icmp* icmpp = + (const struct icmp *) transport_hdr; + bool is_one_way; // dummy + src_port = ntohs(icmpp->icmp_type); + dst_port = ntohs(ICMP4_counterpart(icmpp->icmp_type, + icmpp->icmp_code, + is_one_way)); + } + break; + + case TRANSPORT_TCP: + { + const struct tcphdr* tp = + (const struct tcphdr *) transport_hdr; + src_port = ntohs(tp->th_sport); + dst_port = ntohs(tp->th_dport); + } + break; + + case TRANSPORT_UDP: + { + const struct udphdr* up = + (const struct udphdr *) transport_hdr; + src_port = ntohs(up->uh_sport); + dst_port = ntohs(up->uh_dport); + } + break; + + default: + src_port = dst_port = ntohs(0); + } + } + + RecordVal* iprec = new RecordVal(icmp_context); + RecordVal* id_val = new RecordVal(conn_id); + + id_val->Assign(0, new AddrVal(src_addr)); + id_val->Assign(1, new PortVal(src_port, proto)); + id_val->Assign(2, new AddrVal(dst_addr)); + id_val->Assign(3, new PortVal(dst_port, proto)); + iprec->Assign(0, id_val); + + iprec->Assign(1, new Val(ip_len, TYPE_COUNT)); + iprec->Assign(2, new Val(proto, TYPE_COUNT)); + iprec->Assign(3, new Val(frag_offset, TYPE_COUNT)); + iprec->Assign(4, new Val(bad_hdr_len, TYPE_BOOL)); + iprec->Assign(5, new Val(bad_checksum, TYPE_BOOL)); + iprec->Assign(6, new Val(MF, TYPE_BOOL)); + iprec->Assign(7, new Val(DF, TYPE_BOOL)); + iprec->Assign(8, new Val(ICMP6Flag, TYPE_BOOL)); + + return iprec; + } + + + + +RecordVal* ICMP_Analyzer::ExtractICMP6Context(int len, const u_char*& data) + { + /** + * For use with ICMP6 error message context extraction (possibly very frail function) + */ + + const IP_Hdr ip_hdr_data((const struct ip6_hdr*) data); + const IP_Hdr* ip_hdr = &ip_hdr_data; + int ICMP6Flag = 1; + int DF = 0, MF = 0, bad_hdr_len = 0, bad_checksum = 0; + + uint32 ip_hdr_len = ip_hdr->HdrLen(); //should always be 40 + uint32* src_addr; + uint32* dst_addr; + uint32 ip_len, frag_offset = 0; + TransportProto proto = TRANSPORT_UNKNOWN; + uint32 src_port, dst_port; + + if ( ip_hdr_len < sizeof(struct ip6_hdr) || ip_hdr_len != 40 ) + { bad_hdr_len = 1; - ip_len = frag_offset = 0; - DF = MF = bad_checksum = 0; + ip_len = 0; src_addr = dst_addr = 0; src_port = dst_port = 0; } - else { - bad_hdr_len = 0; - ip_len = ntohs(ip->ip_len); - bad_checksum = ones_complement_checksum((void*) ip, ip_hdr_len, 0) != 0xffff; + ip_len = ip_hdr->TotalLen(); - src_addr = uint32(ip->ip_src.s_addr); - dst_addr = uint32(ip->ip_dst.s_addr); + src_addr = (uint32 *) ip_hdr->SrcAddr(); + dst_addr = (uint32 *) ip_hdr->DstAddr(); - switch ( ip->ip_p ) { + + + switch ( ip_hdr->NextProto() ) { case 1: proto = TRANSPORT_ICMP; break; case 6: proto = TRANSPORT_TCP; break; case 17: proto = TRANSPORT_UDP; break; + case 58: proto = TRANSPORT_ICMP; break; //TransportProto Hack // Default uses TRANSPORT_UNKNOWN, per initialization above. } - uint32 frag_field = ntohs(ip->ip_off); - DF = frag_field & 0x4000; - MF = frag_field & 0x2000; - frag_offset = frag_field & /* IP_OFFMASK not portable */ 0x1fff; - const u_char* transport_hdr = ((u_char *) ip + ip_hdr_len); + + const u_char* transport_hdr = ((u_char *)ip_hdr->IP6_Hdr() + ip_hdr_len); if ( uint32(len) < ip_hdr_len + 4 ) { @@ -179,7 +419,7 @@ RecordVal* ICMP_Analyzer::ExtractICMPContext(int len, const u_char*& data) (const struct icmp *) transport_hdr; bool is_one_way; // dummy src_port = ntohs(icmpp->icmp_type); - dst_port = ntohs(ICMP_counterpart(icmpp->icmp_type, + dst_port = ntohs(ICMP6_counterpart(icmpp->icmp_type, icmpp->icmp_code, is_one_way)); } @@ -215,19 +455,42 @@ RecordVal* ICMP_Analyzer::ExtractICMPContext(int len, const u_char*& data) id_val->Assign(1, new PortVal(src_port, proto)); id_val->Assign(2, new AddrVal(dst_addr)); id_val->Assign(3, new PortVal(dst_port, proto)); - iprec->Assign(0, id_val); + iprec->Assign(0, id_val); iprec->Assign(1, new Val(ip_len, TYPE_COUNT)); - iprec->Assign(2, new Val(proto, TYPE_COUNT)); - iprec->Assign(3, new Val(frag_offset, TYPE_COUNT)); + + //TransportProto Hack + if ( ip_hdr->NextProto() == 58 || 17 ) //if the encap packet is ICMPv6 we force this... (cause there is no IGMP (by that name) for ICMPv6), rather ugly hack once more + { + iprec->Assign(2, new Val(58, TYPE_COUNT)); + } + else + { + iprec->Assign(2, new Val(proto, TYPE_COUNT)); + } + + iprec->Assign(3, new Val(frag_offset, TYPE_COUNT)); //NA for ip6 iprec->Assign(4, new Val(bad_hdr_len, TYPE_BOOL)); iprec->Assign(5, new Val(bad_checksum, TYPE_BOOL)); - iprec->Assign(6, new Val(MF, TYPE_BOOL)); - iprec->Assign(7, new Val(DF, TYPE_BOOL)); + iprec->Assign(6, new Val(MF, TYPE_BOOL)); //NA for ip6 + iprec->Assign(7, new Val(DF, TYPE_BOOL)); //NA for ip6 + iprec->Assign(8, new Val(ICMP6Flag, TYPE_BOOL)); //ICMP6Flag return iprec; } + + + + + + + + + + + + bool ICMP_Analyzer::IsReuse(double /* t */, const u_char* /* pkt */) { return 0; @@ -277,57 +540,156 @@ unsigned int ICMP_Analyzer::MemoryAllocation() const + (icmp_conn_val ? icmp_conn_val->MemoryAllocation() : 0); } -ICMP_Echo_Analyzer::ICMP_Echo_Analyzer(Connection* c) -: ICMP_Analyzer(AnalyzerTag::ICMP_Echo, c) - { - } -void ICMP_Echo_Analyzer::NextICMP(double t, const struct icmp* icmpp, int len, - int caplen, const u_char*& data) - { - EventHandlerPtr f = type == ICMP_ECHO ? icmp_echo_request : icmp_echo_reply; +void ICMP_Analyzer::Echo(double t, const struct icmp* icmpp, int len, + int caplen, const u_char*& data, const IP_Hdr* ip_hdr) + { //For handling all Echo related ICMP messages + EventHandlerPtr f = 0; + int ICMP6Flag = 0; + + //printf("Executing: Echo, NextProto:%d\n",ip_hdr->NextProto()); + + if ( ip_hdr->NextProto() == IPPROTO_ICMPV6 ) + { + f = type == ICMP6_ECHO_REQUEST ? icmp_echo_request : icmp_echo_reply; + ICMP6Flag = 1; + } + else + f = type == ICMP_ECHO ? icmp_echo_request : icmp_echo_reply; + if ( ! f ) return; int iid = ntohs(icmpp->icmp_hun.ih_idseq.icd_id); int iseq = ntohs(icmpp->icmp_hun.ih_idseq.icd_seq); + //printf("Check these values: iid:[%d] iseq:[%d]\n",iid,iseq); + BroString* payload = new BroString(data, caplen, 0); val_list* vl = new val_list; vl->append(BuildConnVal()); - vl->append(BuildICMPVal()); + vl->append(BuildICMPVal(ICMP6Flag)); vl->append(new Val(iid, TYPE_COUNT)); vl->append(new Val(iseq, TYPE_COUNT)); vl->append(new StringVal(payload)); + vl->append(new Val(ICMP6Flag, TYPE_BOOL)); ConnectionEvent(f, vl); } -void ICMP_Context_Analyzer::NextICMP(double t, const struct icmp* icmpp, - int len, int caplen, const u_char*& data) + + + + + + + + +void ICMP_Analyzer::Router(double t, const struct icmp* icmpp, int len, + int caplen, const u_char*& data, const IP_Hdr* /*ip_hdr*/) + //For handling router related ICMP messages, { EventHandlerPtr f = 0; - switch ( type ) { - case ICMP_UNREACH: f = icmp_unreachable; break; - case ICMP_TIMXCEED: f = icmp_time_exceeded; break; + int ICMP6Flag = 1; + + switch ( type ) + { + case ND_ROUTER_ADVERT: f = icmp_router_advertisement; break; + + case ND_REDIRECT: + case ND_ROUTER_SOLICIT: + case ICMP6_ROUTER_RENUMBERING: + default: ICMPEvent(icmp_sent,ICMP6Flag); return; + } + + val_list* vl = new val_list; + vl->append(BuildConnVal()); + vl->append(BuildICMPVal(ICMP6Flag)); + vl->append(new Val(ICMP6Flag, TYPE_BOOL)); + + ConnectionEvent(f, vl); } - if ( f ) - { - val_list* vl = new val_list; - vl->append(BuildConnVal()); - vl->append(BuildICMPVal()); - vl->append(new Val(code, TYPE_COUNT)); - vl->append(ExtractICMPContext(caplen, data)); - ConnectionEvent(f, vl); + + + + + + + + + + + + +void ICMP_Analyzer::Context(double t, const struct icmp* icmpp, + int len, int caplen, const u_char*& data, const IP_Hdr* ip_hdr) + {//For handling the ICMP error messages + + EventHandlerPtr f = 0; + int ICMP6Flag = 0; + + + if ( ip_hdr->NextProto() == IPPROTO_ICMPV6 ) //is ip6 + { + + ICMP6Flag = 1; + //printf("Executing: Context for ICMPv6\n"); + + switch ( type ) + { + case ICMP6_DST_UNREACH: f = icmp_unreachable; break; + case ICMP6_PARAM_PROB: f = icmp_error_message; break; + case ICMP6_TIME_EXCEEDED: f = icmp_error_message; break; + case ICMP6_PACKET_TOO_BIG: f = icmp_error_message; break; + } + + if ( f ) + { + val_list* vl = new val_list; + vl->append(BuildConnVal()); //check for ip6 functionality + vl->append(BuildICMPVal(ICMP6Flag)); //check for ip6 functionality + vl->append(new Val(code, TYPE_COUNT)); + vl->append(ExtractICMP6Context(caplen, data)); + + ConnectionEvent(f, vl); + } + + } + else if ( ip_hdr->NextProto() == IPPROTO_ICMP ) + { + //printf("Executing: Context for ICMP\n"); + switch ( type ) + { + case ICMP_UNREACH: f = icmp_unreachable; break; + case ICMP_TIMXCEED: f = icmp_error_message; break; + } + + if ( f ) + { + val_list* vl = new val_list; + vl->append(BuildConnVal()); + vl->append(BuildICMPVal(ICMP6Flag)); + vl->append(new Val(code, TYPE_COUNT)); + vl->append(ExtractICMP4Context(caplen, data)); + + + ConnectionEvent(f, vl); + } + + } + else + { + Weird("ICMP packet, invalid data\n"); //make this more descriptive } } -int ICMP_counterpart(int icmp_type, int icmp_code, bool& is_one_way) + +int ICMP4_counterpart(int icmp_type, int icmp_code, bool& is_one_way) { is_one_way = false; @@ -338,14 +700,77 @@ int ICMP_counterpart(int icmp_type, int icmp_code, bool& is_one_way) switch ( icmp_type ) { case ICMP_ECHO: return ICMP_ECHOREPLY; case ICMP_ECHOREPLY: return ICMP_ECHO; + case ICMP_TSTAMP: return ICMP_TSTAMPREPLY; case ICMP_TSTAMPREPLY: return ICMP_TSTAMP; + case ICMP_IREQ: return ICMP_IREQREPLY; case ICMP_IREQREPLY: return ICMP_IREQ; + case ICMP_ROUTERSOLICIT: return ICMP_ROUTERADVERT; + case ICMP_MASKREQ: return ICMP_MASKREPLY; case ICMP_MASKREPLY: return ICMP_MASKREQ; default: is_one_way = true; return icmp_code; } } + +int ICMP6_counterpart(int icmp_type, int icmp_code, bool& is_one_way) + { + is_one_way = false; + + /**ICMP6 version of the ICMP4_counterpart, under work**/ + //not yet used anywhere, for the context class + + switch ( icmp_type ) { + + + case ICMP6_ECHO_REQUEST: return ICMP6_ECHO_REPLY; + case ICMP6_ECHO_REPLY: return ICMP6_ECHO_REQUEST; + + case ND_ROUTER_SOLICIT: return ND_ROUTER_ADVERT; + case ND_ROUTER_ADVERT: return ND_ROUTER_SOLICIT; + + case ND_NEIGHBOR_SOLICIT: return ND_NEIGHBOR_ADVERT; + case ND_NEIGHBOR_ADVERT: return ND_NEIGHBOR_SOLICIT; + + case MLD_LISTENER_QUERY: return MLD_LISTENER_REPORT; + case MLD_LISTENER_REPORT: return MLD_LISTENER_QUERY; + + case 139: return 140; //ICMP node information query and response respectively (not defined in icmp6.h) + case 140: return 139; + + case 144: return 145; //Home Agent Address Discovery Request Message and reply + case 145: return 144; + + //check the rest of the counterparts + + default: is_one_way = true; return icmp_code; + } + } + + //For mapping ICMP types and codes of v6 to v4. Because we are using same events for both icmpv4 and icmpv6 there is some overlap + //in ICMP types. If this function is used, the name (checked from a table in the scripts) will be incorrect for the listed + //types, but the names will be correct for all ICMP types. + int Type6to4(int icmp_type) + { + switch ( icmp_type ) //For these three values, the type number will be wrong if this is used! + { //easy way to disable this is just to comment all the cases out, and leave only the default. + case ICMP6_DST_UNREACH: return ICMP_UNREACH; break; + case ICMP6_TIME_EXCEEDED: return ICMP_TIMXCEED; break; + case ICMP6_PARAM_PROB: return ICMP_PARAMPROB; break; + + default: return icmp_type; break; + } + } + + int Code6to4(int icmp_code) //not used yet for anything + { + switch ( icmp_code ) + { + default: return icmp_code; break; + } + } + + diff --git a/src/ICMP.h b/src/ICMP.h index 43921f1aac..14f6971915 100644 --- a/src/ICMP.h +++ b/src/ICMP.h @@ -34,15 +34,27 @@ protected: virtual bool IsReuse(double t, const u_char* pkt); virtual unsigned int MemoryAllocation() const; - void ICMPEvent(EventHandlerPtr f); + void ICMPEvent(EventHandlerPtr f, int ICMP6Flag); + + void Echo(double t, const struct icmp* icmpp, int len, + int caplen, const u_char*& data, const IP_Hdr* ip_hdr); + void Context(double t, const struct icmp* icmpp, int len, + int caplen, const u_char*& data, const IP_Hdr* ip_hdr); + void Router(double t, const struct icmp* icmpp, int len, + int caplen, const u_char*& data, const IP_Hdr* ip_hdr); + + + void Describe(ODesc* d) const; - RecordVal* BuildICMPVal(); + RecordVal* BuildICMPVal(int ICMP6Flag); virtual void NextICMP(double t, const struct icmp* icmpp, - int len, int caplen, const u_char*& data); + int len, int caplen, const u_char*& data, const IP_Hdr* ip_hdr); + + RecordVal* ExtractICMP4Context(int len, const u_char*& data); + RecordVal* ExtractICMP6Context(int len, const u_char*& data); - RecordVal* ExtractICMPContext(int len, const u_char*& data); RecordVal* icmp_conn_val; int type; @@ -54,65 +66,24 @@ protected: RuleMatcherState matcher_state; }; -class ICMP_Echo_Analyzer : public ICMP_Analyzer { -public: - ICMP_Echo_Analyzer(Connection* conn); +/*class ICMP4_Analyzer : public ICMP_Analyzer { - static Analyzer* InstantiateAnalyzer(Connection* conn) - { return new ICMP_Echo_Analyzer(conn); } - static bool Available() { return icmp_echo_request || icmp_echo_reply; } -protected: - ICMP_Echo_Analyzer() { } - - virtual void NextICMP(double t, const struct icmp* icmpp, - int len, int caplen, const u_char*& data); }; -class ICMP_Context_Analyzer : public ICMP_Analyzer { -public: - ICMP_Context_Analyzer(AnalyzerTag::Tag tag, Connection* conn) - : ICMP_Analyzer(tag, conn) { } +class ICMP6_Analyzer : public ICMP_Analyzer { -protected: - ICMP_Context_Analyzer() { } - virtual void NextICMP(double t, const struct icmp* icmpp, - int len, int caplen, const u_char*& data); -}; - -class ICMP_TimeExceeded_Analyzer : public ICMP_Context_Analyzer { -public: - ICMP_TimeExceeded_Analyzer(Connection* conn) - : ICMP_Context_Analyzer(AnalyzerTag::ICMP_TimeExceeded, conn) { } - - static Analyzer* InstantiateAnalyzer(Connection* conn) - { return new ICMP_TimeExceeded_Analyzer(conn); } - - static bool Available() { return icmp_time_exceeded; } - -protected: - ICMP_TimeExceeded_Analyzer() { } -}; - -class ICMP_Unreachable_Analyzer : public ICMP_Context_Analyzer { -public: - ICMP_Unreachable_Analyzer(Connection* conn) - : ICMP_Context_Analyzer(AnalyzerTag::ICMP_Unreachable, conn) { } - - static Analyzer* InstantiateAnalyzer(Connection* conn) - { return new ICMP_Unreachable_Analyzer(conn); } - - static bool Available() { return icmp_unreachable; } - -protected: - ICMP_Unreachable_Analyzer() { } -}; +};*/ // Returns the counterpart type to the given type (e.g., the counterpart // to ICMP_ECHOREPLY is ICMP_ECHO). -extern int ICMP_counterpart(int icmp_type, int icmp_code, bool& is_one_way); +//extern int ICMP_counterpart(int icmp_type, int icmp_code, bool& is_one_way); +extern int ICMP4_counterpart(int icmp_type, int icmp_code, bool& is_one_way); +extern int ICMP6_counterpart(int icmp_type, int icmp_code, bool& is_one_way); +extern int Type6to4(int icmp_type); +extern int Code6to4(int icmp_code); #endif diff --git a/src/Sessions.cc b/src/Sessions.cc index fd443d4dcc..de0a1cb488 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -299,6 +299,7 @@ void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr, } const struct ip* ip = (const struct ip*) (pkt + hdr_size); + if ( ip->ip_v == 4 ) { IP_Hdr ip_hdr(ip); @@ -332,6 +333,8 @@ void NetSessions::NextPacketSecondary(double /* t */, const struct pcap_pkthdr* ++num_packets_processed; + + uint32 caplen = hdr->caplen - hdr_size; if ( caplen < sizeof(struct ip) ) { @@ -459,7 +462,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, int proto = ip_hdr->NextProto(); if ( proto != IPPROTO_TCP && proto != IPPROTO_UDP && - proto != IPPROTO_ICMP ) + proto != IPPROTO_ICMP && proto != IPPROTO_ICMPV6) // Added ICMPV6, Matti { dump_this_packet = 1; return; @@ -530,7 +533,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, caplen -= ip_hdr_len; uint32 min_hdr_len = (proto == IPPROTO_TCP) ? sizeof(struct tcphdr) : - (proto == IPPROTO_UDP ? sizeof(struct udphdr) : ICMP_MINLEN); + (proto == IPPROTO_UDP ? sizeof(struct udphdr) : ICMP_MINLEN); //needs checking for ICMPV6?, Matti if ( len < min_hdr_len ) { @@ -582,7 +585,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, const struct icmp* icmpp = (const struct icmp *) data; id.src_port = icmpp->icmp_type; - id.dst_port = ICMP_counterpart(icmpp->icmp_type, + id.dst_port = ICMP4_counterpart(icmpp->icmp_type, icmpp->icmp_code, id.is_one_way); @@ -593,6 +596,23 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, break; } + case IPPROTO_ICMPV6: // new case, identical to ICMP, is this correct?? Matti + { + const struct icmp* icmpp = (const struct icmp *) data; + + id.src_port = icmpp->icmp_type; + //printf("TYPE: %d\n", id.src_port); //testing, Matti + id.dst_port = ICMP6_counterpart(icmpp->icmp_type, + icmpp->icmp_code, + id.is_one_way); + + id.src_port = htons(id.src_port); + id.dst_port = htons(id.dst_port); + + d = &icmp_conns; + break; + + } default: Weird(fmt("unknown_protocol %d", proto), hdr, pkt); return; @@ -611,6 +631,8 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, else { conn = (Connection*) d->Lookup(h); + + if ( ! conn ) { conn = NewConn(h, t, &id, data, proto); @@ -620,6 +642,9 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, else { // We already know that connection. + + + int consistent = CheckConnectionTag(conn); if ( consistent < 0 ) { @@ -773,6 +798,19 @@ Val* NetSessions::BuildHeader(const struct ip* ip) break; } + case IPPROTO_ICMPV6: //Added, Matti + { + const struct icmp* icmpp = (const struct icmp *) data; + RecordVal* icmp_hdr = new RecordVal(icmp_hdr_type); + + //printf("datalen:%d",data_len); //Testing, Matti + + icmp_hdr->Assign(0, new Val(icmpp->icmp_type, TYPE_COUNT)); + + pkt_hdr->Assign(3, icmp_hdr); + break; + } + default: { // This is not a protocol we understand. @@ -968,7 +1006,7 @@ void NetSessions::Remove(Connection* c) ; else if ( ! tcp_conns.RemoveEntry(k) ) - internal_error("connection missing"); + internal_error(fmt("connection missing")); break; case TRANSPORT_UDP: @@ -1157,6 +1195,9 @@ Connection* NetSessions::NewConn(HashKey* k, double t, const ConnID* id, case IPPROTO_UDP: tproto = TRANSPORT_UDP; break; + case IPPROTO_ICMPV6: //TransportProto Hack + tproto = TRANSPORT_ICMP; + break; default: internal_error("unknown transport protocol"); break; @@ -1242,7 +1283,6 @@ bool NetSessions::IsLikelyServerPort(uint32 port, TransportProto proto) const port |= UDP_PORT_MASK; else if ( proto == TRANSPORT_ICMP ) port |= ICMP_PORT_MASK; - return port_cache.find(port) != port_cache.end(); } diff --git a/src/Val.cc b/src/Val.cc index 9a1ee700ff..66770cbdb1 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -872,7 +872,6 @@ PortVal::PortVal(uint32 p, TransportProto port_type) : Val(TYPE_PORT) case TRANSPORT_ICMP: p |= ICMP_PORT_MASK; break; - default: break; // "other" } diff --git a/src/Val.h b/src/Val.h index 5a2faee9d7..b6effcb9e9 100644 --- a/src/Val.h +++ b/src/Val.h @@ -513,9 +513,10 @@ protected: #define NUM_PORT_SPACES 4 #define PORT_SPACE_MASK 0x30000 -#define TCP_PORT_MASK 0x10000 -#define UDP_PORT_MASK 0x20000 -#define ICMP_PORT_MASK 0x30000 +#define TCP_PORT_MASK 0x10000 +#define UDP_PORT_MASK 0x20000 +#define ICMP_PORT_MASK 0x30000 + typedef enum { TRANSPORT_UNKNOWN, TRANSPORT_TCP, TRANSPORT_UDP, TRANSPORT_ICMP, @@ -537,6 +538,7 @@ public: int IsUDP() const; int IsICMP() const; + TransportProto PortType() const { if ( IsTCP() ) diff --git a/src/event.bif b/src/event.bif index 3171b02dde..ffee9244b7 100644 --- a/src/event.bif +++ b/src/event.bif @@ -52,11 +52,21 @@ event udp_request%(u: connection%); event udp_reply%(u: connection%); event udp_contents%(u: connection, is_orig: bool, contents: string%); event udp_session_done%(u: connection%); -event icmp_sent%(c: connection, icmp: icmp_conn%); -event icmp_echo_request%(c: connection, icmp: icmp_conn, id: count, seq: count, payload: string%); -event icmp_echo_reply%(c: connection, icmp: icmp_conn, id: count, seq: count, payload: string%); + +event icmp_sent%(c: connection, icmp: icmp_conn, ICMP6: bool%); +event icmp_echo_request%(c: connection, icmp: icmp_conn, id: count, seq: count, payload: string, ICMP6: bool%); +event icmp_echo_reply%(c: connection, icmp: icmp_conn, id: count, seq: count, payload: string, ICMP6: bool%); event icmp_unreachable%(c: connection, icmp: icmp_conn, code: count, context: icmp_context%); -event icmp_time_exceeded%(c: connection, icmp: icmp_conn, code: count, context: icmp_context%); +event icmp_error_message%(c: connection, icmp: icmp_conn, code: count, context: icmp_context%); +event icmp_router_advertisement%(c: connection, icmp: icmp_conn, ICMP6: bool%); + + + +event icmp6_placeholder%(c: connection, icmp: icmp_conn, ICMP6: bool%); + + + + event net_stats_update%(t: time, ns: net_stats%); event conn_stats%(c: connection, os: endpoint_stats, rs: endpoint_stats%); event conn_weird%(name: string, c: connection%); diff --git a/src/net_util.cc b/src/net_util.cc index e49d575fa0..4c57f12213 100644 --- a/src/net_util.cc +++ b/src/net_util.cc @@ -86,6 +86,14 @@ int udp_checksum(const struct ip* ip, const struct udphdr* up, int len) #ifdef BROv6 int udp6_checksum(const struct ip6_hdr* ip6, const struct udphdr* up, int len) { + /**From RFC for udp4 (same for udp6, except for different pseudoheader which is same as for icmp6) + Computed as the 16-bit one's complement of the one's complement sum of a + pseudo header of information from the IP header, the UDP header, and the + data, padded as needed with zero bytes at the end to make a multiple of + two bytes. If the checksum is cleared to zero, then checksuming is + disabled. If the computed checksum is zero, then this field must be set + to 0xFFFF. + **/ uint32 sum; if ( len % 2 == 1 ) @@ -97,19 +105,61 @@ int udp6_checksum(const struct ip6_hdr* ip6, const struct udphdr* up, int len) sum = ones_complement_checksum((void*) ip6->ip6_src.s6_addr, 16, sum); sum = ones_complement_checksum((void*) ip6->ip6_dst.s6_addr, 16, sum); - sum = ones_complement_checksum((void*) &len, 4, sum); + uint32 l = htonl(len); + sum = ones_complement_checksum((void*) &l, 4, sum); uint32 addl_pseudo = htons(IPPROTO_UDP); sum = ones_complement_checksum((void*) &addl_pseudo, 4, sum); sum = ones_complement_checksum((void*) up, len, sum); + //printf("checksum, calculated for UDP6: %d\n",sum); + return sum; } + + +int icmp6_checksum(const struct icmp* icmpp, const struct ip6_hdr* ip6, int len) + { + /**From RFC + Checksum that covers the ICMPv6 message. This field contains the 16-bit one's + complement of the one's complement sum of the entire ICMPv6 message starting + with the ICMPv6 message type field, prepended with a pseudo-header of IPv6 + header fields. + **/ + uint32 sum; + + if ( len % 2 == 1 ) + // Add in pad byte. + sum += htons(((const u_char*) icmpp)[len - 1] << 8); + else + sum = 0; + + //pseudoheader as in udp6 above + sum = ones_complement_checksum((void*) ip6->ip6_src.s6_addr, 16, sum); + sum = ones_complement_checksum((void*) ip6->ip6_dst.s6_addr, 16, sum); + uint32 l = htonl(len); + sum = ones_complement_checksum((void*) &l, 4, sum); + uint32 addl_pseudo = htons(IPPROTO_ICMPV6); + sum = ones_complement_checksum((void*) &addl_pseudo, 4, sum); + //pseudoheader complete + + sum = ones_complement_checksum((void*) icmpp, len, sum); + + //printf("checksum, calculated for ICMP6: %d\n",sum); + + return sum; + } + #endif int icmp_checksum(const struct icmp* icmpp, int len) { + /**From RFC + Checksum that covers the ICMP message. This is the 16-bit one's + complement of the one's complement sum of the ICMP message starting + with the Type field. The checksum field should be cleared to zero + before generating the checksum. + **/ uint32 sum; - if ( len % 2 == 1 ) // Add in pad byte. sum = htons(((const u_char*) icmpp)[len - 1] << 8); @@ -118,10 +168,15 @@ int icmp_checksum(const struct icmp* icmpp, int len) sum = ones_complement_checksum((void*) icmpp, len, sum); + //printf("checksum, calculated for ICMP4: %d\n",sum); + return sum; } + + + #define CLASS_A 0x00000000 #define CLASS_B 0x80000000 #define CLASS_C 0xc0000000 diff --git a/src/net_util.h b/src/net_util.h index 25b6b293fc..4c17104573 100644 --- a/src/net_util.h +++ b/src/net_util.h @@ -91,6 +91,7 @@ extern int udp_checksum(const struct ip* ip, const struct udphdr* up, int len); #ifdef BROv6 extern int udp6_checksum(const struct ip6_hdr* ip, const struct udphdr* up, int len); +extern int icmp6_checksum(const struct icmp* icmpp, const struct ip6_hdr* ip6, int len); #endif extern int icmp_checksum(const struct icmp* icmpp, int len); From 50181edd8415e2f698a705932af42e0fb0e5c2f9 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 25 Jan 2011 17:54:20 -0800 Subject: [PATCH 002/964] Significant edit pass over ICMPv6 code. Matti, more per mail. --- policy/bro.init | 13 +- policy/icmp.bro | 243 +++++---------- src/Analyzer.cc | 14 - src/AnalyzerTags.h | 2 +- src/DPM.cc | 34 --- src/ICMP.cc | 741 ++++++++++++++++++--------------------------- src/ICMP.h | 43 ++- src/event.bif | 8 +- src/net_util.cc | 39 +-- src/net_util.h | 6 +- 10 files changed, 424 insertions(+), 719 deletions(-) diff --git a/policy/bro.init b/policy/bro.init index e812db0075..5b963169df 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -28,6 +28,8 @@ type icmp_conn: record { itype: count; icode: count; len: count; + + v6: bool; # true if it's an ICMPv6 packet. }; type icmp_hdr: record { @@ -38,12 +40,11 @@ type icmp_context: record { id: conn_id; len: count; proto: count; - frag_offset: count &optional; #no frag offset for IPv6 - bad_hdr_len: bool &optional; - bad_checksum: bool &optional; #no checksum in IPv6 header - MF: bool &optional; #no MF for IPv6 - DF: bool &optional; #no DF for IPv6 - ICMP6Flag: bool; + bad_hdr_len: bool; + bad_checksum: bool; # always true for ICMPv6. + frag_offset: count; # always 0 for IMCPv6. + MF: bool; # always false for IMCPv6. + DF: bool; # always true for ICMPv6. }; type addr_set: set[addr]; diff --git a/policy/icmp.bro b/policy/icmp.bro index d75b2a3731..d5e06c4afa 100644 --- a/policy/icmp.bro +++ b/policy/icmp.bro @@ -1,5 +1,4 @@ # $Id: icmp.bro 6883 2009-08-19 21:08:09Z vern $ -# While using this script, please notice that the last F/T value is the IPv6 Flag @load hot @load weird @@ -18,6 +17,7 @@ export { ICMPAsymPayload, # payload in echo req-resp not the same ICMPConnectionPair, # too many ICMPs between hosts ICMPAddressScan, + ICMPRogueRouter, # v6 advertisement from unknown router # The following isn't presently sufficiently useful due # to cold start and packet drops. @@ -35,6 +35,12 @@ export { const detect_conn_pairs = F &redef; # switch for connection pair const detect_payload_asym = F &redef; # switch for echo payload const conn_pair_threshold = 200 &redef; + + # If the IPv6 routers in a network are all known, they can be + # whitelisted here. If so, any other router seen sending an + # announcement will be reported. If this set remains empty, no such + # detection will be done. + const router_whitelist: set[addr] &redef; } global conn_pair:table[addr] of set[addr] &create_expire = 1 day; @@ -56,16 +62,10 @@ type flow_info: record { payload: string; }; -#Insert whitelisted routers here, Router advertisements from other -#routers will be logged as possible rogue router attacks -const routers_whitelist: table[string] of bool = { - #["fe80::260:97ff:fe07:69ea"] = T, #an example - } &redef &default = F; - const names: table[count] of string = { [0] = "echo_reply", - [1] = "unreach", #icmpv6 - [2] = "too_big", #icmpv6 + [1] = "unreach", # icmpv6 + [2] = "too_big", # icmpv6 [3] = "unreach", [4] = "quench", [5] = "redirect", @@ -80,33 +80,33 @@ const names: table[count] of string = { [16] = "info_reply", [17] = "mask_req", [18] = "mask_reply", - [128] = "echo_req", #icmpv6 - [129] = "echo_reply", #icmpv6 - [130] = "group_memb_query", #icmpv6 - [131] = "group_memb_report", #icmpv6 - [132] = "group_memb_reduct", #icmpv6 - [133] = "router_sol", #icmpv6 - [134] = "router_ad", #icmpv6 - [135] = "neighbor_sol", #icmpv6 - [136] = "neighbor_ad", #icmpv6 - [137] = "redirect", #icmpv6 - [138] = "router_renum", #icmpv6 - [139] = "node_info_query", #icmpv6 - [140] = "node_info_resp", #icmpv6 - [141] = "inv_neigh_disc_sol", #icmpv6 - [142] = "inv_neigh_disc_ad", #icmpv6 - [143] = "mul_lis_report", #icmpv6 - [144] = "home_agent_addr_req", #icmpv6 - [145] = "home_agent_addr_reply",#icmpv6 - [146] = "mobible_prefx_sol", #icmpv6 - [147] = "mobible_prefx_ad", #icmpv6 - [148] = "cert_path_sol", #icmpv6 - [149] = "cert_path_ad", #icmpv6 - [150] = "experimental", #icmpv6 - [151] = "mcast_router_ad", #icmpv6 - [152] = "mcast_router_sol", #icmpv6 - [153] = "mcast_router_term", #icmpv6 - [154] = "fmip", #icmpv6 + [128] = "echo_req", # icmpv6 + [129] = "echo_reply", # icmpv6 + [130] = "group_memb_query", # icmpv6 + [131] = "group_memb_report", # icmpv6 + [132] = "group_memb_reduct", # icmpv6 + [133] = "router_sol", # icmpv6 + [134] = "router_ad", # icmpv6 + [135] = "neighbor_sol", # icmpv6 + [136] = "neighbor_ad", # icmpv6 + [137] = "redirect", # icmpv6 + [138] = "router_renum", # icmpv6 + [139] = "node_info_query", # icmpv6 + [140] = "node_info_resp", # icmpv6 + [141] = "inv_neigh_disc_sol", # icmpv6 + [142] = "inv_neigh_disc_ad", # icmpv6 + [143] = "mul_lis_report", # icmpv6 + [144] = "home_agent_addr_req", # icmpv6 + [145] = "home_agent_addr_reply",# icmpv6 + [146] = "mobible_prefx_sol", # icmpv6 + [147] = "mobible_prefx_ad", # icmpv6 + [148] = "cert_path_sol", # icmpv6 + [149] = "cert_path_ad", # icmpv6 + [150] = "experimental", # icmpv6 + [151] = "mcast_router_ad", # icmpv6 + [152] = "mcast_router_sol", # icmpv6 + [153] = "mcast_router_term", # icmpv6 + [154] = "fmip", # icmpv6 } &default = function(n: count): string { return fmt("icmp-%d", n); }; @@ -116,8 +116,8 @@ const IP_proto_name: table[count] of string = { [2] = "IGMP", [6] = "TCP", [17] = "UDP", - [41] = "IP6", - [58] = "ICMP6", + [41] = "IPV6", + [58] = "ICMPV6", } &default = function(n: count): string { return fmt("%s", n); } &redef; @@ -160,13 +160,38 @@ global flows: table[flow_id] of flow_info &read_expire = 45 sec &expire_func = flush_flow; -event icmp_sent(c: connection, icmp: icmp_conn, ICMP6: bool) +function print_log(c: connection, icmp: icmp_conn, addl: string) { + if ( ! log_details ) + return; - print icmp_file, fmt("%.6f %.6f %s %s %s %s %s %s %s %s %s %s", + print icmp_file, fmt("%.6f %.6f %s %s %s %s %s %s %s %s", network_time(), 0.0, icmp$orig_h, icmp$resp_h, - names[icmp$itype], icmp$itype, icmp$icode, "icmp", - icmp$len, "0", "SH", ICMP6); + names[icmp$itype], icmp$itype, icmp$icode, + icmp$v6 ? "icmp6" : "icmp", icmp$len, addl); + } + +function print_log_with_context(c: connection, icmp: icmp_conn, context: icmp_context, addl: string) + { + # Due to the connection data contained *within* + # them, each log line will contain two connections' worth + # of data. The initial ICMP connection info is the same + # as logged for connections. + + local ctx = fmt("0 EncapPkt: %s %s %s %s %s %s %s %s %s", + context$id$orig_h, context$id$orig_p, + context$id$resp_h, context$id$resp_p, + context$len, IP_proto_name[context$proto], + context$len, context$bad_hdr_len, + context$bad_checksum); + + print_log(c, icmp, ctx); + } + + +event icmp_sent(c: connection, icmp: icmp_conn) + { + print_log(c, icmp, "0 SH"); } event flow_summary(flow: flow_id, last_time: time) @@ -212,63 +237,18 @@ function update_flow(icmp: icmp_conn, id: count, is_orig: bool, payload: string) } -event icmp_error_message(c: connection, icmp: icmp_conn, code: count, context: icmp_context) #for other but the unreach types, which is preserved +event icmp_error_message(c: connection, icmp: icmp_conn, code: count, context: icmp_context) { - - if ( active_connection(context$id) ) - { - # This section allows Bro to act on ICMP error message packets - # that happen in the context of an active connection. It is - # not currently used. - local c2 = connection_record(context$id); - local os = c2$orig$state; - local rs = c2$resp$state; - local is_attempt = - is_tcp_port(c2$id$orig_p) ? - (os == TCP_SYN_SENT && rs == TCP_INACTIVE) : - (os == UDP_ACTIVE && rs == UDP_INACTIVE); - - # Insert action here. - } - - if ( log_details ) - { - # ICMP error message packets are logged here. - # Due to the connection data contained *within* - # them, each log line will contain two connections' worth - # of data. The initial ICMP connection info is the same - # as logged for connections. - print icmp_file, fmt("%.6f %.6f %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s", - network_time(), 0.0, icmp$orig_h, icmp$resp_h, - names[icmp$itype], icmp$itype, icmp$icode, "icmp", - icmp$len, "0", "EncapPkt:", - # This is the encapsulated packet: - context$id$orig_h, context$id$orig_p, - context$id$resp_h, context$id$resp_p, - context$len, IP_proto_name[context$proto], - context$len, context$bad_hdr_len, - context$bad_checksum, context$ICMP6Flag); - } - + print_log_with_context(c, icmp, context, ""); } - - -event icmp6_placeholder(c: connection, icmp: icmp_conn, ICMP6: bool) #just for testing - { - print "icmp6_placeholder triggered"; - } - - -event icmp_echo_request(c: connection, icmp: icmp_conn, id: count, seq: count, payload: string, ICMP6: bool) +event icmp_echo_request(c: connection, icmp: icmp_conn, id: count, seq: count, payload: string) { update_flow(icmp, id, T, payload); - local orig = icmp$orig_h; local resp = icmp$resp_h; - # Simple ping scan detector. if ( detect_scans && (orig !in Scan::distinct_peers || @@ -320,7 +300,7 @@ event icmp_echo_request(c: connection, icmp: icmp_conn, id: count, seq: count, p } event icmp_echo_reply(c: connection, icmp: icmp_conn, id: count, - seq: count, payload: string, ICMP6: bool) + seq: count, payload: string) { # Check payload with the associated flow. @@ -329,8 +309,6 @@ event icmp_echo_reply(c: connection, icmp: icmp_conn, id: count, fid$resp_h = icmp$orig_h; # it's an echo reply. fid$id = id; - - if ( fid !in flows ) { # NOTICE([$note=ICMPUnpairedEchoReply, @@ -357,78 +335,19 @@ event icmp_echo_reply(c: connection, icmp: icmp_conn, id: count, update_flow(icmp, id, F, payload); } - - event icmp_unreachable(c: connection, icmp: icmp_conn, code: count, context: icmp_context) { - - if ( active_connection(context$id) ) - { - # This section allows Bro to act on ICMP-unreachable packets - # that happen in the context of an active connection. It is - # not currently used. - local c2 = connection_record(context$id); - local os = c2$orig$state; - local rs = c2$resp$state; - local is_attempt = - is_tcp_port(c2$id$orig_p) ? - (os == TCP_SYN_SENT && rs == TCP_INACTIVE) : - (os == UDP_ACTIVE && rs == UDP_INACTIVE); - - # Insert action here. - } - - if ( log_details ) - { - # ICMP unreachable packets are the only ones currently - # logged. Due to the connection data contained *within* - # them, each log line will contain two connections' worth - # of data. The initial ICMP connection info is the same - # as logged for connections. - print icmp_file, fmt("%.6f %.6f %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s", - network_time(), 0.0, icmp$orig_h, icmp$resp_h, - names[icmp$itype], icmp$itype, icmp$icode, "icmp", - icmp$len, "0", "EncapPkt:", - # This is the encapsulated packet: - context$id$orig_h, context$id$orig_p, - context$id$resp_h, context$id$resp_p, - context$len, IP_proto_name[context$proto], - context$len, context$bad_hdr_len, - context$bad_checksum, context$ICMP6Flag); - } + print_log_with_context(c, icmp, context, ""); } - - - event icmp_router_advertisement(c: connection, icmp: icmp_conn, ICMP6: bool) + +event icmp_router_advertisement(c: connection, icmp: icmp_conn) { - if ( routers_whitelist[ fmt("%s",icmp$orig_h) ] ) - { - print icmp_file, fmt("%.6f %.6f %s %s %s %s %s %s %s %s %s %s", - network_time(), 0.0, icmp$orig_h, icmp$resp_h, - names[icmp$itype], icmp$itype, icmp$icode, "icmp", - icmp$len, "0", "SH", ICMP6); - } - else - { - print icmp_file, fmt("%.6f %.6f %s %s %s %s %s %s %s %s", - network_time(), 0.0, icmp$orig_h, icmp$resp_h, - names[icmp$itype], "Possible Rogue Router Detected", icmp$itype, icmp$icode, - icmp$len, ICMP6); - } - - } - + print_log(c, icmp, ""); - - - - - - - - - - - - + if ( |router_whitelist| == 0 || icmp$orig_h in router_whitelist ) + return; + + NOTICE([$note=ICMPRogueRouter, + $msg=fmt("rouge router advertisement from %s", icmp$orig_h)]); + } diff --git a/src/Analyzer.cc b/src/Analyzer.cc index a3f6bbfc2c..06b05960b7 100644 --- a/src/Analyzer.cc +++ b/src/Analyzer.cc @@ -48,20 +48,6 @@ const Analyzer::Config Analyzer::analyzer_configs[] = { { AnalyzerTag::ICMP, "ICMP", ICMP_Analyzer::InstantiateAnalyzer, ICMP_Analyzer::Available, 0, false }, - /*{ AnalyzerTag::ICMP_TimeExceeded, "ICMP_TIMEEXCEEDED", - ICMP_TimeExceeded_Analyzer::InstantiateAnalyzer, - ICMP_TimeExceeded_Analyzer::Available, 0, false }, - { AnalyzerTag::ICMP_Unreachable, "ICMP_UNREACHABLE", - ICMP_Unreachable_Analyzer::InstantiateAnalyzer, - ICMP_Unreachable_Analyzer::Available, 0, false }, - { AnalyzerTag::ICMP_Echo, "ICMP_ECHO", - ICMP_Echo_Analyzer::InstantiateAnalyzer, - ICMP_Echo_Analyzer::Available, 0, false },*/ - - - - - { AnalyzerTag::TCP, "TCP", TCP_Analyzer::InstantiateAnalyzer, TCP_Analyzer::Available, 0, false }, { AnalyzerTag::UDP, "UDP", UDP_Analyzer::InstantiateAnalyzer, diff --git a/src/AnalyzerTags.h b/src/AnalyzerTags.h index eb18a03a73..231b39364a 100644 --- a/src/AnalyzerTags.h +++ b/src/AnalyzerTags.h @@ -22,7 +22,7 @@ namespace AnalyzerTag { PIA_TCP, PIA_UDP, // Transport-layer analyzers. - ICMP,/* ICMP_TimeExceeded, ICMP_Unreachable, ICMP_Echo*/ TCP, UDP, + ICMP, TCP, UDP, // Application-layer analyzers (hand-written). BitTorrent, BitTorrentTracker, diff --git a/src/DPM.cc b/src/DPM.cc index 7be9376b9f..b9afb15196 100644 --- a/src/DPM.cc +++ b/src/DPM.cc @@ -214,42 +214,8 @@ bool DPM::BuildInitialAnalyzerTree(TransportProto proto, Connection* conn, break; case TRANSPORT_ICMP: { - const struct icmp* icmpp = (const struct icmp *) data; - - - //Old code, moving to having only one ICMP analyzer - /*switch ( icmpp->icmp_type ) { - - case ICMP_ECHO: - case ICMP_ECHOREPLY: - if ( ICMP_Echo_Analyzer::Available() ) - { - root = new ICMP_Echo_Analyzer(conn); - DBG_DPD(conn, "activated ICMP Echo analyzer"); - } - break; - - case ICMP_UNREACH: - if ( ICMP_Unreachable_Analyzer::Available() ) - { - root = new ICMP_Unreachable_Analyzer(conn); - DBG_DPD(conn, "activated ICMP Unreachable analyzer"); - } - break; - - case ICMP_TIMXCEED: - if ( ICMP_TimeExceeded_Analyzer::Available() ) - { - root = new ICMP_TimeExceeded_Analyzer(conn); - DBG_DPD(conn, "activated ICMP Time Exceeded analyzer"); - } - break; - }*/ - //if ( ! root ) - root = new ICMP_Analyzer(conn); DBG_DPD(conn, "activated ICMP analyzer"); - analyzed = true; break; } diff --git a/src/ICMP.cc b/src/ICMP.cc index b83cf76a40..1ec1d2901c 100644 --- a/src/ICMP.cc +++ b/src/ICMP.cc @@ -11,15 +11,12 @@ #include - - ICMP_Analyzer::ICMP_Analyzer(Connection* c) : TransportLayerAnalyzer(AnalyzerTag::ICMP, c) { icmp_conn_val = 0; c->SetInactivityTimeout(icmp_inactivity_timeout); request_len = reply_len = -1; - } ICMP_Analyzer::ICMP_Analyzer(AnalyzerTag::Tag tag, Connection* c) @@ -37,7 +34,7 @@ void ICMP_Analyzer::Done() matcher_state.FinishEndpointMatcher(); } -void ICMP_Analyzer::DeliverPacket(int arg_len, const u_char* data, +void ICMP_Analyzer::DeliverPacket(int len, const u_char* data, bool is_orig, int seq, const IP_Hdr* ip, int caplen) { assert(ip); @@ -50,42 +47,39 @@ void ICMP_Analyzer::DeliverPacket(int arg_len, const u_char* data, // Subtract off the common part of ICMP header. PacketContents(data + 8, min(len, caplen) - 8); - const struct icmp* icmpp = (const struct icmp*) data; - len = arg_len; + assert(caplen >= len); // Should have been caught earlier already. + + if ( ! ignore_checksums ) + { + int chksum; - //We need a separate calculation for ICMP6 checksums, pseudoheader is appended to the - //ICMP6 checksum calculation, which is different from ICMP4 #ifdef BROv6 - - - if (ip->NextProto() == IPPROTO_ICMPV6 && ! ignore_checksums && - caplen >= len && icmp6_checksum(icmpp,ip->IP6_Hdr(),len )!= 0xffff ) + switch ( ip->NextProto() ) { - Weird("bad_ICMP6_checksum"); - return; + case IPPROTO_ICMP: + chksum = icmp_checksum(icmpp, len); + break; + + case IPPROTO_ICMPV6: + chksum = icmp6_checksum(icmpp, ip->IP6_Hdr(), len); + break; + + default: + internal_error("unexpected IP proto in ICMP analyzer"); } - else if (ip->NextProto() != IPPROTO_ICMPV6 && ! ignore_checksums && - caplen >= len && icmp_checksum(icmpp, len) != 0xffff ) - { - Weird("bad_ICMP_checksum"); - return; - } - - - #else - - if ( ! ignore_checksums && caplen >= len && - icmp_checksum(icmpp, len) != 0xffff ) - { - Weird("bad_ICMP_checksum"); - return; - } + # Classic v4 version. + chksum = icmp_checksum(icmpp, len); #endif - + if ( chksum != 0xffff ) + { + Weird("bad_ICMP6_checksum"); + return; + } + } Conn()->SetLastTime(current_timestamp); @@ -95,127 +89,104 @@ void ICMP_Analyzer::DeliverPacket(int arg_len, const u_char* data, matcher_state.InitEndpointMatcher(this, ip, len, is_orig, 0); } - type = icmpp->icmp_type; - code = icmpp->icmp_code; - - // Move past common portion of ICMP header. //OK for ICMPv6? + // Move past common portion of ICMP header. data += 8; caplen -= 8; len -= 8; - int& len_stat = is_orig ? request_len : reply_len; - if ( len_stat < 0 ) - len_stat = len; + if ( ip->NextProto() == IPPROTO_ICMP ) + NextICMP4(current_timestamp, icmpp, len, caplen, data, ip); else - len_stat += len; + NextICMP6(current_timestamp, icmpp, len, caplen, data, ip); - NextICMP(current_timestamp, icmpp, len, caplen, data, ip); if ( rule_matcher ) matcher_state.Match(Rule::PAYLOAD, data, len, is_orig, false, false, true); } - - -/********************Generic analyzer for all ICMP4/ICMP6******************************/ -void ICMP_Analyzer::NextICMP(double t , const struct icmp* icmpp , int len , int caplen, +void ICMP_Analyzer::NextICMP4(double t, const struct icmp* icmpp, int len, int caplen, const u_char*& data, const IP_Hdr* ip_hdr ) { - int ICMP6Flag = 0; - - //printf("Executing: ICMP_Analyzer::NextICMP\n"); - //printf("New analyzer structure\n"); - - if ( ip_hdr->NextProto() == IPPROTO_ICMPV6 ) + switch ( icmpp->icmp_type ) { - //printf("ICMP6!\n"); - ICMP6Flag = 1; - - switch (type) //Add new ICMP6 functions here, you can also use codes to narrow the area of single functions. - { - //All the echo stuff here - case ICMP6_ECHO_REQUEST: - case ICMP6_ECHO_REPLY: + case ICMP_ECHO: + case ICMP_ECHOREPLY: Echo(t, icmpp, len, caplen, data, ip_hdr); break; + case ICMP_UNREACH: + case ICMP_TIMXCEED: + Context4(t, icmpp, len, caplen, data, ip_hdr); + break; - //Error messages all have the same structure for their context, and are handled by the same function. - case ICMP6_PARAM_PROB: - case ICMP6_TIME_EXCEEDED: - case ICMP6_PACKET_TOO_BIG: - case ICMP6_DST_UNREACH: - Context(t, icmpp, len, caplen, data, ip_hdr); - break; - - //All router related stuff should eventually be handled by the Router() - case ND_REDIRECT: - case ND_ROUTER_SOLICIT: - case ICMP6_ROUTER_RENUMBERING: - case ND_ROUTER_ADVERT: - Router(t, icmpp, len, caplen, data, ip_hdr); //currently only logs the router stuff for other than router_advert - break; - - /* listed for convenience - case ICMP6_PARAM_PROB: break; - case MLD_LISTENER_QUERY: break; - case MLD_LISTENER_REPORT: break; - case MLD_LISTENER_REDUCTION: break; - case ND_NEIGHBOR_SOLICIT: break; - case ND_NEIGHBOR_ADVERT: break; - case ND_REDIRECT: break; - case ICMP6_ROUTER_RENUMBERING: break; - case ND_NEIGHBOR_SOLICIT: break; - case ND_NEIGHBOR_ADVERT: break; - case ICMP6_TIME_EXCEEDED: break; - */ - - default: ICMPEvent(icmp_sent, ICMP6Flag); break; - } + default: + ICMPEvent(icmp_sent, icmpp, len, 0); break; } - else if ( ip_hdr->NextProto() == IPPROTO_ICMP ) - { + } - switch (type) //Add new ICMP4 functions here - { - case ICMP_ECHO: - case ICMP_ECHOREPLY: +#ifdef BROv6 +void ICMP_Analyzer::NextICMP6(double t, const struct icmp* icmpp, int len, int caplen, + const u_char*& data, const IP_Hdr* ip_hdr ) + { + switch ( icmpp->icmp_type ) + { + // Echo types. + case ICMP6_ECHO_REQUEST: + case ICMP6_ECHO_REPLY: Echo(t, icmpp, len, caplen, data, ip_hdr); break; - case ICMP_UNREACH: - case ICMP_TIMXCEED: - Context(t, icmpp, len, caplen, data, ip_hdr); + // Error messages all have the same structure for their context, + // and are handled by the same function. + case ICMP6_PARAM_PROB: + case ICMP6_TIME_EXCEEDED: + case ICMP6_PACKET_TOO_BIG: + case ICMP6_DST_UNREACH: + Context6(t, icmpp, len, caplen, data, ip_hdr); break; - default: ICMPEvent(icmp_sent, ICMP6Flag); break; - } - + // Router related messages. + case ND_REDIRECT: + case ND_ROUTER_SOLICIT: + case ICMP6_ROUTER_RENUMBERING: + case ND_ROUTER_ADVERT: + Router(t, icmpp, len, caplen, data, ip_hdr); + break; +#if 0 + // Currently not specifically implemented. + case ICMP6_PARAM_PROB: + case MLD_LISTENER_QUERY: + case MLD_LISTENER_REPORT: + case MLD_LISTENER_REDUCTION: + case ND_NEIGHBOR_SOLICIT: + case ND_NEIGHBOR_ADVERT: + case ND_REDIRECT: + case ICMP6_ROUTER_RENUMBERING: + case ND_NEIGHBOR_SOLICIT: + case ND_NEIGHBOR_ADVERT: + case ICMP6_TIME_EXCEEDED: +#endif + default: + ICMPEvent(icmp_sent, icmpp, len, 1); + break; } - else - Weird("Malformed ip header"); - } + } +#endif - -void ICMP_Analyzer::ICMPEvent(EventHandlerPtr f, int ICMP6Flag) +void ICMP_Analyzer::ICMPEvent(EventHandlerPtr f, const struct icmp* icmpp, int len, int icmpv6) { if ( ! f ) - return; - + return; val_list* vl = new val_list; vl->append(BuildConnVal()); - vl->append(BuildICMPVal(ICMP6Flag)); - //if ( f == icmp_sent ) //for now, testing purposes - vl->append(new Val(ICMP6Flag, TYPE_BOOL)); - + vl->append(BuildICMPVal(icmpp, len, icmpv6)); ConnectionEvent(f, vl); } - -RecordVal* ICMP_Analyzer::BuildICMPVal(int ICMP6Flag) +RecordVal* ICMP_Analyzer::BuildICMPVal(const struct icmp* icmpp, int len, int icmpv6) { if ( ! icmp_conn_val ) { @@ -223,15 +194,10 @@ RecordVal* ICMP_Analyzer::BuildICMPVal(int ICMP6Flag) icmp_conn_val->Assign(0, new AddrVal(Conn()->OrigAddr())); icmp_conn_val->Assign(1, new AddrVal(Conn()->RespAddr())); - - if ( ICMP6Flag == 1 ) - icmp_conn_val->Assign(2, new Val(Type6to4(type), TYPE_COUNT)); //to avoid errors in getting the message type *name* right on the scripting level, type number will be different from true ipv6 - else - icmp_conn_val->Assign(2, new Val(type, TYPE_COUNT)); - - - icmp_conn_val->Assign(3, new Val(code, TYPE_COUNT)); + icmp_conn_val->Assign(2, new Val(icmpp->icmp_type, TYPE_COUNT)); + icmp_conn_val->Assign(3, new Val(icmpp->icmp_code, TYPE_COUNT)); icmp_conn_val->Assign(4, new Val(len, TYPE_COUNT)); + icmp_conn_val->Assign(5, new Val(icmpv6, TYPE_BOOL)); } Ref(icmp_conn_val); @@ -239,15 +205,74 @@ RecordVal* ICMP_Analyzer::BuildICMPVal(int ICMP6Flag) return icmp_conn_val; } +TransportProto ICMP_Analyzer::GetContextProtocol(const IP_Hdr* ip_hdr, uint32* src_port, uint32* dst_port) + { + const u_char* transport_hdr; + uint32 ip_hdr_len = ip_hdr->HdrLen(); + bool ip4 = ip_hdr->IP4_Hdr(); + + if ( ip4 ) + transport_hdr = ((u_char *) ip_hdr->IP4_Hdr() + ip_hdr_len); + else + transport_hdr = ((u_char *) ip_hdr->IP6_Hdr() + ip_hdr_len); + + TransportProto proto; + + switch ( ip_hdr->NextProto() ) { + case 1: proto = TRANSPORT_ICMP; break; + case 6: proto = TRANSPORT_TCP; break; + case 17: proto = TRANSPORT_UDP; break; + case 58: proto = TRANSPORT_ICMP; //TransportProto Hack // XXX What's this? + default: proto = TRANSPORT_UNKNOWN; break; + } + + switch ( proto ) { + case TRANSPORT_ICMP: + { + const struct icmp* icmpp = + (const struct icmp *) transport_hdr; + bool is_one_way; // dummy + *src_port = ntohs(icmpp->icmp_type); + + if ( ip4 ) + *dst_port = ntohs(ICMP4_counterpart(icmpp->icmp_type, + icmpp->icmp_code, is_one_way)); + else + *dst_port = ntohs(ICMP6_counterpart(icmpp->icmp_type, + icmpp->icmp_code, is_one_way)); + + break; + } + + case TRANSPORT_TCP: + { + const struct tcphdr* tp = + (const struct tcphdr *) transport_hdr; + *src_port = ntohs(tp->th_sport); + *dst_port = ntohs(tp->th_dport); + break; + } + + case TRANSPORT_UDP: + { + const struct udphdr* up = + (const struct udphdr *) transport_hdr; + *src_port = ntohs(up->uh_sport); + *dst_port = ntohs(up->uh_dport); + break; + } + + default: + *src_port = *dst_port = ntohs(0); + } + + return proto; + } + RecordVal* ICMP_Analyzer::ExtractICMP4Context(int len, const u_char*& data) { - /** - * For use only with ICMP4, ICMPV6 context extraction is still non-functional - */ - const IP_Hdr ip_hdr_data((const struct ip*) data); const IP_Hdr* ip_hdr = &ip_hdr_data; - int ICMP6Flag = 0; uint32 ip_hdr_len = ip_hdr->HdrLen(); @@ -257,132 +282,82 @@ RecordVal* ICMP_Analyzer::ExtractICMP4Context(int len, const u_char*& data) uint32 src_addr, dst_addr,src_addr2, dst_addr2; uint32 src_port, dst_port; - if ( ip_hdr_len < sizeof(struct ip) || ip_hdr_len > uint32(len) ) - { // We don't have an entire IP header. - bad_hdr_len = 1; - ip_len = frag_offset = 0; - DF = MF = bad_checksum = 0; - src_addr = dst_addr = 0; - src_port = dst_port = 0; - } - - else - { - bad_hdr_len = 0; - ip_len = ip_hdr->TotalLen(); - bad_checksum = ones_complement_checksum((void*) ip_hdr->IP4_Hdr(), ip_hdr_len, 0) != 0xffff; - - src_addr = ip_hdr->SrcAddr4(); - dst_addr = ip_hdr->DstAddr4(); - - switch ( ip_hdr->NextProto() ) { - case 1: proto = TRANSPORT_ICMP; break; - case 6: proto = TRANSPORT_TCP; break; - case 17: proto = TRANSPORT_UDP; break; - - // Default uses TRANSPORT_UNKNOWN, per initialization above. - } - - uint32 frag_field = ip_hdr->FragField(); - DF = ip_hdr->DF(); - MF = frag_field & 0x2000; - frag_offset = frag_field & /* IP_OFFMASK not portable */ 0x1fff; - - const u_char* transport_hdr = ((u_char *) ip_hdr->IP4_Hdr() + ip_hdr_len); - - if ( uint32(len) < ip_hdr_len + 4 ) //what is this value for ipv6? - { - // 4 above is the magic number meaning that both - // port numbers are included in the ICMP. - bad_hdr_len = 1; - src_port = dst_port = 0; - } - - switch ( proto ) { - case TRANSPORT_ICMP: - { - const struct icmp* icmpp = - (const struct icmp *) transport_hdr; - bool is_one_way; // dummy - src_port = ntohs(icmpp->icmp_type); - dst_port = ntohs(ICMP4_counterpart(icmpp->icmp_type, - icmpp->icmp_code, - is_one_way)); - } - break; - - case TRANSPORT_TCP: - { - const struct tcphdr* tp = - (const struct tcphdr *) transport_hdr; - src_port = ntohs(tp->th_sport); - dst_port = ntohs(tp->th_dport); - } - break; - - case TRANSPORT_UDP: - { - const struct udphdr* up = - (const struct udphdr *) transport_hdr; - src_port = ntohs(up->uh_sport); - dst_port = ntohs(up->uh_dport); - } - break; - - default: - src_port = dst_port = ntohs(0); - } - } - - RecordVal* iprec = new RecordVal(icmp_context); - RecordVal* id_val = new RecordVal(conn_id); - - id_val->Assign(0, new AddrVal(src_addr)); - id_val->Assign(1, new PortVal(src_port, proto)); - id_val->Assign(2, new AddrVal(dst_addr)); - id_val->Assign(3, new PortVal(dst_port, proto)); - iprec->Assign(0, id_val); - - iprec->Assign(1, new Val(ip_len, TYPE_COUNT)); - iprec->Assign(2, new Val(proto, TYPE_COUNT)); - iprec->Assign(3, new Val(frag_offset, TYPE_COUNT)); - iprec->Assign(4, new Val(bad_hdr_len, TYPE_BOOL)); - iprec->Assign(5, new Val(bad_checksum, TYPE_BOOL)); - iprec->Assign(6, new Val(MF, TYPE_BOOL)); - iprec->Assign(7, new Val(DF, TYPE_BOOL)); - iprec->Assign(8, new Val(ICMP6Flag, TYPE_BOOL)); - - return iprec; + if ( ip_hdr_len < sizeof(struct ip) || ip_hdr_len > uint32(len) ) + { + // We don't have an entire IP header. + bad_hdr_len = 1; + ip_len = frag_offset = 0; + DF = MF = bad_checksum = 0; + src_addr = dst_addr = 0; + src_port = dst_port = 0; } + else + { + bad_hdr_len = 0; + ip_len = ip_hdr->TotalLen(); + bad_checksum = ones_complement_checksum((void*) ip_hdr->IP4_Hdr(), ip_hdr_len, 0) != 0xffff; + src_addr = ip_hdr->SrcAddr4(); + dst_addr = ip_hdr->DstAddr4(); + uint32 frag_field = ip_hdr->FragField(); + DF = ip_hdr->DF(); + MF = frag_field & 0x2000; + frag_offset = frag_field & /* IP_OFFMASK not portable */ 0x1fff; + + if ( uint32(len) >= ip_hdr_len + 4 ) + proto = GetContextProtocol(ip_hdr, &src_port, &dst_port); + else + { + // 4 above is the magic number meaning that both + // port numbers are included in the ICMP. + src_port = dst_port = 0; + bad_hdr_len = 1; + } + } + + RecordVal* iprec = new RecordVal(icmp_context); + RecordVal* id_val = new RecordVal(conn_id); + + id_val->Assign(0, new AddrVal(src_addr)); + id_val->Assign(1, new PortVal(src_port, proto)); + id_val->Assign(2, new AddrVal(dst_addr)); + id_val->Assign(3, new PortVal(dst_port, proto)); + + iprec->Assign(0, id_val); + iprec->Assign(1, new Val(ip_len, TYPE_COUNT)); + iprec->Assign(2, new Val(proto, TYPE_COUNT)); + iprec->Assign(3, new Val(bad_hdr_len, TYPE_BOOL)); + iprec->Assign(4, new Val(bad_checksum, TYPE_BOOL)); + iprec->Assign(5, new Val(frag_offset, TYPE_COUNT)); + iprec->Assign(6, new Val(MF, TYPE_BOOL)); + iprec->Assign(7, new Val(DF, TYPE_BOOL)); + + return iprec; + } RecordVal* ICMP_Analyzer::ExtractICMP6Context(int len, const u_char*& data) { - /** - * For use with ICMP6 error message context extraction (possibly very frail function) - */ - const IP_Hdr ip_hdr_data((const struct ip6_hdr*) data); const IP_Hdr* ip_hdr = &ip_hdr_data; - int ICMP6Flag = 1; int DF = 0, MF = 0, bad_hdr_len = 0, bad_checksum = 0; + TransportProto proto = TRANSPORT_UNKNOWN; uint32 ip_hdr_len = ip_hdr->HdrLen(); //should always be 40 uint32* src_addr; uint32* dst_addr; uint32 ip_len, frag_offset = 0; - TransportProto proto = TRANSPORT_UNKNOWN; uint32 src_port, dst_port; - if ( ip_hdr_len < sizeof(struct ip6_hdr) || ip_hdr_len != 40 ) + if ( ip_hdr_len < sizeof(struct ip6_hdr) || ip_hdr_len != 40 ) // XXX What's the 2nd part doing? { bad_hdr_len = 1; ip_len = 0; src_addr = dst_addr = 0; src_port = dst_port = 0; } + else { ip_len = ip_hdr->TotalLen(); @@ -390,62 +365,15 @@ RecordVal* ICMP_Analyzer::ExtractICMP6Context(int len, const u_char*& data) src_addr = (uint32 *) ip_hdr->SrcAddr(); dst_addr = (uint32 *) ip_hdr->DstAddr(); - - - switch ( ip_hdr->NextProto() ) { - case 1: proto = TRANSPORT_ICMP; break; - case 6: proto = TRANSPORT_TCP; break; - case 17: proto = TRANSPORT_UDP; break; - case 58: proto = TRANSPORT_ICMP; break; //TransportProto Hack - - // Default uses TRANSPORT_UNKNOWN, per initialization above. - } - - - const u_char* transport_hdr = ((u_char *)ip_hdr->IP6_Hdr() + ip_hdr_len); - - if ( uint32(len) < ip_hdr_len + 4 ) + if ( uint32(len) >= ip_hdr_len + 4 ) + proto = GetContextProtocol(ip_hdr, &src_port, &dst_port); + else { // 4 above is the magic number meaning that both // port numbers are included in the ICMP. - bad_hdr_len = 1; src_port = dst_port = 0; + bad_hdr_len = 1; } - - switch ( proto ) { - case TRANSPORT_ICMP: - { - const struct icmp* icmpp = - (const struct icmp *) transport_hdr; - bool is_one_way; // dummy - src_port = ntohs(icmpp->icmp_type); - dst_port = ntohs(ICMP6_counterpart(icmpp->icmp_type, - icmpp->icmp_code, - is_one_way)); - } - break; - - case TRANSPORT_TCP: - { - const struct tcphdr* tp = - (const struct tcphdr *) transport_hdr; - src_port = ntohs(tp->th_sport); - dst_port = ntohs(tp->th_dport); - } - break; - - case TRANSPORT_UDP: - { - const struct udphdr* up = - (const struct udphdr *) transport_hdr; - src_port = ntohs(up->uh_sport); - dst_port = ntohs(up->uh_dport); - } - break; - - default: - src_port = dst_port = ntohs(0); - } } RecordVal* iprec = new RecordVal(icmp_context); @@ -459,7 +387,7 @@ RecordVal* ICMP_Analyzer::ExtractICMP6Context(int len, const u_char*& data) iprec->Assign(0, id_val); iprec->Assign(1, new Val(ip_len, TYPE_COUNT)); - //TransportProto Hack + //TransportProto Hack // XXX Likewise. if ( ip_hdr->NextProto() == 58 || 17 ) //if the encap packet is ICMPv6 we force this... (cause there is no IGMP (by that name) for ICMPv6), rather ugly hack once more { iprec->Assign(2, new Val(58, TYPE_COUNT)); @@ -469,28 +397,18 @@ RecordVal* ICMP_Analyzer::ExtractICMP6Context(int len, const u_char*& data) iprec->Assign(2, new Val(proto, TYPE_COUNT)); } - iprec->Assign(3, new Val(frag_offset, TYPE_COUNT)); //NA for ip6 - iprec->Assign(4, new Val(bad_hdr_len, TYPE_BOOL)); - iprec->Assign(5, new Val(bad_checksum, TYPE_BOOL)); - iprec->Assign(6, new Val(MF, TYPE_BOOL)); //NA for ip6 - iprec->Assign(7, new Val(DF, TYPE_BOOL)); //NA for ip6 - iprec->Assign(8, new Val(ICMP6Flag, TYPE_BOOL)); //ICMP6Flag + iprec->Assign(3, new Val(bad_hdr_len, TYPE_BOOL)); + + // The following are not available for IPv6. + iprec->Assign(4, new Val(0, TYPE_BOOL)); // bad_checksum + iprec->Assign(5, new Val(frag_offset, TYPE_COUNT)); // frag_offset + iprec->Assign(6, new Val(0, TYPE_BOOL)); // MF + iprec->Assign(7, new Val(1, TYPE_BOOL)); // DF return iprec; } - - - - - - - - - - - bool ICMP_Analyzer::IsReuse(double /* t */, const u_char* /* pkt */) { return 0; @@ -504,10 +422,12 @@ void ICMP_Analyzer::Describe(ODesc* d) const d->AddSP(")"); d->Add(dotted_addr(Conn()->OrigAddr())); +#if 0 d->Add("."); d->Add(type); d->Add("."); d->Add(code); +#endif d->SP(); d->AddSP("->"); @@ -543,19 +463,16 @@ unsigned int ICMP_Analyzer::MemoryAllocation() const void ICMP_Analyzer::Echo(double t, const struct icmp* icmpp, int len, int caplen, const u_char*& data, const IP_Hdr* ip_hdr) - { //For handling all Echo related ICMP messages + { + // For handling all Echo related ICMP messages EventHandlerPtr f = 0; - int ICMP6Flag = 0; - - //printf("Executing: Echo, NextProto:%d\n",ip_hdr->NextProto()); +#ifdef BROv6 if ( ip_hdr->NextProto() == IPPROTO_ICMPV6 ) - { - f = type == ICMP6_ECHO_REQUEST ? icmp_echo_request : icmp_echo_reply; - ICMP6Flag = 1; - } + f = (icmpp->icmp_type == ICMP6_ECHO_REQUEST) ? icmp_echo_request : icmp_echo_reply; else - f = type == ICMP_ECHO ? icmp_echo_request : icmp_echo_reply; +#endif + f = (icmpp->icmp_type == ICMP_ECHO) ? icmp_echo_request : icmp_echo_reply; if ( ! f ) return; @@ -563,137 +480,110 @@ void ICMP_Analyzer::Echo(double t, const struct icmp* icmpp, int len, int iid = ntohs(icmpp->icmp_hun.ih_idseq.icd_id); int iseq = ntohs(icmpp->icmp_hun.ih_idseq.icd_seq); - //printf("Check these values: iid:[%d] iseq:[%d]\n",iid,iseq); - BroString* payload = new BroString(data, caplen, 0); val_list* vl = new val_list; vl->append(BuildConnVal()); - vl->append(BuildICMPVal(ICMP6Flag)); + vl->append(BuildICMPVal(icmpp, len, ip_hdr->NextProto() != IPPROTO_ICMP)); vl->append(new Val(iid, TYPE_COUNT)); vl->append(new Val(iseq, TYPE_COUNT)); vl->append(new StringVal(payload)); - vl->append(new Val(ICMP6Flag, TYPE_BOOL)); ConnectionEvent(f, vl); } - - - - - - - - void ICMP_Analyzer::Router(double t, const struct icmp* icmpp, int len, int caplen, const u_char*& data, const IP_Hdr* /*ip_hdr*/) - //For handling router related ICMP messages, { EventHandlerPtr f = 0; - int ICMP6Flag = 1; - switch ( type ) + switch ( icmpp->icmp_type ) { - case ND_ROUTER_ADVERT: f = icmp_router_advertisement; break; + case ND_ROUTER_ADVERT: + f = icmp_router_advertisement; + break; case ND_REDIRECT: case ND_ROUTER_SOLICIT: case ICMP6_ROUTER_RENUMBERING: - default: ICMPEvent(icmp_sent,ICMP6Flag); return; + default: + ICMPEvent(icmp_sent, icmpp, len, 1); + return; } val_list* vl = new val_list; vl->append(BuildConnVal()); - vl->append(BuildICMPVal(ICMP6Flag)); - vl->append(new Val(ICMP6Flag, TYPE_BOOL)); + vl->append(BuildICMPVal(icmpp, len, 1)); ConnectionEvent(f, vl); } - - - - - - - - - - - - -void ICMP_Analyzer::Context(double t, const struct icmp* icmpp, - int len, int caplen, const u_char*& data, const IP_Hdr* ip_hdr) - {//For handling the ICMP error messages - +void ICMP_Analyzer::Context4(double t, const struct icmp* icmpp, + int len, int caplen, const u_char*& data, const IP_Hdr* ip_hdr) + { EventHandlerPtr f = 0; - int ICMP6Flag = 0; - - if ( ip_hdr->NextProto() == IPPROTO_ICMPV6 ) //is ip6 - { - - ICMP6Flag = 1; - //printf("Executing: Context for ICMPv6\n"); - - switch ( type ) - { - case ICMP6_DST_UNREACH: f = icmp_unreachable; break; - case ICMP6_PARAM_PROB: f = icmp_error_message; break; - case ICMP6_TIME_EXCEEDED: f = icmp_error_message; break; - case ICMP6_PACKET_TOO_BIG: f = icmp_error_message; break; - } - - if ( f ) - { - val_list* vl = new val_list; - vl->append(BuildConnVal()); //check for ip6 functionality - vl->append(BuildICMPVal(ICMP6Flag)); //check for ip6 functionality - vl->append(new Val(code, TYPE_COUNT)); - vl->append(ExtractICMP6Context(caplen, data)); - - ConnectionEvent(f, vl); - } - - } - else if ( ip_hdr->NextProto() == IPPROTO_ICMP ) - { - //printf("Executing: Context for ICMP\n"); - switch ( type ) - { - case ICMP_UNREACH: f = icmp_unreachable; break; - case ICMP_TIMXCEED: f = icmp_error_message; break; - } - - if ( f ) - { - val_list* vl = new val_list; - vl->append(BuildConnVal()); - vl->append(BuildICMPVal(ICMP6Flag)); - vl->append(new Val(code, TYPE_COUNT)); - vl->append(ExtractICMP4Context(caplen, data)); - - - ConnectionEvent(f, vl); - } - - } - else + switch ( icmpp->icmp_type ) { - Weird("ICMP packet, invalid data\n"); //make this more descriptive + case ICMP_UNREACH: + f = icmp_unreachable; + break; + + case ICMP_TIMXCEED: + f = icmp_error_message; + break; + } + + if ( f ) + { + val_list* vl = new val_list; + vl->append(BuildConnVal()); + vl->append(BuildICMPVal(icmpp, len, 0)); + vl->append(new Val(icmpp->icmp_code, TYPE_COUNT)); + vl->append(ExtractICMP4Context(caplen, data)); + ConnectionEvent(f, vl); } } +#ifdef BROv6 +void ICMP_Analyzer::Context6(double t, const struct icmp* icmpp, + int len, int caplen, const u_char*& data, const IP_Hdr* ip_hdr) + { + EventHandlerPtr f = 0; + + switch ( icmpp->icmp_type ) + { + case ICMP6_DST_UNREACH: + f = icmp_unreachable; + break; + + case ICMP6_PARAM_PROB: + case ICMP6_TIME_EXCEEDED: + case ICMP6_PACKET_TOO_BIG: + f = icmp_error_message; + break; + } + + if ( f ) + { + val_list* vl = new val_list; + vl->append(BuildConnVal()); + vl->append(BuildICMPVal(icmpp, len, 1)); + vl->append(new Val(icmpp->icmp_code, TYPE_COUNT)); + vl->append(ExtractICMP6Context(caplen, data)); + ConnectionEvent(f, vl); + } + } +#endif int ICMP4_counterpart(int icmp_type, int icmp_code, bool& is_one_way) { is_one_way = false; - // return the counterpart type if one exists. This allows us + // Return the counterpart type if one exists. This allows us // to track corresponding ICMP requests/replies. // Note that for the two-way ICMP messages, icmp_code is // always 0 (RFC 792). @@ -720,57 +610,30 @@ int ICMP6_counterpart(int icmp_type, int icmp_code, bool& is_one_way) { is_one_way = false; - /**ICMP6 version of the ICMP4_counterpart, under work**/ - //not yet used anywhere, for the context class - switch ( icmp_type ) { + case ICMP6_ECHO_REQUEST: return ICMP6_ECHO_REPLY; + case ICMP6_ECHO_REPLY: return ICMP6_ECHO_REQUEST; + case ND_ROUTER_SOLICIT: return ND_ROUTER_ADVERT; + case ND_ROUTER_ADVERT: return ND_ROUTER_SOLICIT; - case ICMP6_ECHO_REQUEST: return ICMP6_ECHO_REPLY; - case ICMP6_ECHO_REPLY: return ICMP6_ECHO_REQUEST; + case ND_NEIGHBOR_SOLICIT: return ND_NEIGHBOR_ADVERT; + case ND_NEIGHBOR_ADVERT: return ND_NEIGHBOR_SOLICIT; - case ND_ROUTER_SOLICIT: return ND_ROUTER_ADVERT; - case ND_ROUTER_ADVERT: return ND_ROUTER_SOLICIT; + case MLD_LISTENER_QUERY: return MLD_LISTENER_REPORT; + case MLD_LISTENER_REPORT: return MLD_LISTENER_QUERY; - case ND_NEIGHBOR_SOLICIT: return ND_NEIGHBOR_ADVERT; - case ND_NEIGHBOR_ADVERT: return ND_NEIGHBOR_SOLICIT; + // ICMP node information query and response respectively (not defined in + // icmp6.h) + case 139: return 140; + case 140: return 139; - case MLD_LISTENER_QUERY: return MLD_LISTENER_REPORT; - case MLD_LISTENER_REPORT: return MLD_LISTENER_QUERY; - - case 139: return 140; //ICMP node information query and response respectively (not defined in icmp6.h) - case 140: return 139; - - case 144: return 145; //Home Agent Address Discovery Request Message and reply + // Home Agent Address Discovery Request Message and reply + case 144: return 145; case 145: return 144; - //check the rest of the counterparts + // TODO: Add further counterparts. default: is_one_way = true; return icmp_code; } } - - //For mapping ICMP types and codes of v6 to v4. Because we are using same events for both icmpv4 and icmpv6 there is some overlap - //in ICMP types. If this function is used, the name (checked from a table in the scripts) will be incorrect for the listed - //types, but the names will be correct for all ICMP types. - int Type6to4(int icmp_type) - { - switch ( icmp_type ) //For these three values, the type number will be wrong if this is used! - { //easy way to disable this is just to comment all the cases out, and leave only the default. - case ICMP6_DST_UNREACH: return ICMP_UNREACH; break; - case ICMP6_TIME_EXCEEDED: return ICMP_TIMXCEED; break; - case ICMP6_PARAM_PROB: return ICMP_PARAMPROB; break; - - default: return icmp_type; break; - } - } - - int Code6to4(int icmp_code) //not used yet for anything - { - switch ( icmp_code ) - { - default: return icmp_code; break; - } - } - - diff --git a/src/ICMP.h b/src/ICMP.h index 14f6971915..aed814d2b1 100644 --- a/src/ICMP.h +++ b/src/ICMP.h @@ -34,7 +34,7 @@ protected: virtual bool IsReuse(double t, const u_char* pkt); virtual unsigned int MemoryAllocation() const; - void ICMPEvent(EventHandlerPtr f, int ICMP6Flag); + void ICMPEvent(EventHandlerPtr f, const struct icmp* icmpp, int len, int icmpv6); void Echo(double t, const struct icmp* icmpp, int len, int caplen, const u_char*& data, const IP_Hdr* ip_hdr); @@ -43,47 +43,40 @@ protected: void Router(double t, const struct icmp* icmpp, int len, int caplen, const u_char*& data, const IP_Hdr* ip_hdr); - - void Describe(ODesc* d) const; - RecordVal* BuildICMPVal(int ICMP6Flag); + RecordVal* BuildICMPVal(const struct icmp* icmpp, int len, int icmpv6); - virtual void NextICMP(double t, const struct icmp* icmpp, - int len, int caplen, const u_char*& data, const IP_Hdr* ip_hdr); + void NextICMP4(double t, const struct icmp* icmpp, int len, int caplen, + const u_char*& data, const IP_Hdr* ip_hdr ); RecordVal* ExtractICMP4Context(int len, const u_char*& data); + + void Context4(double t, const struct icmp* icmpp, int len, int caplen, + const u_char*& data, const IP_Hdr* ip_hdr); + + TransportProto GetContextProtocol(const IP_Hdr* ip_hdr, uint32* src_port, + uint32* dst_port); + +#ifdef BROv6 + void NextICMP6(double t, const struct icmp* icmpp, int len, int caplen, + const u_char*& data, const IP_Hdr* ip_hdr ); + RecordVal* ExtractICMP6Context(int len, const u_char*& data); + void Context6(double t, const struct icmp* icmpp, int len, int caplen, + const u_char*& data, const IP_Hdr* ip_hdr); +#endif RecordVal* icmp_conn_val; - int type; - int code; - int len; - int request_len, reply_len; RuleMatcherState matcher_state; }; -/*class ICMP4_Analyzer : public ICMP_Analyzer { - - - -}; - -class ICMP6_Analyzer : public ICMP_Analyzer { - - - -};*/ - // Returns the counterpart type to the given type (e.g., the counterpart // to ICMP_ECHOREPLY is ICMP_ECHO). -//extern int ICMP_counterpart(int icmp_type, int icmp_code, bool& is_one_way); extern int ICMP4_counterpart(int icmp_type, int icmp_code, bool& is_one_way); extern int ICMP6_counterpart(int icmp_type, int icmp_code, bool& is_one_way); -extern int Type6to4(int icmp_type); -extern int Code6to4(int icmp_code); #endif diff --git a/src/event.bif b/src/event.bif index ffee9244b7..d0cee28c03 100644 --- a/src/event.bif +++ b/src/event.bif @@ -53,12 +53,12 @@ event udp_reply%(u: connection%); event udp_contents%(u: connection, is_orig: bool, contents: string%); event udp_session_done%(u: connection%); -event icmp_sent%(c: connection, icmp: icmp_conn, ICMP6: bool%); -event icmp_echo_request%(c: connection, icmp: icmp_conn, id: count, seq: count, payload: string, ICMP6: bool%); -event icmp_echo_reply%(c: connection, icmp: icmp_conn, id: count, seq: count, payload: string, ICMP6: bool%); +event icmp_sent%(c: connection, icmp: icmp_conn%); +event icmp_echo_request%(c: connection, icmp: icmp_conn, id: count, seq: count, payload: string%); +event icmp_echo_reply%(c: connection, icmp: icmp_conn, id: count, seq: count, payload: string%); event icmp_unreachable%(c: connection, icmp: icmp_conn, code: count, context: icmp_context%); event icmp_error_message%(c: connection, icmp: icmp_conn, code: count, context: icmp_context%); -event icmp_router_advertisement%(c: connection, icmp: icmp_conn, ICMP6: bool%); +event icmp_router_advertisement%(c: connection, icmp: icmp_conn%); diff --git a/src/net_util.cc b/src/net_util.cc index 4c57f12213..75dfd929c2 100644 --- a/src/net_util.cc +++ b/src/net_util.cc @@ -86,14 +86,8 @@ int udp_checksum(const struct ip* ip, const struct udphdr* up, int len) #ifdef BROv6 int udp6_checksum(const struct ip6_hdr* ip6, const struct udphdr* up, int len) { - /**From RFC for udp4 (same for udp6, except for different pseudoheader which is same as for icmp6) - Computed as the 16-bit one's complement of the one's complement sum of a - pseudo header of information from the IP header, the UDP header, and the - data, padded as needed with zero bytes at the end to make a multiple of - two bytes. If the checksum is cleared to zero, then checksuming is - disabled. If the computed checksum is zero, then this field must be set - to 0xFFFF. - **/ + // UDP over IPv6 uses the same checksum function as over IPv4 but a + // different pseuod-header over which it is computed. uint32 sum; if ( len % 2 == 1 ) @@ -108,23 +102,18 @@ int udp6_checksum(const struct ip6_hdr* ip6, const struct udphdr* up, int len) uint32 l = htonl(len); sum = ones_complement_checksum((void*) &l, 4, sum); uint32 addl_pseudo = htons(IPPROTO_UDP); + sum = ones_complement_checksum((void*) &addl_pseudo, 4, sum); sum = ones_complement_checksum((void*) up, len, sum); - //printf("checksum, calculated for UDP6: %d\n",sum); - return sum; } int icmp6_checksum(const struct icmp* icmpp, const struct ip6_hdr* ip6, int len) { - /**From RFC - Checksum that covers the ICMPv6 message. This field contains the 16-bit one's - complement of the one's complement sum of the entire ICMPv6 message starting - with the ICMPv6 message type field, prepended with a pseudo-header of IPv6 - header fields. - **/ + // ICMP6 uses the same checksum function as over ICMP4 but a different + // pseuod-header over which it is computed. uint32 sum; if ( len % 2 == 1 ) @@ -133,19 +122,17 @@ int icmp6_checksum(const struct icmp* icmpp, const struct ip6_hdr* ip6, int len) else sum = 0; - //pseudoheader as in udp6 above + // Pseudo-header as for UDP over IPv6 above. sum = ones_complement_checksum((void*) ip6->ip6_src.s6_addr, 16, sum); sum = ones_complement_checksum((void*) ip6->ip6_dst.s6_addr, 16, sum); uint32 l = htonl(len); sum = ones_complement_checksum((void*) &l, 4, sum); + uint32 addl_pseudo = htons(IPPROTO_ICMPV6); sum = ones_complement_checksum((void*) &addl_pseudo, 4, sum); - //pseudoheader complete sum = ones_complement_checksum((void*) icmpp, len, sum); - //printf("checksum, calculated for ICMP6: %d\n",sum); - return sum; } @@ -153,12 +140,6 @@ int icmp6_checksum(const struct icmp* icmpp, const struct ip6_hdr* ip6, int len) int icmp_checksum(const struct icmp* icmpp, int len) { - /**From RFC - Checksum that covers the ICMP message. This is the 16-bit one's - complement of the one's complement sum of the ICMP message starting - with the Type field. The checksum field should be cleared to zero - before generating the checksum. - **/ uint32 sum; if ( len % 2 == 1 ) // Add in pad byte. @@ -168,15 +149,9 @@ int icmp_checksum(const struct icmp* icmpp, int len) sum = ones_complement_checksum((void*) icmpp, len, sum); - //printf("checksum, calculated for ICMP4: %d\n",sum); - return sum; } - - - - #define CLASS_A 0x00000000 #define CLASS_B 0x80000000 #define CLASS_C 0xc0000000 diff --git a/src/net_util.h b/src/net_util.h index 4c17104573..9a6c12b3c3 100644 --- a/src/net_util.h +++ b/src/net_util.h @@ -88,12 +88,14 @@ extern int ones_complement_checksum(const void* p, int b, uint32 sum); extern int tcp_checksum(const struct ip* ip, const struct tcphdr* tp, int len); extern int udp_checksum(const struct ip* ip, const struct udphdr* up, int len); +extern int icmp_checksum(const struct icmp* icmpp, int len); + #ifdef BROv6 extern int udp6_checksum(const struct ip6_hdr* ip, const struct udphdr* up, int len); -extern int icmp6_checksum(const struct icmp* icmpp, const struct ip6_hdr* ip6, int len); +extern int icmp6_checksum(const struct icmp* icmpp, const struct ip6_hdr* ip6, + int len); #endif -extern int icmp_checksum(const struct icmp* icmpp, int len); // Given an address in host order, returns its "classical network prefix", // also in host order. From 9c388a18091d07e5adf5fa3b621c83fd46d5890a Mon Sep 17 00:00:00 2001 From: Gregor Maier Date: Wed, 3 Aug 2011 14:12:47 -0700 Subject: [PATCH 003/964] Adding support to de-capsulate tunnels. Checkpoint. Decapsulation happens after IP Defragmentation. The "identity" of the enclosing tunnel (the "parent") is added to the connection record of the child (tunneled) connection as an optional field $tunnel_parent. --- policy/bro.init | 6 ++++ src/CMakeLists.txt | 1 + src/Conn.cc | 5 ++- src/Conn.h | 3 +- src/ConnCompressor.cc | 4 +-- src/Sessions.cc | 46 ++++++++++++++++++++------- src/Sessions.h | 7 ++++- src/TunnelHandler.cc | 48 ++++++++++++++++++++++++++++ src/TunnelHandler.h | 73 +++++++++++++++++++++++++++++++++++++++++++ src/types.bif | 7 +++++ 10 files changed, 183 insertions(+), 17 deletions(-) create mode 100644 src/TunnelHandler.cc create mode 100644 src/TunnelHandler.h diff --git a/policy/bro.init b/policy/bro.init index fda8cfd6f4..17607a7113 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -81,6 +81,11 @@ type endpoint_stats: record { type AnalyzerID: count; +type tunnel_parent_t: record { + cid: conn_id; + tunnel_type: tunneltype_t; +}; + type connection: record { id: conn_id; orig: endpoint; @@ -92,6 +97,7 @@ type connection: record { hot: count; # how hot; 0 = don't know or not hot history: string; uid: string; + tunnel_parent: tunnel_parent_t &optional; }; type SYN_packet: record { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1a5f096f70..e79fad4ca0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -388,6 +388,7 @@ set(bro_SRCS Timer.cc Traverse.cc Trigger.cc + TunnelHandler.cc Type.cc UDP.cc Val.cc diff --git a/src/Conn.cc b/src/Conn.cc index bab032cbd0..67e337fda9 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -139,7 +139,7 @@ unsigned int Connection::external_connections = 0; IMPLEMENT_SERIAL(Connection, SER_CONNECTION); -Connection::Connection(NetSessions* s, HashKey* k, double t, const ConnID* id) +Connection::Connection(NetSessions* s, HashKey* k, double t, const ConnID* id, RecordVal *arg_tunnel_parent) { sessions = s; key = k; @@ -183,6 +183,8 @@ Connection::Connection(NetSessions* s, HashKey* k, double t, const ConnID* id) uid = 0; // Will set later. + tunnel_parent = arg_tunnel_parent; + if ( conn_timer_mgr ) { ++external_connections; @@ -370,6 +372,7 @@ RecordVal* Connection::BuildConnVal() char tmp[20]; conn_val->Assign(9, new StringVal(uitoa_n(uid, tmp, sizeof(tmp), 62))); + conn_val->Assign(10, tunnel_parent); } if ( root_analyzer ) diff --git a/src/Conn.h b/src/Conn.h index 8f817fd003..e22c0b83ec 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -86,7 +86,7 @@ class Analyzer; class Connection : public BroObj { public: - Connection(NetSessions* s, HashKey* k, double t, const ConnID* id); + Connection(NetSessions* s, HashKey* k, double t, const ConnID* id, RecordVal *arg_tunnel_parent); virtual ~Connection(); // Invoked when connection is about to be removed. Use Ref(this) @@ -335,6 +335,7 @@ protected: double inactivity_timeout; RecordVal* conn_val; LoginConn* login_conn; // either nil, or this + RecordVal* tunnel_parent; // nil if not tunneled int suppress_event; // suppress certain events to once per conn. unsigned int installed_status_timer:1; diff --git a/src/ConnCompressor.cc b/src/ConnCompressor.cc index e173463205..2d617b0fc4 100644 --- a/src/ConnCompressor.cc +++ b/src/ConnCompressor.cc @@ -521,7 +521,7 @@ Connection* ConnCompressor::Instantiate(HashKey* key, PendingConn* pending) // Fake the first packet. const IP_Hdr* faked_pkt = PendingConnToPacket(pending); Connection* new_conn = sessions->NewConn(key, pending->time, &conn_id, - faked_pkt->Payload(), IPPROTO_TCP); + faked_pkt->Payload(), IPPROTO_TCP, 0); if ( ! new_conn ) { @@ -574,7 +574,7 @@ Connection* ConnCompressor::Instantiate(double t, HashKey* key, conn_id.dst_port = tp->th_dport; Connection* new_conn = - sessions->NewConn(key, t, &conn_id, ip->Payload(), IPPROTO_TCP); + sessions->NewConn(key, t, &conn_id, ip->Payload(), IPPROTO_TCP, 0); if ( ! new_conn ) { diff --git a/src/Sessions.cc b/src/Sessions.cc index 1678f6798f..48fab1bff4 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -33,6 +33,7 @@ #include "DPM.h" #include "PacketSort.h" +#include "TunnelHandler.h" // These represent NetBIOS services on ephemeral ports. They're numbered // so that we can use a single int to hold either an actual TCP/UDP server @@ -128,6 +129,12 @@ NetSessions::NetSessions() arp_analyzer = new ARP_Analyzer(); else arp_analyzer = 0; + + + if ( 1 ) + tunnel_handler = new TunnelHandler(this); + else + tunnel_handler = 0; } NetSessions::~NetSessions() @@ -433,14 +440,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, if ( discarder && discarder->NextPacket(ip_hdr, len, caplen) ) return; - int proto = ip_hdr->NextProto(); - if ( proto != IPPROTO_TCP && proto != IPPROTO_UDP && - proto != IPPROTO_ICMP ) - { - dump_this_packet = 1; - return; - } - FragReassembler* f = 0; uint32 frag_field = ip_hdr->FragField(); @@ -474,6 +473,23 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, } } + TunnelInfo *tunnel_info = tunnel_handler->DecapsulateTunnel(ip_hdr, len, caplen, hdr, pkt); + if (tunnel_info) + { + ip4 = tunnel_info->child->IP4_Hdr(); + ip_hdr = tunnel_info->child; + len -= tunnel_info->hdr_len; + caplen -= tunnel_info->hdr_len; + } + + int proto = ip_hdr->NextProto(); + if ( proto != IPPROTO_TCP && proto != IPPROTO_UDP && + proto != IPPROTO_ICMP ) + { + dump_this_packet = 1; + return; + } + len -= ip_hdr_len; // remove IP header caplen -= ip_hdr_len; @@ -561,7 +577,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, conn = (Connection*) d->Lookup(h); if ( ! conn ) { - conn = NewConn(h, t, &id, data, proto); + conn = NewConn(h, t, &id, data, proto, tunnel_info); if ( conn ) d->Insert(h, conn); } @@ -581,7 +597,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, conn->Event(connection_reused, 0); Remove(conn); - conn = NewConn(h, t, &id, data, proto); + conn = NewConn(h, t, &id, data, proto, tunnel_info); if ( conn ) d->Insert(h, conn); } @@ -609,6 +625,8 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, record_packet, record_content, hdr, pkt, hdr_size); + if ( tunnel_info ) + delete tunnel_info; if ( f ) { // Above we already recorded the fragment in its entirety. @@ -1045,13 +1063,17 @@ void NetSessions::GetStats(SessionStats& s) const } Connection* NetSessions::NewConn(HashKey* k, double t, const ConnID* id, - const u_char* data, int proto) + const u_char* data, int proto, TunnelInfo* tunnel_info) { // FIXME: This should be cleaned up a bit, it's too protocol-specific. // But I'm not yet sure what the right abstraction for these things is. int src_h = ntohs(id->src_port); int dst_h = ntohs(id->dst_port); int flags = 0; + RecordVal *tunnel_parent = 0; + + if ( tunnel_info ) + tunnel_parent = tunnel_info->GetRecordVal(); // Hmm... This is not great. TransportProto tproto; @@ -1098,7 +1120,7 @@ Connection* NetSessions::NewConn(HashKey* k, double t, const ConnID* id, id = &flip_id; } - Connection* conn = new Connection(this, k, t, id); + Connection* conn = new Connection(this, k, t, id, tunnel_parent); conn->SetTransport(tproto); dpm->BuildInitialAnalyzerTree(tproto, conn, data); diff --git a/src/Sessions.h b/src/Sessions.h index 6adc333282..9551ba5254 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -28,6 +28,9 @@ class Discarder; class SteppingStoneManager; class PacketFilter; +class TunnelHandler; +class TunnelInfo; + class PacketSortElement; struct SessionStats { @@ -144,7 +147,7 @@ protected: friend class TimerMgrExpireTimer; Connection* NewConn(HashKey* k, double t, const ConnID* id, - const u_char* data, int proto); + const u_char* data, int proto, TunnelInfo *tunnel_info); // Check whether the tag of the current packet is consistent with // the given connection. Returns: @@ -214,6 +217,8 @@ protected: int num_packets_processed; PacketProfiler* pkt_profiler; + TunnelHandler *tunnel_handler; + // We may use independent timer managers for different sets of related // activity. The managers are identified by an unique tag. typedef std::map TimerMgrMap; diff --git a/src/TunnelHandler.cc b/src/TunnelHandler.cc new file mode 100644 index 0000000000..c739403ad8 --- /dev/null +++ b/src/TunnelHandler.cc @@ -0,0 +1,48 @@ +// $Id: Sessions.cc 7075 2010-09-13 02:39:38Z vern $ +// +// See the file "COPYING" in the main distribution directory for copyright. + + +#include "config.h" + +#include + +#include +#include + +#include "TunnelHandler.h" +#include "Conn.h" +#include "Sessions.h" + + +TunnelHandler::TunnelHandler(NetSessions *arg_s) + { + s = arg_s; + } + +TunnelInfo* TunnelHandler::DecapsulateTunnel(const IP_Hdr *ip_hdr, int len, int caplen, + const struct pcap_pkthdr* hdr, const u_char* const pkt) + { + TunnelInfo *tunnel_info = 0; + switch (ip_hdr->NextProto()) { +#ifdef BROv6 + case IPPROTO_IPV6: /* 6in4 and 6to4 */ + if (len < (int)sizeof(struct ip6_hdr) || caplen < (int)sizeof(struct ip6_hdr)) + { + s->Weird("truncated_header", hdr, pkt); + return 0; + } + // TODO: check if IP6 header makes sense + tunnel_info = new TunnelInfo(); + tunnel_info->child = new IP_Hdr((const struct ip6_hdr*)ip_hdr->Payload()); + tunnel_info->tunneltype = BifEnum::IP6inIP; + tunnel_info->hdr_len = ip_hdr->HdrLen(); + tunnel_info->SetParentIPs(ip_hdr); + return tunnel_info; + break; +#endif + default: + break; + } /* end switch */ + return 0; + } diff --git a/src/TunnelHandler.h b/src/TunnelHandler.h new file mode 100644 index 0000000000..aa4cae0a39 --- /dev/null +++ b/src/TunnelHandler.h @@ -0,0 +1,73 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef tunnelhandler_h +#define tunnelhandler_h + +#include "IP.h" +#include "Conn.h" +#include "Sessions.h" +#include "Val.h" + + +class TunnelInfo { +public: + TunnelInfo() + { + child = 0; + tunneltype = BifEnum::NONE; + hdr_len = 0; + parent.src_addr = parent.dst_addr = 0; + parent.src_port = parent.dst_port = 0; + parent.is_one_way = 0; + } + ~TunnelInfo() + { + if (child) delete child; + } + + void SetParentIPs(const IP_Hdr *ip_hdr) + { + parent.src_addr = ip_hdr->SrcAddr(); + parent.dst_addr = ip_hdr->DstAddr(); + } + void SetParentPorts(uint32 src_port, uint32 dst_port) + { + parent.src_port = src_port; + parent.dst_port = dst_port; + } + + RecordVal* GetRecordVal() const + { + RecordVal *rv = new RecordVal(BifType::Record::tunnel_parent_t); + + RecordVal* id_val = new RecordVal(conn_id); + id_val->Assign(0, new AddrVal(parent.src_addr)); + id_val->Assign(1, new PortVal(ntohs(parent.src_port), TRANSPORT_UNKNOWN)); + id_val->Assign(2, new AddrVal(parent.dst_addr)); + id_val->Assign(3, new PortVal(ntohs(parent.dst_port), TRANSPORT_UNKNOWN)); + rv->Assign(0, id_val); + rv->Assign(1, new EnumVal(tunneltype, BifType::Enum::tunneltype_t)); + return rv; + } + + IP_Hdr *child; + ConnID parent; + int hdr_len; + BifEnum::tunneltype_t tunneltype; +}; + +class TunnelHandler { +public: + TunnelHandler(NetSessions *arg_s); + ~TunnelHandler(); + + TunnelInfo* DecapsulateTunnel(const IP_Hdr* ip_hdr, int len, int caplen, + /* need those for passing them back to NetSessions::Weird() */ + const struct pcap_pkthdr* hdr, const u_char* const pkt); + +protected: + NetSessions *s; +}; + + +#endif diff --git a/src/types.bif b/src/types.bif index 8bc5ab8510..d44f177b82 100644 --- a/src/types.bif +++ b/src/types.bif @@ -167,3 +167,10 @@ enum ID %{ %} module GLOBAL; + +enum tunneltype_t %{ + NONE, + IP6inIP, +%} + +type tunnel_parent_t: record; From 8910cd2dcaf308f9febc575a38dcf32ed47be1d6 Mon Sep 17 00:00:00 2001 From: Gregor Maier Date: Thu, 4 Aug 2011 15:25:13 -0700 Subject: [PATCH 004/964] Adding support to de-capsulate tunnels. Checkpoint. Re-organizing code. Adding UDP tunnel handlers. Using policy level redef'able consts to tune behavior. UDP port settings not working yet. --- policy/bro.init | 36 +++++++++-- src/Sessions.cc | 25 +++++--- src/TunnelHandler.cc | 144 +++++++++++++++++++++++++++++++++++++++---- src/TunnelHandler.h | 34 +++++++--- src/const.bif | 4 ++ src/types.bif | 13 +++- 6 files changed, 216 insertions(+), 40 deletions(-) diff --git a/policy/bro.init b/policy/bro.init index 17607a7113..8be0ab754a 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -81,10 +81,14 @@ type endpoint_stats: record { type AnalyzerID: count; -type tunnel_parent_t: record { - cid: conn_id; - tunnel_type: tunneltype_t; -}; +module Tunnel; +export { + type parent_t: record { + cid: conn_id; + tunnel_type: tunneltype_t; + }; +} # end export +module GLOBAL; type connection: record { id: conn_id; @@ -97,7 +101,7 @@ type connection: record { hot: count; # how hot; 0 = don't know or not hot history: string; uid: string; - tunnel_parent: tunnel_parent_t &optional; + tunnel_parent: Tunnel::parent_t &optional; }; type SYN_packet: record { @@ -1489,6 +1493,28 @@ const skip_http_data = F &redef; # UDP tunnels. See also: udp_tunnel_port, policy/udp-tunnel.bro. const parse_udp_tunnels = F &redef; +module Tunnel; +export { + # Whether to decapsulate IP tunnels (IPinIP, 6in4, 6to4) + const decapsulate_ip = F &redef; + + # Whether to decapsulate URDP tunnels (e.g., Teredo, IPv4 in UDP) + const decapsulate_udp = F &redef; + + # If decapsulating UDP: the set of ports for which to do so + const udp_tunnel_ports: set[port] = { + 3544/udp, # Teredo + 5072/udp, # AYIAY + } &redef; + + # If udp_tunnel_allports is T udp_tunnel_ports is ignored and we + # check every UDP packet for tunnels. + const udp_tunnel_allports = F &redef; +} # end export +module GLOBAL; + + + # Load the site utilities. @load utils/site diff --git a/src/Sessions.cc b/src/Sessions.cc index 48fab1bff4..41a7f4c52d 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -131,10 +131,11 @@ NetSessions::NetSessions() arp_analyzer = 0; - if ( 1 ) + if ( BifConst::Tunnel::decapsulate_ip || BifConst::Tunnel::decapsulate_udp ) tunnel_handler = new TunnelHandler(this); else tunnel_handler = 0; + printf("tunnel_handler: %p\n", tunnel_handler); } NetSessions::~NetSessions() @@ -473,13 +474,20 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, } } - TunnelInfo *tunnel_info = tunnel_handler->DecapsulateTunnel(ip_hdr, len, caplen, hdr, pkt); - if (tunnel_info) + len -= ip_hdr_len; // remove IP header + caplen -= ip_hdr_len; + + TunnelInfo *tunnel_info = 0; + if ( tunnel_handler ) { - ip4 = tunnel_info->child->IP4_Hdr(); - ip_hdr = tunnel_info->child; - len -= tunnel_info->hdr_len; - caplen -= tunnel_info->hdr_len; + tunnel_info = tunnel_handler->DecapsulateTunnel(ip_hdr, len, caplen, hdr, pkt); + if (tunnel_info) + { + ip4 = tunnel_info->child->IP4_Hdr(); + ip_hdr = tunnel_info->child; + len -= tunnel_info->hdr_len; + caplen -= tunnel_info->hdr_len; + } } int proto = ip_hdr->NextProto(); @@ -490,9 +498,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, return; } - len -= ip_hdr_len; // remove IP header - caplen -= ip_hdr_len; - uint32 min_hdr_len = (proto == IPPROTO_TCP) ? sizeof(struct tcphdr) : (proto == IPPROTO_UDP ? sizeof(struct udphdr) : ICMP_MINLEN); diff --git a/src/TunnelHandler.cc b/src/TunnelHandler.cc index c739403ad8..3256894cd0 100644 --- a/src/TunnelHandler.cc +++ b/src/TunnelHandler.cc @@ -18,31 +18,151 @@ TunnelHandler::TunnelHandler(NetSessions *arg_s) { s = arg_s; + for (int i=0; i< 65536; i++) + udp_ports[i] = 0; + udp_ports[3544] = 1; + udp_ports[5072] = 1; } TunnelInfo* TunnelHandler::DecapsulateTunnel(const IP_Hdr *ip_hdr, int len, int caplen, const struct pcap_pkthdr* hdr, const u_char* const pkt) { TunnelInfo *tunnel_info = 0; + switch (ip_hdr->NextProto()) { #ifdef BROv6 case IPPROTO_IPV6: /* 6in4 and 6to4 */ - if (len < (int)sizeof(struct ip6_hdr) || caplen < (int)sizeof(struct ip6_hdr)) - { - s->Weird("truncated_header", hdr, pkt); - return 0; - } - // TODO: check if IP6 header makes sense - tunnel_info = new TunnelInfo(); - tunnel_info->child = new IP_Hdr((const struct ip6_hdr*)ip_hdr->Payload()); - tunnel_info->tunneltype = BifEnum::IP6inIP; - tunnel_info->hdr_len = ip_hdr->HdrLen(); - tunnel_info->SetParentIPs(ip_hdr); - return tunnel_info; + if ( BifConst::Tunnel::decapsulate_ip ) + { + if (len < (int)sizeof(struct ip6_hdr) || caplen < (int)sizeof(struct ip6_hdr)) + { + s->Weird("truncated_header", hdr, pkt); + return 0; + } + // TODO: check if IP6 header makes sense + tunnel_info = new TunnelInfo(); + tunnel_info->child = new IP_Hdr((const struct ip6_hdr*)ip_hdr->Payload()); + tunnel_info->tunneltype = BifEnum::Tunnel::IP6inIP; + tunnel_info->hdr_len = tunnel_info->child->HdrLen(); + tunnel_info->SetParentIPs(ip_hdr); + return tunnel_info; + } break; #endif + // TODO: IP in IP. Find test traces first. IP proto 0 and/or 4 + case IPPROTO_UDP: + if ( BifConst::Tunnel::decapsulate_udp ) + { + if (len < (int)sizeof(struct udphdr) || caplen < (int)sizeof(struct udphdr)) + { + // No weird here. Main packet processing will raise it. + return 0; + } + return HandleUDP(ip_hdr, len, caplen); + } + + break; default: break; } /* end switch */ return 0; } + +TunnelInfo* TunnelHandler::HandleUDP(const IP_Hdr *ip_hdr, int len, int caplen) + { + // We already know we that we have a valid UDP header + const u_char *data = ip_hdr->Payload(); + const struct udphdr* uh = (const struct udphdr*)data; + IP_Hdr *cand_ip_hdr = 0; + BifEnum::Tunnel::tunneltype_t tunneltype = BifEnum::Tunnel::NONE; + + int hdr_len = sizeof(struct udphdr); + data += hdr_len; + + int datalen = (int)ntohs(uh->uh_ulen); + datalen = min(datalen, min(len, caplen)); + datalen -= hdr_len; + + if ( BifConst::Tunnel::udp_tunnel_allports || + udp_ports[ntohs(uh->uh_sport)] || + udp_ports[ntohs(uh->uh_dport)] ) + { + cand_ip_hdr = LookForIPHdr(data, datalen); + if (cand_ip_hdr) + { + tunneltype = (cand_ip_hdr->IP4_Hdr()) ? + BifEnum::Tunnel::IP4inUDP : BifEnum::Tunnel::IP6inUDP; + } + else if (datalen >= 8) + { + // Look for AYIAY tunnels + u_char id_byte = data[0]; + u_char sig_byte = data[1]; + u_char next_hdr = data[3]; + + // identity length field is high bits of id_byte. + // length in octets is 2 to the power of length field + int id_len = (1 << (id_byte>>4)); + + // signature length field is high bits of sig_byte + // length in octets 4 * length field + int sig_len = 4*(sig_byte>>4); + + datalen -= 8 + id_len + sig_len; + data += 8 + id_len + sig_len; + if (datalen <= 0) + return 0; + cand_ip_hdr = LookForIPHdr(data, datalen); + if (cand_ip_hdr) + { + hdr_len += 8 + id_len + sig_len; + tunneltype = (cand_ip_hdr->IP4_Hdr()) ? + BifEnum::Tunnel::IP4inAYIAY : BifEnum::Tunnel::IP6inAYIAY; + } + } + if (cand_ip_hdr) + { + TunnelInfo *tunnel_info = new TunnelInfo(); + tunnel_info->child = cand_ip_hdr; + tunnel_info->tunneltype = tunneltype; + tunnel_info->SetParentIPs(ip_hdr); + tunnel_info->SetParentPorts(uh); + tunnel_info->hdr_len = hdr_len + cand_ip_hdr->HdrLen(); + return tunnel_info; + }; + } + return 0; + } + +IP_Hdr* TunnelHandler::LookForIPHdr(const u_char *data, int datalen) + { + IP_Hdr *cand_ip_hdr = 0; + if (datalen < (int)sizeof(struct ip)) + return 0; + + const struct ip *ip4 = (const struct ip*)(data); + if (ip4->ip_v == 4) + cand_ip_hdr = new IP_Hdr((const struct ip*)ip4); + else if (ip4->ip_v == 6 && (datalen > (int)sizeof(struct ip6_hdr))) + cand_ip_hdr = new IP_Hdr((const struct ip6_hdr*)data); + + if (cand_ip_hdr) + { + switch (cand_ip_hdr->NextProto()) { + case IPPROTO_UDP: + case IPPROTO_TCP: + case IPPROTO_ICMP: + if ((int)cand_ip_hdr->TotalLen() != datalen) + { + delete cand_ip_hdr; + cand_ip_hdr = 0; + } + break; + default: + delete cand_ip_hdr; + cand_ip_hdr = 0; + break; + } // end switch + } + return cand_ip_hdr; + } diff --git a/src/TunnelHandler.h b/src/TunnelHandler.h index aa4cae0a39..31c9791a1c 100644 --- a/src/TunnelHandler.h +++ b/src/TunnelHandler.h @@ -3,6 +3,7 @@ #ifndef tunnelhandler_h #define tunnelhandler_h +#include #include "IP.h" #include "Conn.h" #include "Sessions.h" @@ -14,7 +15,7 @@ public: TunnelInfo() { child = 0; - tunneltype = BifEnum::NONE; + tunneltype = BifEnum::Tunnel::NONE; hdr_len = 0; parent.src_addr = parent.dst_addr = 0; parent.src_port = parent.dst_port = 0; @@ -30,30 +31,39 @@ public: parent.src_addr = ip_hdr->SrcAddr(); parent.dst_addr = ip_hdr->DstAddr(); } - void SetParentPorts(uint32 src_port, uint32 dst_port) + void SetParentPorts(const struct udphdr *uh) { - parent.src_port = src_port; - parent.dst_port = dst_port; + parent.src_port = uh->uh_sport; + parent.dst_port = uh->uh_dport; } RecordVal* GetRecordVal() const { - RecordVal *rv = new RecordVal(BifType::Record::tunnel_parent_t); + RecordVal *rv = new RecordVal(BifType::Record::Tunnel::parent_t); + TransportProto tproto; + switch(tunneltype) { + case BifEnum::Tunnel::IP6inIP: + case BifEnum::Tunnel::IP4inIP: + tproto = TRANSPORT_UNKNOWN; + break; + default: + tproto = TRANSPORT_UDP; + } // end switch RecordVal* id_val = new RecordVal(conn_id); id_val->Assign(0, new AddrVal(parent.src_addr)); - id_val->Assign(1, new PortVal(ntohs(parent.src_port), TRANSPORT_UNKNOWN)); + id_val->Assign(1, new PortVal(ntohs(parent.src_port), tproto)); id_val->Assign(2, new AddrVal(parent.dst_addr)); - id_val->Assign(3, new PortVal(ntohs(parent.dst_port), TRANSPORT_UNKNOWN)); + id_val->Assign(3, new PortVal(ntohs(parent.dst_port), tproto)); rv->Assign(0, id_val); - rv->Assign(1, new EnumVal(tunneltype, BifType::Enum::tunneltype_t)); + rv->Assign(1, new EnumVal(tunneltype, BifType::Enum::Tunnel::tunneltype_t)); return rv; } IP_Hdr *child; ConnID parent; int hdr_len; - BifEnum::tunneltype_t tunneltype; + BifEnum::Tunnel::tunneltype_t tunneltype; }; class TunnelHandler { @@ -61,12 +71,16 @@ public: TunnelHandler(NetSessions *arg_s); ~TunnelHandler(); + // Main entry point. Returns a nil if not tunneled. TunnelInfo* DecapsulateTunnel(const IP_Hdr* ip_hdr, int len, int caplen, - /* need those for passing them back to NetSessions::Weird() */ + // need those for passing them back to NetSessions::Weird() const struct pcap_pkthdr* hdr, const u_char* const pkt); protected: NetSessions *s; + short udp_ports[65536]; // which UDP ports to decapsulate + IP_Hdr* LookForIPHdr(const u_char *data, int datalen); + TunnelInfo* HandleUDP(const IP_Hdr *ip_hdr, int len, int caplen); }; diff --git a/src/const.bif b/src/const.bif index 825c21e7a5..447812a902 100644 --- a/src/const.bif +++ b/src/const.bif @@ -12,3 +12,7 @@ const NFS3::return_data: bool; const NFS3::return_data_max: count; const NFS3::return_data_first_only: bool; +const Tunnel::decapsulate_ip: bool; +const Tunnel::decapsulate_udp: bool; +const Tunnel::udp_tunnel_ports: any; +const Tunnel::udp_tunnel_allports: bool; diff --git a/src/types.bif b/src/types.bif index d44f177b82..35c4db0daf 100644 --- a/src/types.bif +++ b/src/types.bif @@ -166,11 +166,18 @@ enum ID %{ Unknown, %} -module GLOBAL; +module Tunnel; enum tunneltype_t %{ NONE, - IP6inIP, + IP6inIP, + IP4inIP, + IP6inUDP, + IP4inUDP, + IP6inAYIAY, + IP4inAYIAY, %} -type tunnel_parent_t: record; +type parent_t: record; + +module GLOBAL; From 5ed3ec2f38b4a8546fba55cfeace04615021b76c Mon Sep 17 00:00:00 2001 From: Gregor Maier Date: Sun, 7 Aug 2011 09:49:41 -0700 Subject: [PATCH 005/964] Finishing tunnel decapsulation support in C++ core. Policy script is next. --- policy/bro.init | 2 +- src/Sessions.cc | 1 - src/TunnelHandler.cc | 18 +++++++++++++++--- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/policy/bro.init b/policy/bro.init index 8be0ab754a..59acf746dd 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -86,7 +86,7 @@ export { type parent_t: record { cid: conn_id; tunnel_type: tunneltype_t; - }; + } &log; } # end export module GLOBAL; diff --git a/src/Sessions.cc b/src/Sessions.cc index 41a7f4c52d..9241b17005 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -135,7 +135,6 @@ NetSessions::NetSessions() tunnel_handler = new TunnelHandler(this); else tunnel_handler = 0; - printf("tunnel_handler: %p\n", tunnel_handler); } NetSessions::~NetSessions() diff --git a/src/TunnelHandler.cc b/src/TunnelHandler.cc index 3256894cd0..78428c700f 100644 --- a/src/TunnelHandler.cc +++ b/src/TunnelHandler.cc @@ -18,10 +18,22 @@ TunnelHandler::TunnelHandler(NetSessions *arg_s) { s = arg_s; + PortVal *pv = 0; + TableVal *udp_tunnel_ports = BifConst::Tunnel::udp_tunnel_ports->AsTableVal(); + // Find UDP ports we want to analyze. Store them in an array for faster + // lookup. for (int i=0; i< 65536; i++) - udp_ports[i] = 0; - udp_ports[3544] = 1; - udp_ports[5072] = 1; + { + Unref(pv); + pv = new PortVal(i, TRANSPORT_UDP); + if (udp_tunnel_ports->Lookup(pv, false)) + { + udp_ports[i] = 1; + } + else + udp_ports[i] = 0; + } + Unref(pv); } TunnelInfo* TunnelHandler::DecapsulateTunnel(const IP_Hdr *ip_hdr, int len, int caplen, From 32f37c9f6d151d73a47c2a0e558fcbd37b47f49c Mon Sep 17 00:00:00 2001 From: Gregor Maier Date: Sun, 7 Aug 2011 12:26:19 -0700 Subject: [PATCH 006/964] Documenting tunnel decapsulation. Haven't tested the autodoc output yet. --- scripts/base/bro.init | 17 +++++--- scripts/policy/frameworks/tunnel.bro | 61 ++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 scripts/policy/frameworks/tunnel.bro diff --git a/scripts/base/bro.init b/scripts/base/bro.init index cf34a763c5..a6b0ff3890 100644 --- a/scripts/base/bro.init +++ b/scripts/base/bro.init @@ -83,8 +83,14 @@ type AnalyzerID: count; module Tunnel; export { + ## Records the identity of a the parent of a tunneled connection. type parent_t: record { + ## The 4-tuple of the tunnel "connection". In case of an IP-in-IP + ## tunnel the ports will be set to 0. The direction (i.e., orig and + ## resp) of the parent are set according to the tunneled connection + ## and not according to the side that established the tunnel. cid: conn_id; + ## The type of tunnel. tunnel_type: tunneltype_t; } &log; } # end export @@ -1495,20 +1501,21 @@ const parse_udp_tunnels = F &redef; module Tunnel; export { - # Whether to decapsulate IP tunnels (IPinIP, 6in4, 6to4) + ## Whether to decapsulate IP tunnels (IPinIP, 6in4, 6to4) const decapsulate_ip = F &redef; - # Whether to decapsulate URDP tunnels (e.g., Teredo, IPv4 in UDP) + ## Whether to decapsulate URDP tunnels (e.g., Teredo, IPv4 in UDP) const decapsulate_udp = F &redef; - # If decapsulating UDP: the set of ports for which to do so + ## If decapsulating UDP: the set of ports for which to do so. + ## Can be overridden by :bro:id:`Tunnel::udp_tunnel_allports` const udp_tunnel_ports: set[port] = { 3544/udp, # Teredo 5072/udp, # AYIAY } &redef; - # If udp_tunnel_allports is T udp_tunnel_ports is ignored and we - # check every UDP packet for tunnels. + ## If udp_tunnel_allports is T :bro:id:`udp_tunnel_ports` is ignored and we + ## check every UDP packet for tunnels. const udp_tunnel_allports = F &redef; } # end export module GLOBAL; diff --git a/scripts/policy/frameworks/tunnel.bro b/scripts/policy/frameworks/tunnel.bro new file mode 100644 index 0000000000..80e46c31a4 --- /dev/null +++ b/scripts/policy/frameworks/tunnel.bro @@ -0,0 +1,61 @@ +##! Handle tunneled connections. +##! +##! Bro can decapsulate IPinIP and IPinUDP tunnels, were "IP" can be either +##! IPv4 or IPv6. The most common case will be decapsulating Teredo, 6to4, +##! 6in4, and AYIAY. +##! +##! Decapsulation happens early in a packets processing, right after IP +##! defragmentation but before there is a connection context. The tunnel +##! headers are stripped from packet and the identity of the parent is +##! is stored as the ``tunnel_parent`` member of :bro:type:`connection`, +##! which is of type :bro:type:`parent_t`. +##! +##! *Limitation:* The decapsulated packets are not fed through the +##! defragmenter again. +##! +##! + +module Tunnel; + +redef use_connection_compressor = F; +redef Tunnel::decapsulate_ip = T; +redef Tunnel::decapsulate_udp = T; +redef Tunnel::udp_tunnel_allports = T; + +export { + redef enum Log::ID += { TUNNEL }; + + ## This record will be logged + type Info : record { + ## This is the time of the first record + ts: time &log; + ## The uid of the child connection, i.e. the connection in the tunnel + uid: string &log; + ## The connection id of the child + id: conn_id &log; + ## The child's transport protocol + proto: transport_proto &log; + ## The parent connection of IP-pair + parent: parent_t &log; + }; + global log_conn: event(rec: Info); +} + +event bro_init() + { + Log::create_stream(TUNNEL, [$columns=Info, $ev=log_conn]); + } + +event new_connection(c: connection) + { + if (c?$tunnel_parent) + { + local info: Info; + info$ts = c$start_time; + info$uid = c$uid; + info$id = c$id; + info$proto = get_port_transport_proto(c$id$resp_p); + info$parent = c$tunnel_parent; + Log::write(TUNNEL, info); + } + } From cd592203a0e3399ea8fdfe1288f51cce813db6d9 Mon Sep 17 00:00:00 2001 From: Gregor Maier Date: Sun, 7 Aug 2011 13:55:46 -0700 Subject: [PATCH 007/964] Remvoing old tunnel code (encap_hdr_size and udp_tunnels). --- scripts/base/bro.init | 6 ----- src/Sessions.cc | 56 ------------------------------------------- 2 files changed, 62 deletions(-) diff --git a/scripts/base/bro.init b/scripts/base/bro.init index a6b0ff3890..2f83b99bf8 100644 --- a/scripts/base/bro.init +++ b/scripts/base/bro.init @@ -515,12 +515,6 @@ const packet_sort_window = 0 usecs &redef; # state accrual. const frag_timeout = 0.0 sec &redef; -# If positive, indicates the encapsulation header size that should -# be skipped over for each captured packet .... -const encap_hdr_size = 0 &redef; -# ... or just for the following UDP port. -const tunnel_port = 0/udp &redef; - # Whether to use the ConnSize analyzer to count the number of # packets and IP-level bytes transfered by each endpoint. If # true, these values are returned in the connection's endpoint diff --git a/src/Sessions.cc b/src/Sessions.cc index 9241b17005..fa0d573660 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -177,62 +177,6 @@ void NetSessions::DispatchPacket(double t, const struct pcap_pkthdr* hdr, ip_data = pkt + hdr_size + (ip_hdr->ip_hl << 2); } - if ( encap_hdr_size > 0 && ip_data ) - { - // We're doing tunnel encapsulation. Check whether there's - // a particular associated port. - // - // Should we discourage the use of encap_hdr_size for UDP - // tunnneling? It is probably better handled by enabling - // BifConst::parse_udp_tunnels instead of specifying a fixed - // encap_hdr_size. - if ( udp_tunnel_port > 0 ) - { - ASSERT(ip_hdr); - if ( ip_hdr->ip_p == IPPROTO_UDP ) - { - const struct udphdr* udp_hdr = - reinterpret_cast - (ip_data); - - if ( ntohs(udp_hdr->uh_dport) == udp_tunnel_port ) - { - // A match. - hdr_size += encap_hdr_size; - } - } - } - - else - // Blanket encapsulation - hdr_size += encap_hdr_size; - } - - // Check IP packets encapsulated through UDP tunnels. - // Specifying a udp_tunnel_port is optional but recommended (to avoid - // the cost of checking every UDP packet). - else if ( BifConst::parse_udp_tunnels && ip_data && ip_hdr->ip_p == IPPROTO_UDP ) - { - const struct udphdr* udp_hdr = - reinterpret_cast(ip_data); - - if ( udp_tunnel_port == 0 || // 0 matches any port - udp_tunnel_port == ntohs(udp_hdr->uh_dport) ) - { - const u_char* udp_data = - ip_data + sizeof(struct udphdr); - const struct ip* ip_encap = - reinterpret_cast(udp_data); - const int ip_encap_len = - ntohs(udp_hdr->uh_ulen) - sizeof(struct udphdr); - const int ip_encap_caplen = - hdr->caplen - (udp_data - pkt); - - if ( looks_like_IPv4_packet(ip_encap_len, ip_encap) ) - hdr_size = udp_data - pkt; - } - } - if ( src_ps->FilterType() == TYPE_FILTER_NORMAL ) NextPacket(t, hdr, pkt, hdr_size, pkt_elem); else From fe6a05e2ade5151db7d22b7b6d39c04182e77145 Mon Sep 17 00:00:00 2001 From: Gregor Maier Date: Sun, 7 Aug 2011 15:33:50 -0700 Subject: [PATCH 008/964] Tunnel documentation tweaks. Checkpoint. Need to check cross-references once "make doc" works again. --- scripts/policy/frameworks/tunnel.bro | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/scripts/policy/frameworks/tunnel.bro b/scripts/policy/frameworks/tunnel.bro index 80e46c31a4..ebec2b0f06 100644 --- a/scripts/policy/frameworks/tunnel.bro +++ b/scripts/policy/frameworks/tunnel.bro @@ -2,7 +2,21 @@ ##! ##! Bro can decapsulate IPinIP and IPinUDP tunnels, were "IP" can be either ##! IPv4 or IPv6. The most common case will be decapsulating Teredo, 6to4, -##! 6in4, and AYIAY. +##! 6in4, and AYIAY. When this script is loaded, decapsulation will be +##! enabled. "tunnel.log" will log the "parent" for each tunneled +##! connection. The identity (and existence) of the tunnel connection +##! is otherwise lost. +##! +##! Currently handles: +##! +##! * IP6 in IP{4,6}. (IP4 in IP is easy to add, but omitted due to lack +##! of test cases. +##! * IP{4,6} in UDP. This decapsulates e.g., standard *Teredo* packets +##! (without authentication or origin indicator) +##! * IP{4,6} in AYIAY +##! * Only checks for UDP tunnels on Teredo's and AYIAY's default +##! ports. See :bro:id:`udp_tunnel_ports` and +##! :bro:id:`udp_tunnel_allports` ##! ##! Decapsulation happens early in a packets processing, right after IP ##! defragmentation but before there is a connection context. The tunnel @@ -11,13 +25,14 @@ ##! which is of type :bro:type:`parent_t`. ##! ##! *Limitation:* The decapsulated packets are not fed through the -##! defragmenter again. +##! defragmenter again and decapsulation happens only on the primary +##! path, i.e., it's not available for the secondary path. ##! ##! module Tunnel; -redef use_connection_compressor = F; +#redef use_connection_compressor = F; redef Tunnel::decapsulate_ip = T; redef Tunnel::decapsulate_udp = T; redef Tunnel::udp_tunnel_allports = T; From d0a67dc8bfc9bc21a164a17c57f3dcf65a08eedc Mon Sep 17 00:00:00 2001 From: Gregor Maier Date: Wed, 10 Aug 2011 13:47:02 -0700 Subject: [PATCH 009/964] Tweaking tunnel decapsulation. Changing names to comply with "Bro Scripting Conventions" Tweaking documentation. --- doc/scripts/DocSourcesList.cmake | 1 + scripts/base/init-bare.bro | 6 +++--- scripts/policy/frameworks/tunnel.bro | 16 ++++++++++++---- src/TunnelHandler.cc | 8 ++++---- src/TunnelHandler.h | 10 +++++----- src/types.bif | 16 ++++++++-------- 6 files changed, 33 insertions(+), 24 deletions(-) diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index 54783f61b3..fbf93ce869 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -95,6 +95,7 @@ rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro) rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro) rest_target(${psd} policy/frameworks/software/version-changes.bro) rest_target(${psd} policy/frameworks/software/vulnerable.bro) +rest_target(${psd} policy/frameworks/tunnel.bro) rest_target(${psd} policy/integration/barnyard2/base.bro) rest_target(${psd} policy/integration/barnyard2/event.bro) rest_target(${psd} policy/integration/barnyard2/types.bro) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 2f83b99bf8..45357fde77 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -84,14 +84,14 @@ type AnalyzerID: count; module Tunnel; export { ## Records the identity of a the parent of a tunneled connection. - type parent_t: record { + type Parent: record { ## The 4-tuple of the tunnel "connection". In case of an IP-in-IP ## tunnel the ports will be set to 0. The direction (i.e., orig and ## resp) of the parent are set according to the tunneled connection ## and not according to the side that established the tunnel. cid: conn_id; ## The type of tunnel. - tunnel_type: tunneltype_t; + tunnel_type: Tunneltype; } &log; } # end export module GLOBAL; @@ -107,7 +107,7 @@ type connection: record { hot: count; # how hot; 0 = don't know or not hot history: string; uid: string; - tunnel_parent: Tunnel::parent_t &optional; + tunnel_parent: Tunnel::Parent &optional; }; type SYN_packet: record { diff --git a/scripts/policy/frameworks/tunnel.bro b/scripts/policy/frameworks/tunnel.bro index ebec2b0f06..a24bd6e1f6 100644 --- a/scripts/policy/frameworks/tunnel.bro +++ b/scripts/policy/frameworks/tunnel.bro @@ -22,7 +22,7 @@ ##! defragmentation but before there is a connection context. The tunnel ##! headers are stripped from packet and the identity of the parent is ##! is stored as the ``tunnel_parent`` member of :bro:type:`connection`, -##! which is of type :bro:type:`parent_t`. +##! which is of type :bro:type:`Tunnel::Parent`. ##! ##! *Limitation:* The decapsulated packets are not fed through the ##! defragmenter again and decapsulation happens only on the primary @@ -30,9 +30,12 @@ ##! ##! +@load base/protocols/conn + module Tunnel; #redef use_connection_compressor = F; +## enab redef Tunnel::decapsulate_ip = T; redef Tunnel::decapsulate_udp = T; redef Tunnel::udp_tunnel_allports = T; @@ -51,14 +54,19 @@ export { ## The child's transport protocol proto: transport_proto &log; ## The parent connection of IP-pair - parent: parent_t &log; + parent: Parent &log; + }; + global log_tunnel: event(rec: Info); + + redef record Conn::Info += { + ## If the connection is tunneled the type of tunnel + tunnel_type: Tunneltype &log &optional; }; - global log_conn: event(rec: Info); } event bro_init() { - Log::create_stream(TUNNEL, [$columns=Info, $ev=log_conn]); + Log::create_stream(TUNNEL, [$columns=Info, $ev=log_tunnel]); } event new_connection(c: connection) diff --git a/src/TunnelHandler.cc b/src/TunnelHandler.cc index 78428c700f..6b1f78e0c0 100644 --- a/src/TunnelHandler.cc +++ b/src/TunnelHandler.cc @@ -54,7 +54,7 @@ TunnelInfo* TunnelHandler::DecapsulateTunnel(const IP_Hdr *ip_hdr, int len, int // TODO: check if IP6 header makes sense tunnel_info = new TunnelInfo(); tunnel_info->child = new IP_Hdr((const struct ip6_hdr*)ip_hdr->Payload()); - tunnel_info->tunneltype = BifEnum::Tunnel::IP6inIP; + tunnel_info->tunneltype = BifEnum::Tunnel::IP6_IN_IP; tunnel_info->hdr_len = tunnel_info->child->HdrLen(); tunnel_info->SetParentIPs(ip_hdr); return tunnel_info; @@ -86,7 +86,7 @@ TunnelInfo* TunnelHandler::HandleUDP(const IP_Hdr *ip_hdr, int len, int caplen) const u_char *data = ip_hdr->Payload(); const struct udphdr* uh = (const struct udphdr*)data; IP_Hdr *cand_ip_hdr = 0; - BifEnum::Tunnel::tunneltype_t tunneltype = BifEnum::Tunnel::NONE; + BifEnum::Tunnel::Tunneltype tunneltype = BifEnum::Tunnel::NONE; int hdr_len = sizeof(struct udphdr); data += hdr_len; @@ -103,7 +103,7 @@ TunnelInfo* TunnelHandler::HandleUDP(const IP_Hdr *ip_hdr, int len, int caplen) if (cand_ip_hdr) { tunneltype = (cand_ip_hdr->IP4_Hdr()) ? - BifEnum::Tunnel::IP4inUDP : BifEnum::Tunnel::IP6inUDP; + BifEnum::Tunnel::IP4_IN_UDP : BifEnum::Tunnel::IP6_IN_UDP; } else if (datalen >= 8) { @@ -129,7 +129,7 @@ TunnelInfo* TunnelHandler::HandleUDP(const IP_Hdr *ip_hdr, int len, int caplen) { hdr_len += 8 + id_len + sig_len; tunneltype = (cand_ip_hdr->IP4_Hdr()) ? - BifEnum::Tunnel::IP4inAYIAY : BifEnum::Tunnel::IP6inAYIAY; + BifEnum::Tunnel::IP4_IN_AYIAY : BifEnum::Tunnel::IP6_IN_AYIAY; } } if (cand_ip_hdr) diff --git a/src/TunnelHandler.h b/src/TunnelHandler.h index 31c9791a1c..d88e6ff2b4 100644 --- a/src/TunnelHandler.h +++ b/src/TunnelHandler.h @@ -39,11 +39,11 @@ public: RecordVal* GetRecordVal() const { - RecordVal *rv = new RecordVal(BifType::Record::Tunnel::parent_t); + RecordVal *rv = new RecordVal(BifType::Record::Tunnel::Parent); TransportProto tproto; switch(tunneltype) { - case BifEnum::Tunnel::IP6inIP: - case BifEnum::Tunnel::IP4inIP: + case BifEnum::Tunnel::IP6_IN_IP: + case BifEnum::Tunnel::IP4_IN_IP: tproto = TRANSPORT_UNKNOWN; break; default: @@ -56,14 +56,14 @@ public: id_val->Assign(2, new AddrVal(parent.dst_addr)); id_val->Assign(3, new PortVal(ntohs(parent.dst_port), tproto)); rv->Assign(0, id_val); - rv->Assign(1, new EnumVal(tunneltype, BifType::Enum::Tunnel::tunneltype_t)); + rv->Assign(1, new EnumVal(tunneltype, BifType::Enum::Tunnel::Tunneltype)); return rv; } IP_Hdr *child; ConnID parent; int hdr_len; - BifEnum::Tunnel::tunneltype_t tunneltype; + BifEnum::Tunnel::Tunneltype tunneltype; }; class TunnelHandler { diff --git a/src/types.bif b/src/types.bif index 35c4db0daf..5f1c4b850b 100644 --- a/src/types.bif +++ b/src/types.bif @@ -168,16 +168,16 @@ enum ID %{ module Tunnel; -enum tunneltype_t %{ +enum Tunneltype %{ NONE, - IP6inIP, - IP4inIP, - IP6inUDP, - IP4inUDP, - IP6inAYIAY, - IP4inAYIAY, + IP6_IN_IP, + IP4_IN_IP, + IP6_IN_UDP, + IP4_IN_UDP, + IP6_IN_AYIAY, + IP4_IN_AYIAY, %} -type parent_t: record; +type Parent: record; module GLOBAL; From 65921bc61da9a27dffeba46a8d6fe706cdc31e75 Mon Sep 17 00:00:00 2001 From: Gregor Maier Date: Fri, 12 Aug 2011 21:42:47 -0700 Subject: [PATCH 010/964] Bugfix: an #ifdef BROv6 was missing in the tunnel code --- src/TunnelHandler.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/TunnelHandler.cc b/src/TunnelHandler.cc index 6b1f78e0c0..4df3f489ad 100644 --- a/src/TunnelHandler.cc +++ b/src/TunnelHandler.cc @@ -102,6 +102,7 @@ TunnelInfo* TunnelHandler::HandleUDP(const IP_Hdr *ip_hdr, int len, int caplen) cand_ip_hdr = LookForIPHdr(data, datalen); if (cand_ip_hdr) { + // Found and IP hdr directly in the UDP payload tunneltype = (cand_ip_hdr->IP4_Hdr()) ? BifEnum::Tunnel::IP4_IN_UDP : BifEnum::Tunnel::IP6_IN_UDP; } @@ -155,8 +156,10 @@ IP_Hdr* TunnelHandler::LookForIPHdr(const u_char *data, int datalen) const struct ip *ip4 = (const struct ip*)(data); if (ip4->ip_v == 4) cand_ip_hdr = new IP_Hdr((const struct ip*)ip4); +#ifdef BROv6 else if (ip4->ip_v == 6 && (datalen > (int)sizeof(struct ip6_hdr))) cand_ip_hdr = new IP_Hdr((const struct ip6_hdr*)data); +#endif if (cand_ip_hdr) { From 08dc84a2504483a3b5827dbf65e7cc68bf9e9516 Mon Sep 17 00:00:00 2001 From: Gregor Maier Date: Tue, 16 Aug 2011 20:41:36 -0700 Subject: [PATCH 011/964] Tunnel decapsulation bugfix when FlipRoles is called. If FlipRoles() is called the conn_val in Conn.cc gets Unref'ed and thus my tunnel_partent RecordVal was lost. Fixing this. --- src/Conn.cc | 9 ++++-- src/Conn.h | 5 +-- src/Sessions.cc | 8 ++--- src/TunnelHandler.cc | 4 +-- src/TunnelHandler.h | 75 ++++++++++++++++++++++++++++---------------- 5 files changed, 64 insertions(+), 37 deletions(-) diff --git a/src/Conn.cc b/src/Conn.cc index 67e337fda9..e3e56c5eef 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -15,6 +15,7 @@ #include "Timer.h" #include "PIA.h" #include "binpac.h" +#include "TunnelHandler.h" HashKey* ConnID::BuildConnKey() const { @@ -139,7 +140,7 @@ unsigned int Connection::external_connections = 0; IMPLEMENT_SERIAL(Connection, SER_CONNECTION); -Connection::Connection(NetSessions* s, HashKey* k, double t, const ConnID* id, RecordVal *arg_tunnel_parent) +Connection::Connection(NetSessions* s, HashKey* k, double t, const ConnID* id, TunnelParent* arg_tunnel_parent) { sessions = s; key = k; @@ -209,6 +210,9 @@ Connection::~Connection() Unref(conn_val); } + if ( tunnel_parent ) + delete tunnel_parent; + delete key; delete root_analyzer; delete conn_timer_mgr; @@ -372,7 +376,8 @@ RecordVal* Connection::BuildConnVal() char tmp[20]; conn_val->Assign(9, new StringVal(uitoa_n(uid, tmp, sizeof(tmp), 62))); - conn_val->Assign(10, tunnel_parent); + if ( tunnel_parent ) + conn_val->Assign(10, tunnel_parent->GetRecordVal()); } if ( root_analyzer ) diff --git a/src/Conn.h b/src/Conn.h index e22c0b83ec..828ad66e7c 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -23,6 +23,7 @@ class RuleHdrTest; class Specific_RE_Matcher; class TransportLayerAnalyzer; class RuleEndpointState; +class TunnelParent; typedef enum { NUL_IN_LINE, @@ -86,7 +87,7 @@ class Analyzer; class Connection : public BroObj { public: - Connection(NetSessions* s, HashKey* k, double t, const ConnID* id, RecordVal *arg_tunnel_parent); + Connection(NetSessions* s, HashKey* k, double t, const ConnID* id, TunnelParent *arg_tunnel_parent); virtual ~Connection(); // Invoked when connection is about to be removed. Use Ref(this) @@ -335,7 +336,7 @@ protected: double inactivity_timeout; RecordVal* conn_val; LoginConn* login_conn; // either nil, or this - RecordVal* tunnel_parent; // nil if not tunneled + TunnelParent* tunnel_parent; // nil if not tunneled int suppress_event; // suppress certain events to once per conn. unsigned int installed_status_timer:1; diff --git a/src/Sessions.cc b/src/Sessions.cc index fa0d573660..34bfa360dd 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -1018,11 +1018,8 @@ Connection* NetSessions::NewConn(HashKey* k, double t, const ConnID* id, int src_h = ntohs(id->src_port); int dst_h = ntohs(id->dst_port); int flags = 0; - RecordVal *tunnel_parent = 0; + TunnelParent *tunnel_parent = 0; - if ( tunnel_info ) - tunnel_parent = tunnel_info->GetRecordVal(); - // Hmm... This is not great. TransportProto tproto; switch ( proto ) { @@ -1068,6 +1065,9 @@ Connection* NetSessions::NewConn(HashKey* k, double t, const ConnID* id, id = &flip_id; } + if ( tunnel_info ) + tunnel_parent = new TunnelParent(&(tunnel_info->parent)); + Connection* conn = new Connection(this, k, t, id, tunnel_parent); conn->SetTransport(tproto); dpm->BuildInitialAnalyzerTree(tproto, conn, data); diff --git a/src/TunnelHandler.cc b/src/TunnelHandler.cc index 4df3f489ad..a5ee9cae3c 100644 --- a/src/TunnelHandler.cc +++ b/src/TunnelHandler.cc @@ -54,7 +54,7 @@ TunnelInfo* TunnelHandler::DecapsulateTunnel(const IP_Hdr *ip_hdr, int len, int // TODO: check if IP6 header makes sense tunnel_info = new TunnelInfo(); tunnel_info->child = new IP_Hdr((const struct ip6_hdr*)ip_hdr->Payload()); - tunnel_info->tunneltype = BifEnum::Tunnel::IP6_IN_IP; + tunnel_info->parent.tunneltype = BifEnum::Tunnel::IP6_IN_IP; tunnel_info->hdr_len = tunnel_info->child->HdrLen(); tunnel_info->SetParentIPs(ip_hdr); return tunnel_info; @@ -137,7 +137,7 @@ TunnelInfo* TunnelHandler::HandleUDP(const IP_Hdr *ip_hdr, int len, int caplen) { TunnelInfo *tunnel_info = new TunnelInfo(); tunnel_info->child = cand_ip_hdr; - tunnel_info->tunneltype = tunneltype; + tunnel_info->parent.tunneltype = tunneltype; tunnel_info->SetParentIPs(ip_hdr); tunnel_info->SetParentPorts(uh); tunnel_info->hdr_len = hdr_len + cand_ip_hdr->HdrLen(); diff --git a/src/TunnelHandler.h b/src/TunnelHandler.h index d88e6ff2b4..b5c69b7eff 100644 --- a/src/TunnelHandler.h +++ b/src/TunnelHandler.h @@ -4,37 +4,29 @@ #define tunnelhandler_h #include +#include "net_util.h" #include "IP.h" #include "Conn.h" #include "Sessions.h" #include "Val.h" - -class TunnelInfo { +class TunnelParent { public: - TunnelInfo() + TunnelParent() { - child = 0; tunneltype = BifEnum::Tunnel::NONE; - hdr_len = 0; - parent.src_addr = parent.dst_addr = 0; - parent.src_port = parent.dst_port = 0; - parent.is_one_way = 0; - } - ~TunnelInfo() - { - if (child) delete child; + src_port = dst_port = 0; + for (int i=0; iSrcAddr(); - parent.dst_addr = ip_hdr->DstAddr(); - } - void SetParentPorts(const struct udphdr *uh) - { - parent.src_port = uh->uh_sport; - parent.dst_port = uh->uh_dport; + tunneltype = other->tunneltype; + copy_addr(other->src_addr, src_addr); + copy_addr(other->dst_addr, dst_addr); + src_port = other->src_port; + dst_port = other->dst_port; } RecordVal* GetRecordVal() const @@ -51,21 +43,50 @@ public: } // end switch RecordVal* id_val = new RecordVal(conn_id); - id_val->Assign(0, new AddrVal(parent.src_addr)); - id_val->Assign(1, new PortVal(ntohs(parent.src_port), tproto)); - id_val->Assign(2, new AddrVal(parent.dst_addr)); - id_val->Assign(3, new PortVal(ntohs(parent.dst_port), tproto)); + id_val->Assign(0, new AddrVal(src_addr)); + id_val->Assign(1, new PortVal(ntohs(src_port), tproto)); + id_val->Assign(2, new AddrVal(dst_addr)); + id_val->Assign(3, new PortVal(ntohs(dst_port), tproto)); rv->Assign(0, id_val); rv->Assign(1, new EnumVal(tunneltype, BifType::Enum::Tunnel::Tunneltype)); return rv; } - IP_Hdr *child; - ConnID parent; - int hdr_len; + uint32 src_addr[NUM_ADDR_WORDS]; + uint32 dst_addr[NUM_ADDR_WORDS]; + uint16 src_port; + uint16 dst_port; BifEnum::Tunnel::Tunneltype tunneltype; }; +class TunnelInfo { +public: + TunnelInfo() + { + child = 0; + hdr_len = 0; + } + ~TunnelInfo() + { + if (child) delete child; + } + + void SetParentIPs(const IP_Hdr *ip_hdr) + { + copy_addr(ip_hdr->SrcAddr(), parent.src_addr); + copy_addr(ip_hdr->DstAddr(), parent.dst_addr); + } + void SetParentPorts(const struct udphdr *uh) + { + parent.src_port = uh->uh_sport; + parent.dst_port = uh->uh_dport; + } + + IP_Hdr *child; + TunnelParent parent; + int hdr_len; +}; + class TunnelHandler { public: TunnelHandler(NetSessions *arg_s); From 4214d8d9057d491cb6066d0b4e579c83fbbfc4cb Mon Sep 17 00:00:00 2001 From: Gregor Maier Date: Wed, 17 Aug 2011 07:31:25 -0700 Subject: [PATCH 012/964] Fixing memleak in tunnel code. --- src/Sessions.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Sessions.cc b/src/Sessions.cc index 34bfa360dd..b6e19f4cf4 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -355,6 +355,12 @@ static bool looks_like_IPv4_packet(int len, const struct ip* ip_hdr) return false; } +static inline void delete_tunnel_info(TunnelInfo *ti) + { + if ( ti ) + delete ti; + } + void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, const IP_Hdr* ip_hdr, const u_char* const pkt, int hdr_size) @@ -438,6 +444,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, proto != IPPROTO_ICMP ) { dump_this_packet = 1; + delete_tunnel_info(tunnel_info); return; } @@ -449,6 +456,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, Weird("truncated_header", hdr, pkt); if ( f ) Remove(f); // ### + delete_tunnel_info(tunnel_info); return; } if ( caplen < min_hdr_len ) @@ -456,6 +464,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, Weird("internally_truncated_header", hdr, pkt); if ( f ) Remove(f); // ### + delete_tunnel_info(tunnel_info); return; } @@ -507,6 +516,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, default: Weird(fmt("unknown_protocol %d", proto), hdr, pkt); + delete_tunnel_info(tunnel_info); return; } @@ -536,6 +546,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, if ( consistent < 0 ) { delete h; + delete_tunnel_info(tunnel_info); return; } @@ -558,7 +569,10 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, } if ( ! conn ) + { + delete_tunnel_info(tunnel_info); return; + } int record_packet = 1; // whether to record the packet at all int record_content = 1; // whether to record its data From f3a92ec30b94be0fca8dfec5447e901b8847c7cc Mon Sep 17 00:00:00 2001 From: Gregor Maier Date: Tue, 23 Aug 2011 20:28:11 -0700 Subject: [PATCH 013/964] Minor fixes for possible leaks on uncommon code path or on initialization. --- src/Sessions.cc | 8 ++++++++ src/TunnelHandler.cc | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Sessions.cc b/src/Sessions.cc index b6e19f4cf4..382bd9c5b3 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -444,6 +444,8 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, proto != IPPROTO_ICMP ) { dump_this_packet = 1; + if ( f ) + Remove(t); delete_tunnel_info(tunnel_info); return; } @@ -516,6 +518,8 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, default: Weird(fmt("unknown_protocol %d", proto), hdr, pkt); + if ( f ) + Remove(f); delete_tunnel_info(tunnel_info); return; } @@ -546,6 +550,8 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, if ( consistent < 0 ) { delete h; + if ( f ) + Remove(f); delete_tunnel_info(tunnel_info); return; } @@ -571,6 +577,8 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, if ( ! conn ) { delete_tunnel_info(tunnel_info); + if ( f ) + Remove(f); return; } diff --git a/src/TunnelHandler.cc b/src/TunnelHandler.cc index a5ee9cae3c..a7fbd4ce29 100644 --- a/src/TunnelHandler.cc +++ b/src/TunnelHandler.cc @@ -24,7 +24,8 @@ TunnelHandler::TunnelHandler(NetSessions *arg_s) // lookup. for (int i=0; i< 65536; i++) { - Unref(pv); + if (pv) + Unref(pv); pv = new PortVal(i, TRANSPORT_UDP); if (udp_tunnel_ports->Lookup(pv, false)) { From 1936989422311d62bb08dc1466f6e82bea8409c8 Mon Sep 17 00:00:00 2001 From: Gregor Maier Date: Tue, 23 Aug 2011 20:46:33 -0700 Subject: [PATCH 014/964] Fix compile error due to typo. --- src/Sessions.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Sessions.cc b/src/Sessions.cc index 382bd9c5b3..47696a1651 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -445,7 +445,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, { dump_this_packet = 1; if ( f ) - Remove(t); + Remove(f); delete_tunnel_info(tunnel_info); return; } From 32cb6d6bc31c03d746bb0b72b012dd6730308c54 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 2 Sep 2011 23:55:32 -0700 Subject: [PATCH 015/964] Adding group "comm" to communication tests for parallelizing execution with new btest version. --- testing/btest/istate/broccoli.bro | 1 + testing/btest/istate/events-ssl.bro | 2 +- testing/btest/istate/events.bro | 2 +- testing/btest/istate/pybroccoli.py | 1 + testing/btest/istate/sync.bro | 2 +- testing/btest/policy/frameworks/cluster/start-it-up.bro | 3 ++- .../btest/policy/frameworks/control/configuration_update.bro | 3 ++- testing/btest/policy/frameworks/control/id_value.bro | 1 + testing/btest/policy/frameworks/control/shutdown.bro | 1 + testing/btest/policy/frameworks/logging/remote-types.bro | 2 +- testing/btest/policy/frameworks/logging/remote.bro | 2 +- testing/btest/policy/frameworks/metrics/basic-cluster.bro | 3 ++- 12 files changed, 15 insertions(+), 8 deletions(-) diff --git a/testing/btest/istate/broccoli.bro b/testing/btest/istate/broccoli.bro index 7f97f40585..def9a786d4 100644 --- a/testing/btest/istate/broccoli.bro +++ b/testing/btest/istate/broccoli.bro @@ -1,6 +1,7 @@ # @TEST-REQUIRES: grep -vq '#define BROv6' $BUILD/config.h # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib # +# @TEST-GROUP: comm # @TEST-EXEC: btest-bg-run bro bro %INPUT $DIST/aux/broccoli/test/broping-record.bro # @TEST-EXEC: btest-bg-run broccoli $BUILD/aux/broccoli/test/broping -r -c 3 127.0.0.1 # @TEST-EXEC: btest-bg-wait -k 20 diff --git a/testing/btest/istate/events-ssl.bro b/testing/btest/istate/events-ssl.bro index cfacae9da8..02fd957a4d 100644 --- a/testing/btest/istate/events-ssl.bro +++ b/testing/btest/istate/events-ssl.bro @@ -1,4 +1,4 @@ -# +# @TEST-GROUP: comm # @TEST-EXEC: btest-bg-run sender bro -C -r $TRACES/web.trace --pseudo-realtime ../sender.bro # @TEST-EXEC: btest-bg-run receiver bro ../receiver.bro # @TEST-EXEC: btest-bg-wait -k 20 diff --git a/testing/btest/istate/events.bro b/testing/btest/istate/events.bro index ecf2f2e2ad..d2dfc63239 100644 --- a/testing/btest/istate/events.bro +++ b/testing/btest/istate/events.bro @@ -1,4 +1,4 @@ -# +# @TEST-GROUP: comm # @TEST-EXEC: btest-bg-run sender bro -C -r $TRACES/web.trace --pseudo-realtime ../sender.bro # @TEST-EXEC: btest-bg-run receiver bro ../receiver.bro # @TEST-EXEC: btest-bg-wait -k 20 diff --git a/testing/btest/istate/pybroccoli.py b/testing/btest/istate/pybroccoli.py index b7fb53a955..ec4097ac55 100644 --- a/testing/btest/istate/pybroccoli.py +++ b/testing/btest/istate/pybroccoli.py @@ -1,3 +1,4 @@ +# @TEST-GROUP: comm # @TEST-REQUIRES: grep -vq '#define BROv6' $BUILD/config.h # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/bindings/broccoli-python/_broccoli_intern.so diff --git a/testing/btest/istate/sync.bro b/testing/btest/istate/sync.bro index 567bbf2af1..c41542d074 100644 --- a/testing/btest/istate/sync.bro +++ b/testing/btest/istate/sync.bro @@ -1,4 +1,4 @@ -# +# @TEST-GROUP: comm # @TEST-EXEC: btest-bg-run sender bro %INPUT ../sender.bro # @TEST-EXEC: btest-bg-run receiver bro %INPUT ../receiver.bro # @TEST-EXEC: btest-bg-wait 20 diff --git a/testing/btest/policy/frameworks/cluster/start-it-up.bro b/testing/btest/policy/frameworks/cluster/start-it-up.bro index d1eb94d5e1..c7d6c4746a 100644 --- a/testing/btest/policy/frameworks/cluster/start-it-up.bro +++ b/testing/btest/policy/frameworks/cluster/start-it-up.bro @@ -1,3 +1,4 @@ +# @TEST-GROUP: comm # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT # @TEST-EXEC: btest-bg-run proxy-1 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT # @TEST-EXEC: btest-bg-run proxy-2 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT @@ -23,4 +24,4 @@ redef Cluster::nodes = { event remote_connection_handshake_done(p: event_peer) { print "Connected to a peer"; - } \ No newline at end of file + } diff --git a/testing/btest/policy/frameworks/control/configuration_update.bro b/testing/btest/policy/frameworks/control/configuration_update.bro index 23b4998a1b..6d0f886d13 100644 --- a/testing/btest/policy/frameworks/control/configuration_update.bro +++ b/testing/btest/policy/frameworks/control/configuration_update.bro @@ -1,3 +1,4 @@ +# @TEST-GROUP: comm # @TEST-EXEC: btest-bg-run controllee BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controllee Communication::listen_port_clear=65531/tcp # @TEST-EXEC: btest-bg-run controller BROPATH=$BROPATH:.. bro %INPUT test-redef frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=configuration_update # @TEST-EXEC: btest-bg-run controller2 BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=shutdown @@ -23,4 +24,4 @@ event bro_init() event bro_done() { print test_var; - } \ No newline at end of file + } diff --git a/testing/btest/policy/frameworks/control/id_value.bro b/testing/btest/policy/frameworks/control/id_value.bro index 9f0cb76861..c751928506 100644 --- a/testing/btest/policy/frameworks/control/id_value.bro +++ b/testing/btest/policy/frameworks/control/id_value.bro @@ -1,3 +1,4 @@ +# @TEST-GROUP: comm # @TEST-EXEC: btest-bg-run controllee BROPATH=$BROPATH:.. bro %INPUT only-for-controllee frameworks/control/controllee Communication::listen_port_clear=65532/tcp # @TEST-EXEC: btest-bg-run controller BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65532/tcp Control::cmd=id_value Control::arg=test_var # @TEST-EXEC: btest-bg-wait -k 1 diff --git a/testing/btest/policy/frameworks/control/shutdown.bro b/testing/btest/policy/frameworks/control/shutdown.bro index 55af973faa..e7aabea686 100644 --- a/testing/btest/policy/frameworks/control/shutdown.bro +++ b/testing/btest/policy/frameworks/control/shutdown.bro @@ -1,3 +1,4 @@ +# @TEST-GROUP: comm # @TEST-EXEC: btest-bg-run controllee BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controllee Communication::listen_port_clear=65530/tcp # @TEST-EXEC: btest-bg-run controller BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65530/tcp Control::cmd=shutdown # @TEST-EXEC: btest-bg-wait 1 diff --git a/testing/btest/policy/frameworks/logging/remote-types.bro b/testing/btest/policy/frameworks/logging/remote-types.bro index 08960eba86..164e7f3984 100644 --- a/testing/btest/policy/frameworks/logging/remote-types.bro +++ b/testing/btest/policy/frameworks/logging/remote-types.bro @@ -1,4 +1,4 @@ -# +# @TEST-GROUP: comm # @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro # @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro # @TEST-EXEC: btest-bg-wait -k 1 diff --git a/testing/btest/policy/frameworks/logging/remote.bro b/testing/btest/policy/frameworks/logging/remote.bro index 00d7d69463..34cf714446 100644 --- a/testing/btest/policy/frameworks/logging/remote.bro +++ b/testing/btest/policy/frameworks/logging/remote.bro @@ -1,4 +1,4 @@ -# +# @TEST-GROUP: comm # @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro # @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro diff --git a/testing/btest/policy/frameworks/metrics/basic-cluster.bro b/testing/btest/policy/frameworks/metrics/basic-cluster.bro index eda41c3759..a282a962e4 100644 --- a/testing/btest/policy/frameworks/metrics/basic-cluster.bro +++ b/testing/btest/policy/frameworks/metrics/basic-cluster.bro @@ -1,3 +1,4 @@ +# @TEST-GROUP: comm # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT # @TEST-EXEC: btest-bg-run proxy-1 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT # @TEST-EXEC: sleep 1 @@ -35,4 +36,4 @@ event bro_init() Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1); } -@endif \ No newline at end of file +@endif From 935f1f55b58fccd372ea862ed8a05fc96ee199be Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 5 Sep 2011 08:26:07 -0700 Subject: [PATCH 016/964] Updating submodule(s). --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aux/binpac b/aux/binpac index a3a9410ded..032b4e0f02 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit a3a9410dedc842f6bb9859642f334ed354633b57 +Subproject commit 032b4e0f028a08257be0c703b27a7559e57bd40a diff --git a/aux/bro-aux b/aux/bro-aux index d68b98bb99..04d149a194 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit d68b98bb995a105b257f805ec4ff22c4929c7476 +Subproject commit 04d149a194e06ed5410ea3af924ff48b9129cd3b diff --git a/aux/broccoli b/aux/broccoli index 743f10dda8..89c20c7f06 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 743f10dda8cd5655ea3dc6eb705ff5414ed4f535 +Subproject commit 89c20c7f063afe5f39aa72bfec02d6996b291c13 diff --git a/aux/broctl b/aux/broctl index cf4ce9dfc5..2b9053d40d 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit cf4ce9dfc5d6dc4e6d311955eeaec2d679e8669b +Subproject commit 2b9053d40d7ef497c8cef6357b59f43129976d65 From 849046851f6007634b97ec7dfccf9b46920db0f4 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 2 Nov 2011 17:18:28 -0500 Subject: [PATCH 017/964] Start Broxygen cleanup with crude adaptation of website styles. --- doc/scripts/conf.py.in | 6 +- doc/scripts/source/_static/bro-base.css_t | 237 ++++++ doc/scripts/source/_static/bro-colors.css | 34 + doc/scripts/source/_static/bro-elements.css_t | 747 ++++++++++++++++++ doc/scripts/source/_static/bro-framing.css_t | 482 +++++++++++ doc/scripts/source/_templates/layout.html | 24 +- 6 files changed, 1524 insertions(+), 6 deletions(-) create mode 100644 doc/scripts/source/_static/bro-base.css_t create mode 100644 doc/scripts/source/_static/bro-colors.css create mode 100644 doc/scripts/source/_static/bro-elements.css_t create mode 100644 doc/scripts/source/_static/bro-framing.css_t diff --git a/doc/scripts/conf.py.in b/doc/scripts/conf.py.in index 419209f580..503b2bb612 100644 --- a/doc/scripts/conf.py.in +++ b/doc/scripts/conf.py.in @@ -27,7 +27,7 @@ sys.path.insert(0, os.path.abspath('source/ext')) extensions = ['bro'] # Add any paths that contain templates here, relative to this directory. -templates_path = ['source/_templates'] +templates_path = ['source/_templates', 'source/_static'] # The suffix of source filenames. source_suffix = '.rst' @@ -90,7 +90,7 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = 'basic' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -163,7 +163,7 @@ html_static_path = ['source/_static'] #html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'Brodoc' +htmlhelp_basename = 'Broxygen' # -- Options for LaTeX output -------------------------------------------------- diff --git a/doc/scripts/source/_static/bro-base.css_t b/doc/scripts/source/_static/bro-base.css_t new file mode 100644 index 0000000000..30885c9d72 --- /dev/null +++ b/doc/scripts/source/_static/bro-base.css_t @@ -0,0 +1,237 @@ + +{% import "bro-colors.css" as color %} + +#bro-main, #bro-standalone-main { + padding: 0 0 0 0; + position:relative; + z-index:1; +} + +#bro-main { + margin-bottom: 2em; + } + +#bro-standalone-main { + margin-bottom: 0em; + padding-left: 50px; + padding-right: 50px; + } + +#bro-outer { + color: {{color.text}}; + background: {{color.bg}}; +} + +#bro-title { + font-weight: bold; + font-size: 32px; + line-height:32px; + text-align: center; + padding-top: 3px; + margin-bottom: 30px; + font-family: Palatino,'Palatino Linotype',Georgia,serif;; + color: #000; + } + +.opening:first-letter { + font-size: 24px; + font-weight: bold; + letter-spacing: 0.05em; + } + +.opening { + font-size: 17px; +} + +.version { + text-align: right; + font-size: 12px; + color: #aaa; + line-height: 0; + height: 0; +} + +.git-info-version { + position: relative; + height: 2em; + top: -1em; + color: #ccc; + float: left; + font-size: 12px; +} + +.git-info-date { + position: relative; + height: 2em; + top: -1em; + color: #ccc; + float: right; + font-size: 12px; +} + +body { + font-family:Arial, Helvetica, sans-serif; + font-size:15px; + line-height:22px; + color: {{color.text}}; + margin: 0px; +} + +h1, h2, h3, h4, h5, h6, +h1 a, h2 a, h3 a, h4 a, h5 a, h6 a { + padding:0 0 20px 0; + font-weight:bold; + text-decoration:none; +} + +h1, h2 { + font-size:27px; + letter-spacing:-1px; +} + +h3 { + margin-top: 1em; + font-size:18px; +} + +h4 { + font-size:16px; +} + +h5 { + font-size:15px; +} + +h6 { + font-size:12px; +} + +p { + padding:0 0 20px 0; +} + +hr { + background:none; + height:1px; + line-height:1px; + border:0; + margin:0 0 20px 0; +} + +ul, ol { + margin:0 20px 20px 0; + padding-left:40px; +} + +blockquote { + margin:0 0 0 40px; +} + +strong, dfn { + font-weight:bold; +} + +em, dfn { + font-style:italic; +} + +sup, sub { + line-height:0; +} + +pre { + white-space:pre; +} + +pre, code, tt { + font-family:"Courier New", Courier, mono; +} + +dl { + margin: 0 0 20px 0; +} + +dl dt { + font-weight: bold; +} + +dd { + margin:0 0 20px 20px; +} + +small { + font-size:75%; +} + +a:link, +a:visited, +a:active +{ + color: {{color.link}}; +} + +a:hover +{ + color:{{color.emphasis_text}}; +} + +h1, h2, h3, h4, h5, h6, +h1 a, h2 a, h3 a, h4 a, h5 a, h6 a +{ + color: {{color.text}}; +} + +hr { + border-bottom:1px solid #ddd; +} + +pre { + color: {{color.text}}; + background: {{color.box_bg}}; + padding: 7px 5px 3px 5px; + margin-bottom: 25px; + margin-top: 0px; +} + +ul { + padding-bottom: 5px; + } + +h1, h2 { + margin-top: 30px; + } + +h1 { + margin-bottom: 50px; + margin-bottom: 20px; + padding-bottom: 5px; + border-bottom: 1px solid; + border-color: {{color.underline}}; + } + +h2 { + font-size: 24px; + } + +pre { + -moz-box-shadow:0 0 6px #ddd; + -webkit-box-shadow:0 0 6px #ddd; + box-shadow:0 0 6px #ddd; +} + +a { + text-decoration:none; + } + +p { + padding-bottom: 15px; + } + +p, dd, li { + text-align: justify; + } + +li { + margin-bottom: 5px; + } + diff --git a/doc/scripts/source/_static/bro-colors.css b/doc/scripts/source/_static/bro-colors.css new file mode 100644 index 0000000000..0744a721d7 --- /dev/null +++ b/doc/scripts/source/_static/bro-colors.css @@ -0,0 +1,34 @@ + +{% set bg = "#ffffff" %} +{% set text = "#333" %} +{% set underline = "#aaa" %} +{% set link = "#2a85a7" %} +{% set emphasis_text = "#c24444" %} + +{% set topbar_bg = "#000" %} + +{% set header_bg = "#f0f0f0" %} +{% set header_text = "#000" %} + +{% set footer_bg = "#333" %} +{% set footer_text = "#ddd" %} +{% set footer_links = "#ddd" %} + +{% set footer_bottom_bg = "#2b2b2b" %} +{% set footer_bottom_text = "#555" %} + +{% set menu_bg = "#fff" %} +{% set menu_hover_bg = "#ddd" %} +{% set menu_selected_bg = "#ddd" %} +{% set menu_shadow = "#eee" %} +{% set menu_border = "#ccc" %} + +{% set highlight_bg = "#ff6" %} +{% set box_bg = "#FFFAE2" %} +{% set box_hover_bg = "#FCF3A4" %} +{% set box_border = "#eee" %} + +{% set table_hover_bg = "#FFFAE2" %} + +{% set note_bg = "#FEE0C6" %} +{% set note_border = "#aaa" %} diff --git a/doc/scripts/source/_static/bro-elements.css_t b/doc/scripts/source/_static/bro-elements.css_t new file mode 100644 index 0000000000..f2c7a30f10 --- /dev/null +++ b/doc/scripts/source/_static/bro-elements.css_t @@ -0,0 +1,747 @@ + +{% import "bro-colors.css" as color %} + +#searchform { + position:relative; +} + +#searchform #searchsubmit { + position:absolute; + top:0; + right:4px; + border:0; + height:22px; + width:20px; + text-indent:-9999px; + cursor:pointer; +} + +#searchform #s { + width:99%; + height:20px; + border:0; + position:absolute; + top:0; + left:0; + text-indent:5px; +} + +#searchform #searchsubmit { + background:url({{docroot}}/images/searchsubmit.gif) no-repeat center center; +} + +#searchform #s { + background:#555; +} + +.postmeta, +#searchform #s { + color: {{color.footer_bottom_text}}; +} + +#breadcrumbs { + font-size:12px; + float: left; + z-index: 1; + margin-bottom: 2em; +} + +#logo { + position: relative; + top: -5px; + margin-left: auto; + margin-right: auto; + display: block; + padding: 0px; + } + +#logo h1 { + padding:0; + margin:0; +} + +#logo-menubar { + position: absolute; + padding-left: 1em; +} + +#bro-eyes { + margin-bottom: 3em; + margin-left: auto; + margin-right: auto; + } + +.post { + margin-bottom:40px; + padding-bottom:20px; +} + +.entry-title { + padding-bottom:5px; +} + +.postmeta { + font-size:11px; + text-transform:uppercase; +} + +.post:last-child { + border:0; + margin-bottom:0; + padding-bottom:20px; +} + +.postmeta { +} + +a.more-link { + font-weight:bold; + text-decoration:none; +} + +.clearfix:after { + content: "."; + display: block; + clear: both; + visibility: hidden; + line-height: 0; + height: 0; +} + +.clearfix { + display: inline-block; +} + +html[xmlns] .clearfix { + display: block; +} +* html .clearfix { + height: 1%; +} + +.rounded { + -moz-border-radius:5px; + -webkit-border-radius:5px; + border-radius:5px; +} + +.message { + padding:8px; + margin-bottom:20px; +} + +.left { + float:left; +} + +.center { + display:block; + margin-left:auto; + margin-right:auto; + text-align:center; +} + +.right { + float:right; +} + +img.left { + padding:3px 20px 20px 0; +} + +img.right { + padding:3px 0 20px 20px; +} + +.widget img.left { + padding:5px 10px 10px 0; +} + +.widget img.right { + padding:5px 0 10px 10px; +} + +.hidden { + display:none; +} + +.pad { + padding:10px; +} + +UL.clean, OL.clean { + list-style-type:none; + margin:0; + padding:0; +} + +.primary-promo-container { + margin:0 0 40px 0; +} + +.promo-container { + position:relative; +} + +.promo-navigation { + position:absolute; + bottom:20px; + right:20px; + z-index:20; +} + +.promo-navigation a { + display:block; + float:left; + width:20px; + height:20px; + text-indent:-9999px; + margin-left:5px; +} + +.promo-navigation-small a { + width:10px; + height:10px; +} + +.promo-bottom-navigation { + padding-bottom:20px; + clear:both; +} + +.promo-bottom-navigation .promo-navigation { + bottom:0; + right:0; +} + +.pagination { + margin:0 0 40px 0; + line-height:30px; + font-size:12px; +} + +.pagination a, .pagination a span { + padding:5px 10px; + margin:2px; + text-decoration:none; +} + +.pagination span.current { + font-weight:bold; + padding:3px 3px; + margin:2px; +} + +.iconlist h5 { + line-height:100%; + font-size:15px; +} + +.iconlist h5, .iconlist h5 a { + font-weight:bold; + text-decoration:none; + padding:0; +} + +.iconlist p { + font-size:12px; +} + +.iconlist img.left { + margin:0 10px 10px 0; + padding:0; +} + +ul.iconlist, ul.iconlist li { + margin:0; + padding:0; +} + +p.quotebox { + padding:14px 20px 14px 40px; + line-height:100%; + margin:0 0 40px 0; + font-size:12px; +} + +p.quotebox { + background-position:10px center; + background-repeat:no-repeat; + background-image:url({{docroot}}/images/quote_left.gif); +} + +.ads { + padding:16px 0 0 16px; +} + +.ads img { + padding:0 16px 16px 0; + display:block; + float:left; +} + +.ads .widgettitle { + margin-right:16px; +} + +.grid-item { + padding:10px 0 0 0; + margin:0 0 40px 0; +} + +.grid-item p, .grid-item h3 { + padding-bottom:5px; +} + +/* Contact Form */ +form.standard .field { + padding:0 0 15px 0; +} + +form.standard label { + display:block; + padding:0 0 2px 2px; +} + +form.standard .textbox, +form.standard .textarea, +form.standard .select, +form.standard .button, +form.standard .checkbox, +form.standard .radio { + padding:8px 5px; +} + +form.standard .checkbox, form.standard .radio { + border:0; +} + +form.standard .button { + cursor:pointer; + padding:5px 8px; + line-height:100%; +} + +form.standard .error { + padding:1px 6px; + line-height:100%; +} + +form.standard .textbox, +form.standard .textarea, +form.standard .select, +form.standard .button, +form.standard .checkbox, +form.standard .radio { + border:1px solid #ccc; +} + +form.standard .button:hover { + background:#e5e5e5; +} + +.demogrid p { + font-size:12px; + font-weight:bold; + text-align:center; + margin:10px 0; + padding:8px 0; +} + +.faq h2 { + font-size: 17px; + } + +.faq p { + margin-left: 2em; + } + + +.invisible { + visibility: hidden; +} + +.paper-block { + font-size: 12px; + margin-left: 20px; +} + +.paper-title { + font-weight: bold; +} + +.contents { + padding: 10px; + background: #FFFAE2; + margin: 20px; + } + +.topic-title { + font-size: 20px; + font-weight: bold; + padding: 0px 0px 5px 0px; + text-align: center; + padding-top: .5em; +} + +.contents li { + margin-bottom: 0px; + list-style-type: square; +} + +.contents ul ul li { + margin-left: 0px; + padding-left: 0px; + padding-top: 0em; + font-size: 90%; + list-style-type: square; + font-weight: normal; +} + +.contents ul ul ul li { + list-style-type: none; +} + +.contents ul ul ul ul li { + display:none; +} + +.contents ul li { + padding-top: 1em; + list-style-type: none; + font-weight: bold; +} + +.contents ul { + margin-left: 0px; + padding-left: 2em; + margin: 0px 0px 0px 0px; +} + +#page-title { + font-size: 30px; + font-weight: bold; + color: {{color.text}}; + background: {{color.menu_selected_bg}}; + padding: 2px 2px 2px 0px; +} + + +.widget { + margin-left: 1em; +} + +.sidebar-toc ul li { + padding-bottom: 0px; + text-align: left; + list-style-type: square; + list-style-position: inside; + padding-left: 1em; + text-indent: -1em; + } + +.sidebar-toc ul li li { + margin-left: 1em; + margin-bottom: 0px; + list-style-type: square; + } + +.sidebar-toc ul li li a { + font-size: 8pt; +} + +a.toc-top-link { + font-size: 14px; + float: right; + margin-right: 15px; + } + +.small { + font-size: 16px; +} + +.heading { + font-weight: bold; + font-size: 18px; + color:{{color.emphasis_text}}; +} + +.shadow, .download, .note, .exercise, .question, .solution, .visible_solution, .contents, .warning, .error, .success, pre { + -moz-box-shadow:0 0 6px #ddd; + -webkit-box-shadow:0 0 6px #ddd; + box-shadow:0 0 6px #ddd; +} + +.exercise, .question, .solution, .visible_solution, .note, .warning, .error, .success, pre { + margin-left: 2em; + margin-right: 2em; + overflow: auto; + } + +.exercise, .question, .solution, .visible_solution, .note, .warning, .error, .success, pre { + margin-top: .5em; + margin-bottom: 1.5em; + } + +.exercise, .question, .solution, .visible_solution, .note, .warning, .error, .success { + border: solid 0px #aaa; + padding: 1em 1em 0em 1em; + font-size: 15px; + color: {{color.text}}; +} + +.note { + background: #fcfff4; /* Old browsers */ + background: -moz-linear-gradient(top, #fcfff4 0%, #e9e9ce 100%); /* FF3.6+ */ + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#fcfff4), color-stop(100%,#e9e9ce)); /* Chrome,Safari4+ */ + background: -webkit-linear-gradient(top, #fcfff4 0%,#e9e9ce 100%); /* Chrome10+,Safari5.1+ */ + background: -o-linear-gradient(top, #fcfff4 0%,#e9e9ce 100%); /* Opera 11.10+ */ + background: -ms-linear-gradient(top, #fcfff4 0%,#e9e9ce 100%); /* IE10+ */ + background: linear-gradient(top, #fcfff4 0%,#e9e9ce 100%); /* W3C */ + filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#fcfff4', endColorstr='#e9e9ce',GradientType=0 ); /* IE6-9 */ +} + +.exercise, .question { + background: #f0f0f0; /* Old browsers */ + background: -moz-linear-gradient(top, #f0f0f0 0%, #e9e9ce 100%); /* FF3.6+ */ + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#f0f0f0), color-stop(100%,#e9e9ce)); /* Chrome,Safari4+ */ + background: -webkit-linear-gradient(top, #f0f0f0 0%,#e9e9ce 100%); /* Chrome10+,Safari5.1+ */ + background: -o-linear-gradient(top, #f0f0f0 0%,#e9e9ce 100%); /* Opera 11.10+ */ + background: -ms-linear-gradient(top, #f0f0f0 0%,#e9e9ce 100%); /* IE10+ */ + background: linear-gradient(top, #f0f0f0 0%,#e9e9ce 100%); /* W3C */ + filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#f0f0f0', endColorstr='#e9e9ce',GradientType=0 ); /* IE6-9 */ +} + +.warning, .error { + background: #ffc578; /* Old browsers */ + background: -moz-linear-gradient(top, #ffc578 0%, #fb9d23 100%); /* FF3.6+ */ + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#ffc578), color-stop(100%,#fb9d23)); /* Chrome,Safari4+ */ + background: -webkit-linear-gradient(top, #ffc578 0%,#fb9d23 100%); /* Chrome10+,Safari5.1+ */ + background: -o-linear-gradient(top, #ffc578 0%,#fb9d23 100%); /* Opera 11.10+ */ + background: -ms-linear-gradient(top, #ffc578 0%,#fb9d23 100%); /* IE10+ */ + background: linear-gradient(top, #ffc578 0%,#fb9d23 100%); /* W3C */ + filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#ffc578', endColorstr='#fb9d23',GradientType=0 ); /* IE6-9 */ +} + +.solution, .visible_solution { + background: #f0c0c0; /* Old browsers */ + background: -moz-linear-gradient(top, #f0c0c0 0%, #f0c0f0 100%); /* FF3.6+ */ + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#f0c0c0), color-stop(100%,#f0c0f0)); /* Chrome,Safari4+ */ + background: -webkit-linear-gradient(top, #f0c0c0 0%,#f0c0f0 100%); /* Chrome10+,Safari5.1+ */ + background: -o-linear-gradient(top, #f0c0c0 0%,#f0c0f0 100%); /* Opera 11.10+ */ + background: -ms-linear-gradient(top, #f0c0c0 0%,#f0c0f0 100%); /* IE10+ */ + background: linear-gradient(top, #f0c0c0 0%,#f0c0f0 100%); /* W3C */ + filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#f0c0c0', endColorstr='#f0c0f0',GradientType=0 ); /* IE6-9 */ +} + +.admonition-title { + font-weight: normal; + font-size: 16px; + color: #000; + border-bottom: solid 1px #aaa; + padding-bottom: 0em; + margin-bottom: .5em; + margin-top: 0em; + } + +.download:hover { + background-color: #FCF3A4 +} + +.download { + border: solid 0px #eee; + background: #FFFAE2 url('../images/icons/download.png') 8px center no-repeat; + color: #333; + font-size: 20px; + right: 32px; + padding-top: 10px; + padding-bottom: 5px; + padding-left: 50px; + padding-right: 20px; + } + +.download-beta { + background: #FFCC99 url('../images/icons/download.png') 8px center no-repeat; +} + +/* .hll is pygments; see pygments.css */ +.hll { + color: {{color.text}}; + background: {{color.box_bg}}; + padding: 7px 5px 3px 5px; + margin-bottom: 25px; + margin-top: 0px; +} + +.field-list tbody tr:hover td { + background: {{color.bg}}; +} + +.field-list td, .field-list th { + border-bottom: 0px; + padding: 0 6px 0 0; +} + +.large-table td, .large-table th { + font-size: 80%; +} + +.large-table ul { + padding-left: 1em; + text-indent: -1em; + list-style-position: inside; +} + +.highlight { + background-color: {{color.highlight_bg}}; +} + +.hovershadow:hover, +.pagination a:hover { + -moz-box-shadow:0 0 3px #d0d0d0; + -webkit-box-shadow:0 0 3px #d0d0d0; + box-shadow:0 0 3px #d0d0d0; +} + +.iconlist h5, +.iconlist h5 a { + color:{{color.emphasis_text}}; +} +.iconlist h5, +.iconlist h5 a { + color:{{color.emphasis_text}}; +} + +form.standard .button, +.widgettitle +{ + color: {{color.text}}; +} + +.heading { + font-weight: bold; + font-size: 18px; + color:{{color.emphasis_text}}; +} + +.post, +.widgettitle { + border-bottom:1px solid #ddd; +} + +.line-block { + margin-bottom: 1em; + } + +.bibliography-title { + padding-top: 1em; + font-size: 18px; + font-weight: bold; + } + +.line-block { + margin-bottom: 1em; + } + +.bibliography-title { + padding-top: 1em; + font-size: 18px; + font-weight: bold; + } + +.promo-navigation a, +.pagination a, +.pagination a span, +p.quotebox, +.ads, +.boxed { + background:#fff; +} + +.promo-navigation a, +.pagination a, +.pagination a span, +.ads, +.demogrid p { + border:1px solid #ddd; +} + +table.docutils { + margin-bottom: 40px; +} + +table.docutils th { + font-size: 15px; + font-weight: bold; + padding: 10px 8px; + border-bottom: 1px solid #aaa; + vertical-align: top; +} + +table.docutils td { + font-size: 14px; + border-bottom: 1px solid #aaa; + padding: 6px 8px; + vertical-align: top; +} + +table.docutils tr:first-child td { + border-top: 1px solid #aaa; +} + +table.docutils tbody tr:hover td { + background: {{color.table_hover_bg}}; +} + +div.sphinxsidebar { + top: 30px; + bottom: 0; + margin: 0; + position: fixed; + overflow: auto; + height: auto; + float: right; + right: 0; +} + +/* this is nice, but it it leads to hidden headings when jumping + to an anchor */ +/* +div.related { + position: fixed; +} + +div.documentwrapper { + margin-top: 30px; +} +*/ + +div.sphinxsidebar h3 { + font-size: 1.4em; + font-weight: normal; + margin: 0; + padding: 0; +} + + +div.sphinxsidebar h4 { + font-size: 1.3em; + font-weight: normal; + margin: 5px 0 0 0; + padding: 0; +} + +div.sphinxsidebar p.topless { + margin: 5px 10px 10px 10px; +} + +div.sphinxsidebar ul { + margin: 10px; + padding: 0; +} + +div.sphinxsidebar input { + border: 1px solid {{ color.link }}; + font-family: sans-serif; + font-size: 1em; +} diff --git a/doc/scripts/source/_static/bro-framing.css_t b/doc/scripts/source/_static/bro-framing.css_t new file mode 100644 index 0000000000..1458c239a4 --- /dev/null +++ b/doc/scripts/source/_static/bro-framing.css_t @@ -0,0 +1,482 @@ + +{% import "bro-colors.css" as color %} + +#header { + padding: 20px 0; + position:relative; + z-index:2; +} + +#header { + background: {{color.header_bg}}; /* old browsers */ + background: -moz-linear-gradient(top, {{color.header_bg}} 80%, {{color.bg}} 100%); /* firefox */ + background: -webkit-gradient(linear, left top, left bottom, color-stop(80%,{{color.header_bg}}), color-stop(100%,{{color.bg}})); /* webkit */ + filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='{{color.header_bg}}', endColorstr='{{color.bg}}',GradientType=0 ); /* ie */ + zoom: 1; +} + +#header { + /* border-top: 20px solid #fff; + border-top:10px solid #fff; + border-bottom:1px solid #ddd; */ +} + +#header { + padding-bottom: 0em; + margin-bottom: 1em; + } + +#bro-main .widgettitle, +{ + color: {{color.text}}; +} + +#topbar p { + padding:0; + font-size:12px; + line-height:35px; +} + +#topbar a +{ + color: {{color.bg}}; +} + + +#topbar { + background: {{color.topbar_bg}}; +} + +#topbar a { + text-decoration:none; +} + +#topbar #searchform { + margin-top:6px; +} + +#topbar { + color: {{color.footer_bg}}; +} + +#footer { + padding:40px 0 0 0; +} + +#footer { + color: {{color.footer_text}}; + background: {{color.footer_bg}}; + } + +#footer .widgettitle { + color: {{color.footer_text}}; +} + +#footer-bottom { + padding:10px 0; + font-size:12px; +} + +#footer-bottom a.scroll-top { + padding-left:15px; + text-decoration:none; +} + +#footer a { + text-decoration:none; +} + +#footer .widget li { + padding-bottom:10px; +} + +#footer .widget_links li { + padding-bottom:1px; +} + +#footer .widget li:last-child { + padding-bottom:0; +} + +#footer-bottom { + color: {{color.footer_bottom_text}}; + background: {{color.footer_bottom_bg}}; +} + +#footer-bottom a:hover, +#footer .widget_links ul a:hover, +#footer .widget_links ol a:hover +{ + color:{{color.emphasis_text}}; +} + +#footer .widget_links ul a, +#footer .widget_links ol a +{ + color: {{color.footer_links}}; +} + + +#footer { + border-bottom:1px solid #444; +} + +#footer-bottom { + border-top:1px solid #222; +} + +#footer-bottom a.scroll-top { + background:url({{docroot}}/images/to-top.gif) no-repeat left center; +} + + +li.tweet_odd, li.tweet_even { + text-align: left; +} + +.line-block { + margin-bottom: 1em; + } + +.bibliography-title { + padding-top: 1em; + font-size: 18px; + font-weight: bold; + } + +.postmeta, +#footer-bottom a, +#searchform #s { + color: {{color.footer_bottom_text}}; +} +.line-block { + margin-bottom: 1em; + } + +.bibliography-title { + padding-top: 1em; + font-size: 18px; + font-weight: bold; + } + +#footer-bottom a { + color: {{color.footer_bottom_text}}; +} + +.widget { + margin:0 0 40px 0; +} + +.widget, .widgettitle { + font-size:12px; + line-height:18px; +} + +.widgettitle { + font-weight:bold; + text-transform:uppercase; + padding:0 0 10px 0; + margin:0 0 20px 0; + line-height:100%; +} + +.widget UL, .widget OL { + list-style-type:none; + margin:0; + padding:0; +} + +.widget p { + padding:0; +} + +.textwidget p { + padding:0 0 10px 0; +} + +.widget li { + padding-bottom:10px; +} + +.widget a { + text-decoration:none; +} + +.sf-menu, .sf-menu * { + margin: 0; + padding: 0; + list-style: none; + font-size:16px; +} +.sf-menu { + line-height:100%; + right:0; + bottom:0; + float: left; +} + +.sf-menu ul { + position: absolute; + top: -999em; + width: 20em; /* left offset of submenus need to match (see below) */ +} +.sf-menu ul li { + width: 100%; +} +.sf-menu li:hover { + visibility: inherit; /* fixes IE7 'sticky bug' */ +} +.sf-menu li { + float: left; + position: relative; +} +.sf-menu a { + display: block; + position: relative; + padding: .75em 1em; + text-decoration:none; + font-weight:bold; +} + +.sf-menu ul ul { + width: 100%; +} + +.sf-menu li:hover ul, +.sf-menu li.sfHover ul { +/* left: 0; */ + top: 2.5em; /* match top ul list item height */ + z-index: 99; +} +ul.sf-menu li:hover li ul, +ul.sf-menu li.sfHover li ul { + top: -999em; +} +ul.sf-menu li li:hover ul, +ul.sf-menu li li.sfHover ul { +/* left: 10em; */ /* match ul width */ + top: -11px; + margin-left: 2px; +} +ul.sf-menu li li:hover li ul, +ul.sf-menu li li.sfHover li ul { + top: -999em; +} +ul.sf-menu li li li:hover ul, +ul.sf-menu li li li.sfHover ul { + left: 10em; /* match ul width */ + top: -11px; +} + +.sf-menu ul a { + font-size:12px; +} + +.sf-menu ul li a { + font-weight: normal; + font-size: 13px; +} + +.sf-menu li ul { + padding:10px; +} + +.sf-menu a.sf-with-ul { + padding-right: 30px; + min-width: 1px; /* trigger IE7 hasLayout so spans position accurately */ +} + +.sf-menu li li +.sf-menu a { + border-bottom:1px solid #ddd; +} + +.sf-sub-indicator { + position: absolute; + display: block; + right: 10px; + top: 1.05em; /* IE6 only */ + width: 10px; + height: 10px; + text-indent: -999em; + overflow: hidden; +} + +.sf-menu a:hover, +.sfHover a, +.sf-menu ul li.sfHover > a +{ + color:{{color.emphasis_text}}; +} + +a > .sf-sub-indicator { /* give all except IE6 the correct values */ + top: 1em; + background-position: 0 -100px; /* use translucent arrow for modern browsers*/ +} + +a:focus > .sf-sub-indicator, +a:hover > .sf-sub-indicator, +a:active > .sf-sub-indicator, +li:hover > a > .sf-sub-indicator, +li.sfHover > a > .sf-sub-indicator { + background-position: -10px -100px; /* arrow hovers for modern browsers*/ +} + +.sf-menu ul .sf-sub-indicator { background-position: -10px 0; top:0.75em; } +.sf-menu ul a > .sf-sub-indicator { background-position: 0 0; } + +.sf-menu ul a:focus > .sf-sub-indicator, +.sf-menu ul a:hover > .sf-sub-indicator, +.sf-menu ul a:active > .sf-sub-indicator, +.sf-menu ul li:hover > a > .sf-sub-indicator, +.sf-menu ul li.sfHover > a > .sf-sub-indicator { + background-position: -10px 0; /* arrow hovers for modern browsers*/ +} + +.sf-menu ul a:hover { + background: {{color.menu_hover_bg}}; +} +/* +.sf-menu li li { + background: {{color.menu_bg}}; + border-left:1px solid {{color.menu_border}}; + border-right:1px solid {{color.menu_border}}; +} +.sf-menu li li:first-child { + border-top:1px solid {{color.menu_border}}; +} +.sf-menu li ul { + -moz-box-shadow:2px 2px 2px {{color.menu_shadow}}; + -webkit-box-shadow:2px 2px 2px {{color.menu_shadow}}; + box-shadow:2px 2px 2px {{color.menu_shadow}}; +} +*/ + +.sf-menu li { + border-bottom: 1px solid #000; +} + +.sf-menu ul li { + border-top: 1px solid #000; +} + +.sf-menu ul li { + border-bottom: 0px; +} + +.sf-menu li { + display: inline; +} + +.sf-menu a +{ + color: {{color.text}}; +} + +.sf-sub-indicator { + background:url('{{docroot}}/images/menu/default-submenu-sprite.png') no-repeat -10px -100px; /* 8-bit indexed alpha png. IE6 gets solid image only */ +} + +.sf-navbar { + height: 2.5em; + padding-bottom: 2.5em; + position: relative; +} +.sf-navbar li { + position: static; +} +.sf-navbar a { + border-top: none; +} +.sf-navbar li li { + position: relative; +} +.sf-navbar li li ul { + width: 13em; +} +.sf-navbar li li li { + width: 100%; +} +.sf-navbar ul li { + width: auto; + float: left; +} +.sf-navbar a, .sf-navbar a:visited { + border: none; +} +.sf-navbar li.current-menu { +} + +ul.sf-navbar ul li:hover li, +ul.sf-navbar ul li.sfHover li, +.sf-navbar ul a:focus, .sf-navbar ul a:hover, .sf-navbar ul a:active { + background: {{color.menu_hover_bg}}; +} +ul.sf-navbar li li li:hover, +ul.sf-navbar li li li.sfHover, +.sf-navbar li li.current-menu li.current-menu, +.sf-navbar ul li li a:focus, .sf-navbar ul li li a:hover, .sf-navbar ul li li a:active { + background: {{color.menu_hover_bg}}; +} +ul.sf-navbar .current-menu ul, +ul.sf-navbar ul li:hover ul, +ul.sf-navbar ul li.sfHover ul { + left: 0; + top: 2.5em; /* match top ul list item height */ +} +ul.sf-navbar .current-menu ul ul { + top: -999em; +} + +.sf-navbar li li.current-menu > a { + font-weight: bold; +} + +/*** point all arrows down ***/ +/* point right for anchors in subs */ +.sf-navbar ul .sf-sub-indicator { background-position: -10px -100px; } +.sf-navbar ul a > .sf-sub-indicator { background-position: 0 -100px; } +/* apply hovers to modern browsers */ +.sf-navbar ul a:focus > .sf-sub-indicator, +.sf-navbar ul a:hover > .sf-sub-indicator, +.sf-navbar ul a:active > .sf-sub-indicator, +.sf-navbar ul li:hover > a > .sf-sub-indicator, +.sf-navbar ul li.sfHover > a > .sf-sub-indicator { + background-position: -10px -100px; /* arrow hovers for modern browsers*/ +} + +/*** remove shadow on first submenu ***/ +.sf-navbar > li > ul { + background: transparent; + padding: 0; + -moz-border-radius-bottomleft: 0; + -moz-border-radius-topright: 0; + -webkit-border-top-right-radius: 0; + -webkit-border-bottom-left-radius: 0; + +} + +.selected-menu { +} + +#current-menu { + background: {{color.menu_selected_bg}}; +} + +#current-submenu { + font-weight: bold; + background: transparent; +} + +#current-submenu:hover { + background: {{color.menu_hover_bg}}; +} + +#current-submenu { + color: {{ color.text }}; +} + +#current-submenu:hover { + color:{{color.emphasis_text}}; +} + diff --git a/doc/scripts/source/_templates/layout.html b/doc/scripts/source/_templates/layout.html index 5685e1dc37..120c942c4c 100644 --- a/doc/scripts/source/_templates/layout.html +++ b/doc/scripts/source/_templates/layout.html @@ -1,5 +1,23 @@ {% extends "!layout.html" %} -{% block extrahead %} - - {{ super() }} +{% set css_files = css_files + ["_static/bro-base.css", "_static/bro-framing.css", "_static/bro-elements.css"] %} + +{% block relbar1 %}{% endblock %} + +{% block relbar2 %}{% endblock %} + +{% block sidebar2 %} +
+ {{ super() }} +
+{% endblock %} + +{% block content %} +
+ {{ relbar() }} +
+
+ {{ super() }} +
+
+
{% endblock %} From 6e6073ff4c4e01ab3171d3724dd6fe5f44df463e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 17 Oct 2011 14:19:17 -0700 Subject: [PATCH 018/964] it compiles (but doesn't do anything useful) --- src/CMakeLists.txt | 4 ++ src/Func.cc | 3 ++ src/InputMgr.cc | 90 +++++++++++++++++++++++++++++++++++++++++ src/InputMgr.h | 24 +++++++++++ src/InputReader.cc | 12 ++++++ src/InputReader.h | 29 +++++++++++++ src/InputReaderAscii.cc | 12 ++++++ src/InputReaderAscii.h | 20 +++++++++ src/NetVar.cc | 2 + src/input.bif | 16 ++++++++ src/main.cc | 4 ++ src/types.bif | 7 ++++ 12 files changed, 223 insertions(+) create mode 100644 src/InputMgr.cc create mode 100644 src/InputMgr.h create mode 100644 src/InputReader.cc create mode 100644 src/InputReader.h create mode 100644 src/InputReaderAscii.cc create mode 100644 src/InputReaderAscii.h create mode 100644 src/input.bif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b4779e1557..1693bad4eb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -141,6 +141,7 @@ endmacro(GET_BIF_OUTPUT_FILES) set(BIF_SRCS bro.bif logging.bif + input.bif event.bif const.bif types.bif @@ -332,6 +333,9 @@ set(bro_SRCS IRC.cc List.cc Reporter.cc + InputMgr.cc + InputReader.cc + InputReaderAscii.cc LogMgr.cc LogWriter.cc LogWriterAscii.cc diff --git a/src/Func.cc b/src/Func.cc index 65cb22b09d..829bc89238 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -523,11 +523,13 @@ void builtin_error(const char* msg, BroObj* arg) #include "bro.bif.func_h" #include "logging.bif.func_h" +#include "input.bif.func_h" #include "reporter.bif.func_h" #include "strings.bif.func_h" #include "bro.bif.func_def" #include "logging.bif.func_def" +#include "input.bif.func_def" #include "reporter.bif.func_def" #include "strings.bif.func_def" @@ -542,6 +544,7 @@ void init_builtin_funcs() #include "bro.bif.func_init" #include "logging.bif.func_init" +#include "input.bif.func_init" #include "reporter.bif.func_init" #include "strings.bif.func_init" diff --git a/src/InputMgr.cc b/src/InputMgr.cc new file mode 100644 index 0000000000..cacc512bcf --- /dev/null +++ b/src/InputMgr.cc @@ -0,0 +1,90 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include + +#include "InputMgr.h" +#include "Event.h" +#include "EventHandler.h" +#include "NetVar.h" +#include "Net.h" + + +#include "InputReader.h" + +#include "InputReaderAscii.h" + + +struct InputReaderDefinition { + bro_int_t type; // the type + const char *name; // descriptive name for error messages + bool (*init)(); // optional one-time inifializing function + InputReader* (*factory)(); // factory function for creating instances +}; + +InputReaderDefinition input_readers[] = { + { BifEnum::Input::READER_ASCII, "Ascii", 0, InputReaderAscii::Instantiate }, + + // End marker + { BifEnum::Input::READER_DEFAULT, "None", 0, (InputReader* (*)())0 } +}; + +InputMgr::InputMgr() +{ + DBG_LOG(DBG_LOGGING, "this has to happen"); +} + + +// create a new input reader object to be used at whomevers leisure lateron. +InputReader* InputMgr::CreateReader(EnumVal* reader, string source) +{ + InputReaderDefinition* ir = input_readers; + exit(12); + + while ( true ) { + if ( ir->type == BifEnum::Input::READER_DEFAULT ) + { + DBG_LOG(DBG_LOGGING, "unknown reader when creating reader"); + reporter->Error("unknown reader when creating reader"); + return 0; + } + + if ( ir->type != reader->AsEnum() ) { + // no, didn't find the right one... + ++ir; + continue; + } + + + // call init function of writer if presnt + if ( ir->init ) + { + if ( (*ir->init)() ) + { + //clear it to be not called again + ir->init = 0; + } else { + // ohok. init failed, kill factory for all eternity + ir->factory = 0; + DBG_LOG(DBG_LOGGING, "failed to init input class %s", ir->name); + return 0; + } + + } + + if ( !ir->factory ) + // no factory? + return 0; + + // all done. break. + break; + } + + assert(ir->factory); + InputReader* reader_obj = (*ir->factory)(); + assert(reader_obj); + + return reader_obj; + +} + + diff --git a/src/InputMgr.h b/src/InputMgr.h new file mode 100644 index 0000000000..978481afba --- /dev/null +++ b/src/InputMgr.h @@ -0,0 +1,24 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef INPUTMGR_H +#define INPUTMGR_H + +#include "InputReader.h" +#include "BroString.h" + +#include "Val.h" +#include "EventHandler.h" +#include "RemoteSerializer.h" + + +class InputMgr { +public: + InputMgr(); + + InputReader* CreateReader(EnumVal* reader, string source); +}; + +extern InputMgr* input_mgr; + + +#endif /* INPUTMGR_H */ diff --git a/src/InputReader.cc b/src/InputReader.cc new file mode 100644 index 0000000000..f2b3b05801 --- /dev/null +++ b/src/InputReader.cc @@ -0,0 +1,12 @@ + +#include "InputReader.h" + +InputReader::InputReader() +{ + +} + +InputReader::~InputReader() +{ + +} \ No newline at end of file diff --git a/src/InputReader.h b/src/InputReader.h new file mode 100644 index 0000000000..ce8303383a --- /dev/null +++ b/src/InputReader.h @@ -0,0 +1,29 @@ +// See the file "COPYING" in the main distribution directory for copyright. +// +// Same notes about thread safety as in LogWriter.h apply. + + +#ifndef INPUTREADER_H +#define INPUTREADER_H + +class InputReader { +public: + InputReader(); + virtual ~InputReader(); + +protected: + // Methods that have to be overwritten by the individual readers + +private: + friend class InputMgr; + + // When an error occurs, this method is called to set a flag marking the + // writer as disabled. + + bool disabled; + + bool Disabled() { return disabled; } +}; + + +#endif /* INPUTREADER_H */ diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc new file mode 100644 index 0000000000..97933c2a6e --- /dev/null +++ b/src/InputReaderAscii.cc @@ -0,0 +1,12 @@ + +#include "InputReaderAscii.h" +#include "DebugLogger.h" + +InputReaderAscii::InputReaderAscii() +{ + DBG_LOG(DBG_LOGGING, "input reader initialized"); +} + +InputReaderAscii::~InputReaderAscii() +{ +} \ No newline at end of file diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h new file mode 100644 index 0000000000..c471639fd4 --- /dev/null +++ b/src/InputReaderAscii.h @@ -0,0 +1,20 @@ + +#ifndef INPUTREADERASCII_H +#define INPUTREADERASCII_H + +#include "InputReader.h" + +class InputReaderAscii : public InputReader { +public: + InputReaderAscii(); + ~InputReaderAscii(); + + static InputReader* Instantiate() { return new InputReaderAscii; } + +protected: + +private: +}; + + +#endif /* INPUTREADERASCII_H */ diff --git a/src/NetVar.cc b/src/NetVar.cc index 25e4f7a0bc..5a6ac96fba 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -255,6 +255,7 @@ StringVal* cmd_line_bpf_filter; #include "types.bif.netvar_def" #include "event.bif.netvar_def" #include "logging.bif.netvar_def" +#include "input.bif.netvar_def" #include "reporter.bif.netvar_def" void init_event_handlers() @@ -315,6 +316,7 @@ void init_net_var() #include "const.bif.netvar_init" #include "types.bif.netvar_init" #include "logging.bif.netvar_init" +#include "input.bif.netvar_init" #include "reporter.bif.netvar_init" conn_id = internal_type("conn_id")->AsRecordType(); diff --git a/src/input.bif b/src/input.bif new file mode 100644 index 0000000000..edb6b4e9bb --- /dev/null +++ b/src/input.bif @@ -0,0 +1,16 @@ +# functions and types for the input framework + +module Input; + +%%{ +#include "InputMgr.h" +#include "NetVar.h" +%%} + +function Input::__create_reader%(reader: Input::Reader, source: string%) : bool + %{ + exit(5); + InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString()); + return new Val( the_reader != 0, TYPE_BOOL ); + %} + diff --git a/src/main.cc b/src/main.cc index dfa46c3050..b3f2512b40 100644 --- a/src/main.cc +++ b/src/main.cc @@ -30,6 +30,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "File.h" #include "Reporter.h" #include "LogMgr.h" +#include "InputMgr.h" #include "Net.h" #include "NetVar.h" #include "Var.h" @@ -72,6 +73,7 @@ name_list prefixes; DNS_Mgr* dns_mgr; TimerMgr* timer_mgr; LogMgr* log_mgr; +InputMgr* input_mgr; Stmt* stmts; EventHandlerPtr net_done = 0; RuleMatcher* rule_matcher = 0; @@ -724,6 +726,8 @@ int main(int argc, char** argv) remote_serializer = new RemoteSerializer(); event_registry = new EventRegistry(); log_mgr = new LogMgr(); + + input_mgr = new InputMgr(); if ( events_file ) event_player = new EventPlayer(events_file); diff --git a/src/types.bif b/src/types.bif index da6bd6e031..ee43207ddd 100644 --- a/src/types.bif +++ b/src/types.bif @@ -167,4 +167,11 @@ enum ID %{ Unknown, %} +module Input; + +enum Reader %{ + READER_DEFAULT, + READER_ASCII, +%} + module GLOBAL; From 0eafeb03693e3134bb47219df43066a390a42d55 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 19 Oct 2011 13:16:09 -0700 Subject: [PATCH 019/964] works (thanks to robin) --- scripts/base/init-bare.bro | 2 ++ src/NetVar.h | 1 + src/input.bif | 1 - 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 859a69f2dc..8d1d2a312f 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -1508,3 +1508,5 @@ const parse_udp_tunnels = F &redef; # Load the logging framework here because it uses fairly deep integration with # BiFs and script-land defined types. @load base/frameworks/logging + +@load base/input.bif diff --git a/src/NetVar.h b/src/NetVar.h index f8def230c0..957a86aeb3 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -264,6 +264,7 @@ extern void init_net_var(); #include "types.bif.netvar_h" #include "event.bif.netvar_h" #include "logging.bif.netvar_h" +#include "input.bif.netvar_h" #include "reporter.bif.netvar_h" #endif diff --git a/src/input.bif b/src/input.bif index edb6b4e9bb..3ff5284c63 100644 --- a/src/input.bif +++ b/src/input.bif @@ -9,7 +9,6 @@ module Input; function Input::__create_reader%(reader: Input::Reader, source: string%) : bool %{ - exit(5); InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString()); return new Val( the_reader != 0, TYPE_BOOL ); %} From f8be3519c7b868cd23829141b9588396df50e6ac Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 19 Oct 2011 15:41:07 -0700 Subject: [PATCH 020/964] well, it compiles. and perhaps it sends an event. billiant. --- src/InputMgr.cc | 20 +++++++++++++++++--- src/InputMgr.h | 10 +++++++++- src/InputReader.cc | 19 +++++++++++++++++++ src/InputReader.h | 15 ++++++++++++++- src/input.bif | 6 ++++-- 5 files changed, 63 insertions(+), 7 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index cacc512bcf..5ab602f4f3 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -35,11 +35,17 @@ InputMgr::InputMgr() // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* reader, string source) +InputReader* InputMgr::CreateReader(EnumVal* reader, string source, string eventName, RecordVal* eventDescription) { InputReaderDefinition* ir = input_readers; - exit(12); - + + RecordType* rtype = eventDescription->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::Event, 0) ) + { + reporter->Error("eventDescription argument not of right type"); + return 0; + } + while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) { @@ -86,5 +92,13 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, string source) return reader_obj; } + +void InputMgr::Error(InputReader* reader, const char* msg) +{ + reporter->Error(fmt("error with input reader for %s: %s", + reader->Source().c_str(), msg)); +} + + diff --git a/src/InputMgr.h b/src/InputMgr.h index 978481afba..3d3d0f2d3f 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -10,12 +10,20 @@ #include "EventHandler.h" #include "RemoteSerializer.h" +class InputReader; class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* reader, string source); + InputReader* CreateReader(EnumVal* reader, string source, string eventName, RecordVal* eventDescription); + +protected: + friend class InputReader; + + // Reports an error for the given reader. + void Error(InputReader* reader, const char* msg); + }; extern InputMgr* input_mgr; diff --git a/src/InputReader.cc b/src/InputReader.cc index f2b3b05801..ef47bb1e10 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -1,5 +1,7 @@ #include "InputReader.h" +#include "EventRegistry.h" +#include "Event.h" InputReader::InputReader() { @@ -9,4 +11,21 @@ InputReader::InputReader() InputReader::~InputReader() { +} + +void InputReader::Error(const char *msg) +{ + input_mgr->Error(this, msg); +} + +bool InputReader::Init(string source, string eventName) { + EventHandler* handler = event_registry->Lookup(eventName.c_str()); + + if ( handler == 0 ) { + reporter->Error("Event %s not found", eventName.c_str()); + return false; + } + + mgr.Dispatch(new Event(handler, 0)); + return true; } \ No newline at end of file diff --git a/src/InputReader.h b/src/InputReader.h index ce8303383a..58a56e8221 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -6,16 +6,29 @@ #ifndef INPUTREADER_H #define INPUTREADER_H +#include "InputMgr.h" +#include "BroString.h" + class InputReader { public: InputReader(); virtual ~InputReader(); + + bool Init(string source, string eventName); protected: // Methods that have to be overwritten by the individual readers - + + // Reports an error to the user. + void Error(const char *msg); + + // The following methods return the information as passed to Init(). + const string Source() const { return source; } + private: friend class InputMgr; + + string source; // When an error occurs, this method is called to set a flag marking the // writer as disabled. diff --git a/src/input.bif b/src/input.bif index 3ff5284c63..e2fc2ae91e 100644 --- a/src/input.bif +++ b/src/input.bif @@ -7,9 +7,11 @@ module Input; #include "NetVar.h" %%} -function Input::__create_reader%(reader: Input::Reader, source: string%) : bool +type Event: record; + +function Input::__create_reader%(reader: Input::Reader, source: string, eventName: string, eventDescription: Input::Event%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString()); + InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString(), eventName->AsString()->CheckString(), eventDescription->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} From 9c8b0dec3b8622a592aa1c787d9bde7d53f45f5f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 19 Oct 2011 16:35:34 -0700 Subject: [PATCH 021/964] event from c++ to script works (at last...) --- scripts/base/frameworks/input/__load__.bro | 1 + scripts/base/frameworks/input/main.bro | 11 +++++++++++ scripts/base/init-bare.bro | 3 ++- src/InputMgr.cc | 7 ++++--- src/InputMgr.h | 2 +- src/InputReader.cc | 19 +++++++++++-------- src/LogMgr.cc | 2 +- src/input.bif | 4 ++-- 8 files changed, 33 insertions(+), 16 deletions(-) create mode 100644 scripts/base/frameworks/input/__load__.bro create mode 100644 scripts/base/frameworks/input/main.bro diff --git a/scripts/base/frameworks/input/__load__.bro b/scripts/base/frameworks/input/__load__.bro new file mode 100644 index 0000000000..a10fe855df --- /dev/null +++ b/scripts/base/frameworks/input/__load__.bro @@ -0,0 +1 @@ +@load ./main diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro new file mode 100644 index 0000000000..cb071872ac --- /dev/null +++ b/scripts/base/frameworks/input/main.bro @@ -0,0 +1,11 @@ + +module Input; + +export { + type Event: record { + name: string; + columns: any; + }; +} + +@load base/input.bif diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 8d1d2a312f..ade92bfd6e 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -1509,4 +1509,5 @@ const parse_udp_tunnels = F &redef; # BiFs and script-land defined types. @load base/frameworks/logging -@load base/input.bif +@load base/frameworks/input + diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5ab602f4f3..f3fb2ff935 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -35,11 +35,11 @@ InputMgr::InputMgr() // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* reader, string source, string eventName, RecordVal* eventDescription) +InputReader* InputMgr::CreateReader(EnumVal* reader, string source, RecordVal* event) { InputReaderDefinition* ir = input_readers; - RecordType* rtype = eventDescription->Type()->AsRecordType(); + RecordType* rtype = InputReaderDefinition->Type()->AsRecordType(); if ( ! same_type(rtype, BifType::Record::Input::Event, 0) ) { reporter->Error("eventDescription argument not of right type"); @@ -49,7 +49,6 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, string source, string event while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) { - DBG_LOG(DBG_LOGGING, "unknown reader when creating reader"); reporter->Error("unknown reader when creating reader"); return 0; } @@ -89,6 +88,8 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, string source, string event InputReader* reader_obj = (*ir->factory)(); assert(reader_obj); + reader_obj->Init(source, eventName); + return reader_obj; } diff --git a/src/InputMgr.h b/src/InputMgr.h index 3d3d0f2d3f..e98f3d77b7 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -16,7 +16,7 @@ class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* reader, string source, string eventName, RecordVal* eventDescription); + InputReader* CreateReader(EnumVal* reader, string source, RecordVal* event); protected: friend class InputReader; diff --git a/src/InputReader.cc b/src/InputReader.cc index ef47bb1e10..fc9be7f2b6 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -1,7 +1,7 @@ #include "InputReader.h" -#include "EventRegistry.h" -#include "Event.h" +// #include "EventRegistry.h" +// #include "Event.h" InputReader::InputReader() { @@ -19,13 +19,16 @@ void InputReader::Error(const char *msg) } bool InputReader::Init(string source, string eventName) { - EventHandler* handler = event_registry->Lookup(eventName.c_str()); + //EventHandler* handler = event_registry->Lookup(eventName.c_str()); + + //if ( handler == 0 ) { + // reporter->Error("Event %s not found", eventName.c_str()); + // return false; + //} - if ( handler == 0 ) { - reporter->Error("Event %s not found", eventName.c_str()); - return false; - } + //val_list* vl = new val_list; + //vl->append(new Val(12, TYPE_COUNT)); - mgr.Dispatch(new Event(handler, 0)); + //mgr.Dispatch(new Event(handler, vl)); return true; } \ No newline at end of file diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 9e320f8810..fc4b89b0ed 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -475,7 +475,7 @@ void LogMgr::RemoveDisabledWriters(Stream* stream) stream->writers.erase(*j); } -bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) +bool LogMgr::(EnumVal* id, RecordVal* sval) { RecordType* rtype = sval->Type()->AsRecordType(); diff --git a/src/input.bif b/src/input.bif index e2fc2ae91e..88d4e32129 100644 --- a/src/input.bif +++ b/src/input.bif @@ -9,9 +9,9 @@ module Input; type Event: record; -function Input::__create_reader%(reader: Input::Reader, source: string, eventName: string, eventDescription: Input::Event%) : bool +function Input::__create_reader%(reader: Input::Reader, source: string, eventDescription: Input::Event%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString(), eventName->AsString()->CheckString(), eventDescription->AsRecordVal()); + InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString(), eventDescription->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} From 365406024627c8c2b0b3d41263840a7fbaf029ed Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 21 Oct 2011 14:01:18 -0700 Subject: [PATCH 022/964] compiles. sill doesn't do much. --- scripts/base/frameworks/input/main.bro | 8 ++++-- src/InputMgr.cc | 35 ++++++++++++++++++++---- src/InputMgr.h | 8 +++++- src/InputReader.cc | 29 ++++++++++---------- src/InputReader.h | 10 ++++++- src/InputReaderAscii.cc | 38 +++++++++++++++++++++++++- src/InputReaderAscii.h | 10 +++++++ src/LogMgr.cc | 2 +- src/input.bif | 6 ++-- 9 files changed, 117 insertions(+), 29 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index cb071872ac..b5a05af3b6 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -2,9 +2,11 @@ module Input; export { - type Event: record { - name: string; - columns: any; + type ReaderDescription: record { + source: string; + idx: any; + val: any; + destination: any; }; } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index f3fb2ff935..500a325249 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -35,14 +35,14 @@ InputMgr::InputMgr() // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* reader, string source, RecordVal* event) +InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) { InputReaderDefinition* ir = input_readers; - RecordType* rtype = InputReaderDefinition->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::Event, 0) ) + RecordType* rtype = description->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::ReaderDescription, 0) ) { - reporter->Error("eventDescription argument not of right type"); + reporter->Error("readerDescription argument not of right type"); return 0; } @@ -88,7 +88,11 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, string source, RecordVal* e InputReader* reader_obj = (*ir->factory)(); assert(reader_obj); - reader_obj->Init(source, eventName); + // get the source... + const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); + string source((const char*) bsource->Bytes(), bsource->Len()); + + reader_obj->Init(source, 0, NULL); return reader_obj; @@ -100,6 +104,27 @@ void InputMgr::Error(InputReader* reader, const char* msg) reader->Source().c_str(), msg)); } +/* + TODO: + +void InputMgr::SendEvent(string name) { + //EventHandler* handler = event_registry->Lookup(eventName.c_str()); + + //if ( handler == 0 ) { + // reporter->Error("Event %s not found", eventName.c_str()); + // return false; + //} + + //val_list* vl = new val_list; + //vl->append(new Val(12, TYPE_COUNT)); + + //mgr.Dispatch(new Event(handler, vl)); + + +} + +*/ + diff --git a/src/InputMgr.h b/src/InputMgr.h index e98f3d77b7..255d61fe5e 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -9,20 +9,26 @@ #include "Val.h" #include "EventHandler.h" #include "RemoteSerializer.h" +#include "LogMgr.h" // for the LogVal and LogType data types class InputReader; + class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* reader, string source, RecordVal* event); + InputReader* CreateReader(EnumVal* reader, RecordVal* description); protected: friend class InputReader; // Reports an error for the given reader. void Error(InputReader* reader, const char* msg); + +private: + // required functionality + // InputValsToRecord to convert received inputvals back to bro records / tables / whatever }; diff --git a/src/InputReader.cc b/src/InputReader.cc index fc9be7f2b6..4d29040e60 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -5,7 +5,7 @@ InputReader::InputReader() { - + disabled = true; // disabled will be set correcty in init. } InputReader::~InputReader() @@ -18,17 +18,18 @@ void InputReader::Error(const char *msg) input_mgr->Error(this, msg); } -bool InputReader::Init(string source, string eventName) { - //EventHandler* handler = event_registry->Lookup(eventName.c_str()); - - //if ( handler == 0 ) { - // reporter->Error("Event %s not found", eventName.c_str()); - // return false; - //} - - //val_list* vl = new val_list; - //vl->append(new Val(12, TYPE_COUNT)); - - //mgr.Dispatch(new Event(handler, vl)); - return true; +bool InputReader::Init(string arg_source, int arg_num_fields, + const LogField* const * arg_fields) +{ + source = arg_source; + num_fields = arg_num_fields; + fields = arg_fields; + + // disable if DoInit returns error. + disabled = !DoInit(arg_source, arg_num_fields, arg_fields); + return !disabled; +} + +void InputReader::Finish() { + DoFinish(); } \ No newline at end of file diff --git a/src/InputReader.h b/src/InputReader.h index 58a56e8221..f3638c7246 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -8,16 +8,22 @@ #include "InputMgr.h" #include "BroString.h" +#include "LogMgr.h" class InputReader { public: InputReader(); virtual ~InputReader(); - bool Init(string source, string eventName); + bool Init(string arg_source, int num_fields, const LogField* const* fields); + void Finish(); + protected: // Methods that have to be overwritten by the individual readers + virtual bool DoInit(string arg_source, int num_fields, const LogField* const * fields) = 0; + + virtual void DoFinish() = 0; // Reports an error to the user. void Error(const char *msg); @@ -29,6 +35,8 @@ private: friend class InputMgr; string source; + int num_fields; + const LogField* const * fields; // When an error occurs, this method is called to set a flag marking the // writer as disabled. diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 97933c2a6e..b9aab16815 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -2,11 +2,47 @@ #include "InputReaderAscii.h" #include "DebugLogger.h" +#include + InputReaderAscii::InputReaderAscii() { - DBG_LOG(DBG_LOGGING, "input reader initialized"); + //DBG_LOG(DBG_LOGGING, "input reader initialized"); + file = 0; } InputReaderAscii::~InputReaderAscii() { +} + +void InputReaderAscii::DoFinish() +{ +} + +bool InputReaderAscii::DoInit(string path, int num_fields, + const LogField* const * fields) +{ + fname = path; + + file = new ifstream(path.c_str()); + if ( !file->is_open() ) { + return false; + } + + // try to read the header line... + string line; + if ( !getline(*file, line) ) + return false; + + // split on tabs... + istringstream ss(line); + while ( ss ) { + string s; + if ( !getline(ss, s, '\t')) + break; + + + } + + + return false; } \ No newline at end of file diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index c471639fd4..0d2008ed7f 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -3,6 +3,9 @@ #define INPUTREADERASCII_H #include "InputReader.h" +#include +#include + class InputReaderAscii : public InputReader { public: @@ -12,8 +15,15 @@ public: static InputReader* Instantiate() { return new InputReaderAscii; } protected: + + virtual bool DoInit(string path, int num_fields, + const LogField* const * fields); + virtual void DoFinish(); private: + + ifstream* file; + string fname; }; diff --git a/src/LogMgr.cc b/src/LogMgr.cc index fc4b89b0ed..9e320f8810 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -475,7 +475,7 @@ void LogMgr::RemoveDisabledWriters(Stream* stream) stream->writers.erase(*j); } -bool LogMgr::(EnumVal* id, RecordVal* sval) +bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) { RecordType* rtype = sval->Type()->AsRecordType(); diff --git a/src/input.bif b/src/input.bif index 88d4e32129..3da869ea08 100644 --- a/src/input.bif +++ b/src/input.bif @@ -7,11 +7,11 @@ module Input; #include "NetVar.h" %%} -type Event: record; +type ReaderDescription: record; -function Input::__create_reader%(reader: Input::Reader, source: string, eventDescription: Input::Event%) : bool +function Input::__create_reader%(reader: Input::Reader, description: Input::ReaderDescription%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString(), eventDescription->AsRecordVal()); + InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} From d7a3b85fcda42a1485ef7a53c5419100e4b5ea95 Mon Sep 17 00:00:00 2001 From: amannb Date: Tue, 25 Oct 2011 11:47:23 -0700 Subject: [PATCH 023/964] many helper functions --- src/InputMgr.cc | 71 +++++++++++++++------ src/InputMgr.h | 4 +- src/InputReader.cc | 33 +++++++++- src/InputReader.h | 12 ++++ src/InputReaderAscii.cc | 134 +++++++++++++++++++++++++++++++++++++--- src/InputReaderAscii.h | 22 +++++++ 6 files changed, 245 insertions(+), 31 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 500a325249..ec46c55813 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -100,31 +100,62 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) void InputMgr::Error(InputReader* reader, const char* msg) { - reporter->Error(fmt("error with input reader for %s: %s", - reader->Source().c_str(), msg)); + reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); } -/* - TODO: -void InputMgr::SendEvent(string name) { - //EventHandler* handler = event_registry->Lookup(eventName.c_str()); - - //if ( handler == 0 ) { - // reporter->Error("Event %s not found", eventName.c_str()); - // return false; - //} - - //val_list* vl = new val_list; - //vl->append(new Val(12, TYPE_COUNT)); - - //mgr.Dispatch(new Event(handler, vl)); +void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) +{ + EventHandler* handler = event_registry->Lookup(name.c_str()); + if ( handler == 0 ) { + reporter->Error("Event %s not found", name.c_str()); + return; + } + val_list* vl = new val_list; + for ( int i = 0; i < num_vals; i++) { + vl->append(LogValToVal(vals[i])); + } + + mgr.Dispatch(new Event(handler, vl)); +} + +Val* InputMgr::LogValToVal(const LogVal* val) { + switch ( val->type ) { + case TYPE_BOOL: + case TYPE_INT: + return new Val(val->val.int_val, val->type); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + return new Val(val->val.uint_val, val->type); + break; + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + return new Val(val->val.double_val, val->type); + break; + + case TYPE_STRING: + { + BroString *s = new BroString(*(val->val.string_val)); + return new StringVal(s); + break; + } + + case TYPE_PORT: + return new PortVal(val->val.uint_val); + break; + + default: + reporter->InternalError("unsupported type for input_read"); + } + + + reporter->InternalError("Impossible error"); + return NULL; } - -*/ - - diff --git a/src/InputMgr.h b/src/InputMgr.h index 255d61fe5e..79f76a1e6f 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -13,7 +13,6 @@ class InputReader; - class InputMgr { public: InputMgr(); @@ -29,6 +28,9 @@ protected: private: // required functionality // InputValsToRecord to convert received inputvals back to bro records / tables / whatever + Val* LogValToVal(const LogVal* val); + + void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); }; diff --git a/src/InputReader.cc b/src/InputReader.cc index 4d29040e60..d812349733 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -32,4 +32,35 @@ bool InputReader::Init(string arg_source, int arg_num_fields, void InputReader::Finish() { DoFinish(); -} \ No newline at end of file +} + +bool InputReader::Update() { + return DoUpdate(); +} + +// stolen from logwriter +const char* InputReader::Fmt(const char* format, ...) + { + if ( ! buf ) + buf = (char*) malloc(buf_len); + + va_list al; + va_start(al, format); + int n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + + if ( (unsigned int) n >= buf_len ) + { // Not enough room, grow the buffer. + buf_len = n + 32; + buf = (char*) realloc(buf, buf_len); + + // Is it portable to restart? + va_start(al, format); + n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + } + + return buf; + } + + diff --git a/src/InputReader.h b/src/InputReader.h index f3638c7246..9d776276fa 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -18,12 +18,17 @@ public: bool Init(string arg_source, int num_fields, const LogField* const* fields); void Finish(); + + bool Update(); protected: // Methods that have to be overwritten by the individual readers virtual bool DoInit(string arg_source, int num_fields, const LogField* const * fields) = 0; virtual void DoFinish() = 0; + + // update file contents to logmgr + virtual bool DoUpdate() = 0; // Reports an error to the user. void Error(const char *msg); @@ -31,6 +36,9 @@ protected: // The following methods return the information as passed to Init(). const string Source() const { return source; } + // A thread-safe version of fmt(). (stolen from logwriter) + const char* Fmt(const char* format, ...); + private: friend class InputMgr; @@ -44,6 +52,10 @@ private: bool disabled; bool Disabled() { return disabled; } + + // For implementing Fmt(). + char* buf; + unsigned int buf_len; }; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index b9aab16815..4bc4b81cda 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -1,9 +1,22 @@ +// See the file "COPYING" in the main distribution directory for copyright. #include "InputReaderAscii.h" #include "DebugLogger.h" #include +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position) + : name(arg_name), type(arg_type) +{ + position = arg_position; +} + +FieldMapping::FieldMapping(const FieldMapping& arg) + : name(arg.name), type(arg.type) +{ + position = arg.position; +} + InputReaderAscii::InputReaderAscii() { //DBG_LOG(DBG_LOGGING, "input reader initialized"); @@ -18,31 +31,134 @@ void InputReaderAscii::DoFinish() { } -bool InputReaderAscii::DoInit(string path, int num_fields, - const LogField* const * fields) +bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const * fields) { fname = path; file = new ifstream(path.c_str()); if ( !file->is_open() ) { + Error(Fmt("cannot open %s", path.c_str())); return false; } // try to read the header line... string line; - if ( !getline(*file, line) ) + if ( !getline(*file, line) ) { + Error("could not read first line"); return false; + } // split on tabs... - istringstream ss(line); - while ( ss ) { + istringstream splitstream(line); + unsigned int currTab = 0; + int wantFields = 0; + while ( splitstream ) { string s; - if ( !getline(ss, s, '\t')) + if ( !getline(splitstream, s, '\t')) break; - + // current found heading in s... compare if we want it + for ( int i = 0; i < num_fields; i++ ) { + const LogField* field = fields[i]; + if ( field->name == s ) { + // cool, found field. note position + FieldMapping f(field->name, field->type, i); + columnMap.push_back(f); + wantFields++; + break; // done with searching + } + } + + // look if we did push something... + if ( columnMap.size() == currTab ) { + // no, we didn't. note that... + FieldMapping empty; + columnMap.push_back(empty); + } + + // done + currTab++; } + + if ( wantFields != num_fields ) { + // we did not find all fields? + // :( + Error("wantFields != num_fields"); + return false; + } + + this->num_fields = num_fields; - return false; -} \ No newline at end of file + // well, that seems to have worked... + return true; +} + +// read the entire file and send appropriate thingies back to InputMgr +bool InputReaderAscii::DoUpdate() { + // TODO: all the stuff we need for a second reading. + // *cough* + // + + + string line; + while ( getline(*file, line ) ) { + // split on tabs + + istringstream splitstream(line); + string s; + + LogVal fields[num_fields]; + + unsigned int currTab = 0; + unsigned int currField = 0; + while ( splitstream ) { + if ( !getline(splitstream, s, '\t') ) + break; + + + if ( currTab >= columnMap.size() ) { + Error("Tabs in heading do not match tabs in data?"); + //disabled = true; + return false; + } + + FieldMapping currMapping = columnMap[currTab]; + currTab++; + + if ( currMapping.IsEmpty() ) { + // well, that was easy + continue; + } + + if ( currField >= num_fields ) { + Error("internal error - fieldnum greater as possible"); + return false; + } + + LogVal val(currMapping.type, true); + + switch ( currMapping.type ) { + case TYPE_STRING: + val.val.string_val = new string(s); + + default: + Error(Fmt("unsupported field format %d for %s", currMapping.type, + currMapping.name.c_str())); + return false; + } + + currField++; + } + + if ( currField != num_fields ) { + Error("curr_field != num_fields in DoUpdate"); + return false; + } + + // ok, now we have built our line. send it back to... whomever. + + } + + return true; +} diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 0d2008ed7f..551a08b02e 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -1,3 +1,4 @@ +// See the file "COPYING" in the main distribution directory for copyright. #ifndef INPUTREADERASCII_H #define INPUTREADERASCII_H @@ -5,6 +6,19 @@ #include "InputReader.h" #include #include +#include + +// Description for input field mapping +struct FieldMapping { + string name; + TypeTag type; + int position; + + FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); + FieldMapping(const FieldMapping& arg); + FieldMapping() { position = -1; } + bool IsEmpty() { return position == -1; } +}; class InputReaderAscii : public InputReader { @@ -19,11 +33,19 @@ protected: virtual bool DoInit(string path, int num_fields, const LogField* const * fields); virtual void DoFinish(); + + virtual bool DoUpdate(); private: ifstream* file; string fname; + + unsigned int num_fields; + + // map columns in the file to columns to send back to the manager + vector columnMap; + }; From 5b0c307f87a7213951e586f6e881501dc658f03d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 25 Oct 2011 14:11:21 -0700 Subject: [PATCH 024/964] very basic input to event working... --- scripts/base/frameworks/input/main.bro | 4 ++-- src/InputMgr.cc | 18 ++++++++++++++++-- src/InputReader.cc | 11 +++++++++++ src/InputReader.h | 3 +++ src/InputReaderAscii.cc | 14 ++++++++++---- 5 files changed, 42 insertions(+), 8 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index b5a05af3b6..4bea9d73d2 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -5,8 +5,8 @@ export { type ReaderDescription: record { source: string; idx: any; - val: any; - destination: any; + val: any &optional; + destination: any &optional; }; } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index ec46c55813..5cd82b613f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -91,8 +91,22 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) // get the source... const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); string source((const char*) bsource->Bytes(), bsource->Len()); - - reader_obj->Init(source, 0, NULL); + + RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); + + LogField** fields = new LogField*[idx->NumFields()]; + for ( int i = 0; i < idx->NumFields(); i++ ) + { + // FIXME: do type checking... + LogField* field = new LogField(); + field->name = idx->FieldName(i); + field->type = idx->FieldType(i)->Tag(); + fields[i] = field; + } + + + reader_obj->Init(source, idx->NumFields(), fields); + reader_obj->Update(); return reader_obj; diff --git a/src/InputReader.cc b/src/InputReader.cc index d812349733..d512bc1699 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -5,6 +5,8 @@ InputReader::InputReader() { + buf = 0; + buf_len = 1024; disabled = true; // disabled will be set correcty in init. } @@ -18,6 +20,11 @@ void InputReader::Error(const char *msg) input_mgr->Error(this, msg); } +void InputReader::Error(const string &msg) + { + input_mgr->Error(this, msg.c_str()); + } + bool InputReader::Init(string arg_source, int arg_num_fields, const LogField* const * arg_fields) { @@ -38,6 +45,10 @@ bool InputReader::Update() { return DoUpdate(); } +void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { + input_mgr->SendEvent(name, num_vals, vals); +} + // stolen from logwriter const char* InputReader::Fmt(const char* format, ...) { diff --git a/src/InputReader.h b/src/InputReader.h index 9d776276fa..3bfb0adf91 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -31,6 +31,7 @@ protected: virtual bool DoUpdate() = 0; // Reports an error to the user. + void Error(const string &msg); void Error(const char *msg); // The following methods return the information as passed to Init(). @@ -39,6 +40,8 @@ protected: // A thread-safe version of fmt(). (stolen from logwriter) const char* Fmt(const char* format, ...); + void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + private: friend class InputMgr; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 4bc4b81cda..5d6b23416a 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -37,7 +37,7 @@ bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const file = new ifstream(path.c_str()); if ( !file->is_open() ) { - Error(Fmt("cannot open %s", path.c_str())); + Error(Fmt("cannot open %s", fname.c_str())); return false; } @@ -108,7 +108,7 @@ bool InputReaderAscii::DoUpdate() { istringstream splitstream(line); string s; - LogVal fields[num_fields]; + LogVal** fields = new LogVal*[num_fields]; unsigned int currTab = 0; unsigned int currField = 0; @@ -136,11 +136,12 @@ bool InputReaderAscii::DoUpdate() { return false; } - LogVal val(currMapping.type, true); + LogVal* val = new LogVal(currMapping.type, true); switch ( currMapping.type ) { case TYPE_STRING: - val.val.string_val = new string(s); + val->val.string_val = new string(s); + break; default: Error(Fmt("unsupported field format %d for %s", currMapping.type, @@ -148,6 +149,8 @@ bool InputReaderAscii::DoUpdate() { return false; } + fields[currField] = val; + currField++; } @@ -157,6 +160,9 @@ bool InputReaderAscii::DoUpdate() { } // ok, now we have built our line. send it back to... whomever. + // for testing purposes: fixed event. + + SendEvent("inputEvent", num_fields, fields); } From b245d4168a6c9d9ce9243a34f7fcc704d526a7d9 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 26 Oct 2011 17:02:57 -0700 Subject: [PATCH 025/964] yay, basic table assignment. --- scripts/base/frameworks/input/main.bro | 4 +- src/InputMgr.cc | 60 ++++++++++++++++++++++++-- src/InputMgr.h | 8 +++- src/InputReader.cc | 5 +++ src/InputReader.h | 2 + src/InputReaderAscii.cc | 16 +++++++ 6 files changed, 87 insertions(+), 8 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 4bea9d73d2..b5a05af3b6 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -5,8 +5,8 @@ export { type ReaderDescription: record { source: string; idx: any; - val: any &optional; - destination: any &optional; + val: any; + destination: any; }; } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5cd82b613f..5c30922863 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -13,6 +13,16 @@ #include "InputReaderAscii.h" +struct InputMgr::ReaderInfo { + EnumVal* id; + EnumVal* type; + InputReader* reader; + unsigned int num_idx_fields; + unsigned int num_val_fields; + + TableVal* tab; + + }; struct InputReaderDefinition { bro_int_t type; // the type @@ -30,7 +40,7 @@ InputReaderDefinition input_readers[] = { InputMgr::InputMgr() { - DBG_LOG(DBG_LOGGING, "this has to happen"); + //DBG_LOG(DBG_LOGGING, "this has to happen"); } @@ -93,8 +103,10 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) string source((const char*) bsource->Bytes(), bsource->Len()); RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); + RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); - LogField** fields = new LogField*[idx->NumFields()]; + LogField** fields = new LogField*[idx->NumFields() + val->NumFields()]; for ( int i = 0; i < idx->NumFields(); i++ ) { // FIXME: do type checking... @@ -103,15 +115,43 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) field->type = idx->FieldType(i)->Tag(); fields[i] = field; } + for ( int i = 0; i < val->NumFields(); i++ ) + { + // FIXME: do type checking... + LogField* field = new LogField(); + field->name = val->FieldName(i); + field->type = val->FieldType(i)->Tag(); + fields[idx->NumFields() + i] = field; + } + + ReaderInfo* info = new ReaderInfo; + info->reader = reader_obj; + info->type = reader; + info->num_idx_fields = idx->NumFields(); + info->num_val_fields = val->NumFields(); + info->tab = dst; + readers.push_back(info); - reader_obj->Init(source, idx->NumFields(), fields); + reader_obj->Init(source, idx->NumFields() + val->NumFields(), fields); reader_obj->Update(); return reader_obj; } +void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return; + } + + i->tab->Assign(LogValToVal(vals[0]), LogValToVal(vals[1])); + reporter->Error("assigned"); +} + + void InputMgr::Error(InputReader* reader, const char* msg) { reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); @@ -172,4 +212,16 @@ Val* InputMgr::LogValToVal(const LogVal* val) { return NULL; } - +InputMgr::ReaderInfo* InputMgr::FindReader(const InputReader* reader) + { + for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) + { + if ( (*s)->reader == reader ) + { + return *s; + } + } + + return 0; + } + diff --git a/src/InputMgr.h b/src/InputMgr.h index 79f76a1e6f..136be2d608 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -24,14 +24,18 @@ protected: // Reports an error for the given reader. void Error(InputReader* reader, const char* msg); + + void Put(const InputReader* reader, const LogVal* const *vals); private: - // required functionality - // InputValsToRecord to convert received inputvals back to bro records / tables / whatever + struct ReaderInfo; + Val* LogValToVal(const LogVal* val); void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + ReaderInfo* FindReader(const InputReader* reader); + vector readers; }; extern InputMgr* input_mgr; diff --git a/src/InputReader.cc b/src/InputReader.cc index d512bc1699..6502fdb421 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -25,6 +25,11 @@ void InputReader::Error(const string &msg) input_mgr->Error(this, msg.c_str()); } +void InputReader::Put(const LogVal* const *val) +{ + input_mgr->Put(this, val); +} + bool InputReader::Init(string arg_source, int arg_num_fields, const LogField* const * arg_fields) { diff --git a/src/InputReader.h b/src/InputReader.h index 3bfb0adf91..3725c3d461 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -42,6 +42,8 @@ protected: void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + void Put(const LogVal* const *val); + private: friend class InputMgr; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 5d6b23416a..d0d4a3014c 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -143,6 +143,21 @@ bool InputReaderAscii::DoUpdate() { val->val.string_val = new string(s); break; + case TYPE_BOOL: + case TYPE_INT: + val->val.int_val = atoi(s.c_str()); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + val->val.double_val = atof(s.c_str()); + break; + + case TYPE_COUNT: + val->val.uint_val = atoi(s.c_str()); + break; + default: Error(Fmt("unsupported field format %d for %s", currMapping.type, currMapping.name.c_str())); @@ -163,6 +178,7 @@ bool InputReaderAscii::DoUpdate() { // for testing purposes: fixed event. SendEvent("inputEvent", num_fields, fields); + Put(fields); } From 86730c13dda7df06fa96b151c17ec437aa742b4f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 26 Oct 2011 17:46:43 -0700 Subject: [PATCH 026/964] more complex types... --- scripts/base/frameworks/input/main.bro | 3 + src/InputMgr.cc | 314 +++++++++++++++++++++++-- src/InputMgr.h | 17 +- src/InputReader.cc | 11 + src/InputReader.h | 2 + src/InputReaderAscii.cc | 35 ++- src/input.bif | 17 +- src/types.bif | 4 + 8 files changed, 372 insertions(+), 31 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index b5a05af3b6..4bb7129d03 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -2,11 +2,14 @@ module Input; export { + const default_reader = READER_ASCII &redef; + type ReaderDescription: record { source: string; idx: any; val: any; destination: any; + reader: Reader &default=default_reader; }; } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5c30922863..648a933a22 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -21,6 +21,8 @@ struct InputMgr::ReaderInfo { unsigned int num_val_fields; TableVal* tab; + RecordType* rtype; + RecordType* itype; }; @@ -45,7 +47,7 @@ InputMgr::InputMgr() // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) +InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) { InputReaderDefinition* ir = input_readers; @@ -55,6 +57,8 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) reporter->Error("readerDescription argument not of right type"); return 0; } + + EnumVal* reader = description->Lookup(rtype->FieldOffset("reader"))->AsEnumVal(); while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) @@ -106,38 +110,191 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); - LogField** fields = new LogField*[idx->NumFields() + val->NumFields()]; - for ( int i = 0; i < idx->NumFields(); i++ ) - { - // FIXME: do type checking... - LogField* field = new LogField(); - field->name = idx->FieldName(i); - field->type = idx->FieldType(i)->Tag(); - fields[i] = field; + + vector fieldsV; // vector, because we don't know the length beforehands + + + bool status = !UnrollRecordType(&fieldsV, idx, ""); + + int idxfields = fieldsV.size(); + + status = status || !UnrollRecordType(&fieldsV, val, ""); + int valfields = fieldsV.size() - idxfields; + + if ( status ) { + reporter->Error("Problem unrolling"); + return 0; } - for ( int i = 0; i < val->NumFields(); i++ ) - { - // FIXME: do type checking... - LogField* field = new LogField(); - field->name = val->FieldName(i); - field->type = val->FieldType(i)->Tag(); - fields[idx->NumFields() + i] = field; + + + LogField** fields = new LogField*[fieldsV.size()]; + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { + fields[i] = fieldsV[i]; } ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; info->type = reader; - info->num_idx_fields = idx->NumFields(); - info->num_val_fields = val->NumFields(); + Ref(reader); + info->num_idx_fields = idxfields; + info->num_val_fields = valfields; info->tab = dst; + Ref(dst); + info->rtype = val; + Ref(val); // we save a pointer of it... I really hope that this wasn't already done anywhere. + info->id = id; + Ref(id); // ditto... + info->itype = idx; + Ref(idx); readers.push_back(info); - reader_obj->Init(source, idx->NumFields() + val->NumFields(), fields); + reader_obj->Init(source, fieldsV.size(), fields); reader_obj->Update(); return reader_obj; +} +bool InputMgr::IsCompatibleType(BroType* t) + { + if ( ! t ) + return false; + + switch ( t->Tag() ) { + case TYPE_BOOL: + case TYPE_INT: + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + case TYPE_SUBNET: + case TYPE_ADDR: + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_RECORD: + // for record: check, if all elements are compatible? But... LogMgr also doesn't do this. + // ^ recursive checking is done in UnrollRecordType. + return true; + + case TYPE_FILE: + case TYPE_FUNC: + return false; + + + case TYPE_TABLE: + return false; + + case TYPE_VECTOR: + { + return IsCompatibleType(t->AsVectorType()->YieldType()); + } + + default: + return false; + } + + return false; + } + + +bool InputMgr::RemoveReader(EnumVal* id) { + ReaderInfo *i = 0; + for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) + { + if ( (*s)->id == id ) + { + i = (*s); + readers.erase(s); // remove from vector + break; + } + } + + if ( i == 0 ) { + return false; // not found + } + + Unref(i->type); + Unref(i->tab); + Unref(i->itype); + Unref(i->rtype); + Unref(i->id); + + delete(i->reader); + delete(i); + + return true; +} + +bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { + for ( int i = 0; i < rec->NumFields(); i++ ) + { + + if ( !IsCompatibleType(rec->FieldType(i)) ) { + reporter->Error("Incompatible type \"%s\" in table definition for InputReader", type_name(rec->FieldType(i)->Tag())); + return false; + } + + if ( rec->FieldType(i)->Tag() == TYPE_RECORD ) + { + + string prep = nameprepend + rec->FieldName(i) + "."; + + if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep) ) + { + return false; + } + + } else { + LogField* field = new LogField(); + field->name = nameprepend + rec->FieldName(i); + field->type = rec->FieldType(i)->Tag(); + + fields->push_back(field); + } + } + + return true; + +} + +bool InputMgr::ForceUpdate(EnumVal* id) +{ + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->Error("Reader not found"); + return false; + } + + i->reader->Update(); + return true; +} + +Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const LogVal* const *vals) { + Val* idxval; + int position = 0; + + + if ( num_fields == 1 ) { + idxval = LogValToVal(vals[0]); + } else { + ListVal *l = new ListVal(TYPE_ANY); + for ( int j = 0 ; j < type->NumFields(); j++ ) { + if ( type->FieldType(j)->Tag() == TYPE_RECORD ) { + l->Append(LogValToRecordVal(vals, type->FieldType(j)->AsRecordType(), &position)); + } else { + l->Append(LogValToVal(vals[position], type->FieldType(j)->Tag())); + position++; + } + } + idxval = l; + } + + assert ( position == num_fields ); + + return idxval; + } void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { @@ -147,10 +304,65 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { return; } - i->tab->Assign(LogValToVal(vals[0]), LogValToVal(vals[1])); - reporter->Error("assigned"); + Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + Val* valval; + + int position = i->num_idx_fields; + if ( i->num_val_fields == 1 ) { + valval = LogValToVal(vals[i->num_idx_fields]); + } else { + RecordVal * r = new RecordVal(i->rtype); + + /* if ( i->rtype->NumFields() != (int) i->num_val_fields ) { + reporter->InternalError("Type mismatch"); + return; + } */ + + for ( int j = 0; j < i->rtype->NumFields(); j++) { + + Val* val = 0; + if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); + } else { + val = LogValToVal(vals[position], i->rtype->FieldType(j)->Tag()); + position++; + } + + if ( val == 0 ) { + reporter->InternalError("conversion error"); + return; + } + + r->Assign(j,val); + + } + valval = r; + } + + i->tab->Assign(idxval, valval); } +void InputMgr::Clear(const InputReader* reader) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return; + } + + i->tab->RemoveAll(); +} + +bool InputMgr::Delete(const InputReader* reader, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return false; + } + + Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + + return ( i->tab->Delete(idxval) != 0 ); +} void InputMgr::Error(InputReader* reader, const char* msg) { @@ -174,7 +386,46 @@ void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c mgr.Dispatch(new Event(handler, vl)); } -Val* InputMgr::LogValToVal(const LogVal* val) { + +Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { + if ( position == 0 ) { + reporter->InternalError("Need position"); + return 0; + } + + /* + if ( request_type->Tag() != TYPE_RECORD ) { + reporter->InternalError("I only work with records"); + return 0; + } */ + + + RecordVal* rec = new RecordVal(request_type->AsRecordType()); + for ( int i = 0; i < request_type->NumFields(); i++ ) { + + Val* fieldVal = 0; + if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) { + fieldVal = LogValToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); + } else { + fieldVal = LogValToVal(vals[*position], request_type->FieldType(i)->Tag()); + (*position)++; + } + + rec->Assign(i, fieldVal); + } + + return rec; + +} + +Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { + + if ( request_type != TYPE_ANY && request_type != val->type ) { + reporter->InternalError("Typetags don't match: %d vs %d", request_type, val->type); + return 0; + } + + switch ( val->type ) { case TYPE_BOOL: case TYPE_INT: @@ -203,6 +454,10 @@ Val* InputMgr::LogValToVal(const LogVal* val) { return new PortVal(val->val.uint_val); break; + case TYPE_ADDR: + return new AddrVal(val->val.addr_val); + break; + default: reporter->InternalError("unsupported type for input_read"); } @@ -225,3 +480,18 @@ InputMgr::ReaderInfo* InputMgr::FindReader(const InputReader* reader) return 0; } + +InputMgr::ReaderInfo* InputMgr::FindReader(const EnumVal* id) + { + for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) + { + if ( (*s)->id == id ) + { + return *s; + } + } + + return 0; + } + + diff --git a/src/InputMgr.h b/src/InputMgr.h index 136be2d608..d5f732935c 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -11,13 +11,17 @@ #include "RemoteSerializer.h" #include "LogMgr.h" // for the LogVal and LogType data types +#include + class InputReader; class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* reader, RecordVal* description); + InputReader* CreateReader(EnumVal* id, RecordVal* description); + bool ForceUpdate(EnumVal* id); + bool RemoveReader(EnumVal* id); protected: friend class InputReader; @@ -26,14 +30,23 @@ protected: void Error(InputReader* reader, const char* msg); void Put(const InputReader* reader, const LogVal* const *vals); + void Clear(const InputReader* reader); + bool Delete(const InputReader* reader, const LogVal* const *vals); private: struct ReaderInfo; - Val* LogValToVal(const LogVal* val); + bool IsCompatibleType(BroType* t); + + bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); + + Val* LogValToVal(const LogVal* val, TypeTag request_type = TYPE_ANY); + Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); + Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); ReaderInfo* FindReader(const InputReader* reader); + ReaderInfo* FindReader(const EnumVal* id); vector readers; }; diff --git a/src/InputReader.cc b/src/InputReader.cc index 6502fdb421..1facc57c7f 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -30,6 +30,17 @@ void InputReader::Put(const LogVal* const *val) input_mgr->Put(this, val); } +void InputReader::Clear() +{ + input_mgr->Clear(this); +} + +void InputReader::Delete(const LogVal* const *val) +{ + input_mgr->Delete(this, val); +} + + bool InputReader::Init(string arg_source, int arg_num_fields, const LogField* const * arg_fields) { diff --git a/src/InputReader.h b/src/InputReader.h index 3725c3d461..0e93344e1a 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -43,6 +43,8 @@ protected: void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); void Put(const LogVal* const *val); + void Clear(); + void Delete(const LogVal* const *val); private: friend class InputMgr; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index d0d4a3014c..e4d581c0d8 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -144,6 +144,13 @@ bool InputReaderAscii::DoUpdate() { break; case TYPE_BOOL: + if ( s == "T" ) { + val->val.int_val = 1; + } else { + val->val.int_val = 0; + } + break; + case TYPE_INT: val->val.int_val = atoi(s.c_str()); break; @@ -155,16 +162,37 @@ bool InputReaderAscii::DoUpdate() { break; case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: val->val.uint_val = atoi(s.c_str()); break; + case TYPE_SUBNET: { + int pos = s.find("/"); + string width = s.substr(pos); + val->val.subnet_val.width = atoi(width.c_str()); + string addr = s.substr(0, pos); + s = addr; + // fallthrough + } + case TYPE_ADDR: { + addr_type t = dotted_to_addr(s.c_str()); +#ifdef BROv6 + copy_addr(t, val->val.addr_val); +#else + copy_addr(&t, val->val.addr_val); +#endif + break; + } + + default: Error(Fmt("unsupported field format %d for %s", currMapping.type, currMapping.name.c_str())); return false; } - fields[currField] = val; + fields[currMapping.position] = val; currField++; } @@ -174,10 +202,7 @@ bool InputReaderAscii::DoUpdate() { return false; } - // ok, now we have built our line. send it back to... whomever. - // for testing purposes: fixed event. - - SendEvent("inputEvent", num_fields, fields); + // ok, now we have built our line. send it back to the input manager Put(fields); } diff --git a/src/input.bif b/src/input.bif index 3da869ea08..4e2cbf07b5 100644 --- a/src/input.bif +++ b/src/input.bif @@ -9,9 +9,22 @@ module Input; type ReaderDescription: record; -function Input::__create_reader%(reader: Input::Reader, description: Input::ReaderDescription%) : bool +function Input::__create_reader%(id: ID, description: Input::ReaderDescription%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), description->AsRecordVal()); + InputReader *the_reader = input_mgr->CreateReader(id->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} + +function Input::__force_update%(id: ID%) : bool + %{ + bool res = input_mgr->ForceUpdate(id->AsEnumVal()); + return new Val( res, TYPE_BOOL ); + %} + +function Input::__remove_reader%(id: ID%) : bool + %{ + bool res = input_mgr->RemoveReader(id->AsEnumVal()); + return new Val( res, TYPE_BOOL ); + %} + diff --git a/src/types.bif b/src/types.bif index ee43207ddd..7b81a7f631 100644 --- a/src/types.bif +++ b/src/types.bif @@ -174,4 +174,8 @@ enum Reader %{ READER_ASCII, %} +enum ID %{ + Unknown, +%} + module GLOBAL; From f20125d22de20a7147026602f83fa1ed710700ac Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 28 Oct 2011 13:11:47 -0700 Subject: [PATCH 027/964] little snag with hashing functionality... --- src/InputMgr.cc | 232 +++++++++++++++++++++++++++++++++++++++- src/InputMgr.h | 11 +- src/InputReader.cc | 13 ++- src/InputReader.h | 9 +- src/InputReaderAscii.cc | 80 ++++++++++++-- src/InputReaderAscii.h | 5 +- src/Val.h | 5 +- 7 files changed, 333 insertions(+), 22 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 648a933a22..5270c83d5a 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -13,6 +13,18 @@ #include "InputReaderAscii.h" +#include "CompHash.h" + + +class InputHash { +public: + HashKey* valhash; + HashKey* idxkey; // does not need ref or whatever - if it is present here, it is also still present in the TableVal. +}; + +declare(PDict, InputHash); + + struct InputMgr::ReaderInfo { EnumVal* id; EnumVal* type; @@ -24,6 +36,9 @@ struct InputMgr::ReaderInfo { RecordType* rtype; RecordType* itype; + PDict(InputHash)* currDict; + PDict(InputHash)* lastDict; + }; struct InputReaderDefinition { @@ -147,10 +162,20 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->itype = idx; Ref(idx); readers.push_back(info); + info->currDict = new PDict(InputHash); + info->lastDict = new PDict(InputHash); - reader_obj->Init(source, fieldsV.size(), fields); - reader_obj->Update(); + int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); + if ( success == false ) { + RemoveReader(id); + return 0; + } + success = reader_obj->Update(); + if ( success == false ) { + RemoveReader(id); + return 0; + } return reader_obj; @@ -198,7 +223,6 @@ bool InputMgr::IsCompatibleType(BroType* t) return false; } - bool InputMgr::RemoveReader(EnumVal* id) { ReaderInfo *i = 0; for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) @@ -215,6 +239,9 @@ bool InputMgr::RemoveReader(EnumVal* id) { return false; // not found } + i->reader->Finish(); + + Unref(i->type); Unref(i->tab); Unref(i->itype); @@ -267,8 +294,7 @@ bool InputMgr::ForceUpdate(EnumVal* id) return false; } - i->reader->Update(); - return true; + return i->reader->Update(); } Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const LogVal* const *vals) { @@ -297,6 +323,105 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo } + +void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return; + } + + HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); + HashKey* valhash = HashLogVals(i->num_val_fields, vals+i->num_idx_fields); + + InputHash *h = i->lastDict->Lookup(idxhash); + if ( h != 0 ) { + // seen before + if ( h->valhash->Hash() == valhash->Hash() ) { + // ok, double. + i->lastDict->Remove(idxhash); + i->currDict->Insert(idxhash, h); + return; + } else { + // updated + i->lastDict->Remove(idxhash); + delete(h); + } + } + + + Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + Val* valval; + + int position = i->num_idx_fields; + if ( i->num_val_fields == 1 ) { + valval = LogValToVal(vals[i->num_idx_fields]); + } else { + RecordVal * r = new RecordVal(i->rtype); + + /* if ( i->rtype->NumFields() != (int) i->num_val_fields ) { + reporter->InternalError("Type mismatch"); + return; + } */ + + for ( int j = 0; j < i->rtype->NumFields(); j++) { + + Val* val = 0; + if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); + } else { + val = LogValToVal(vals[position], i->rtype->FieldType(j)->Tag()); + position++; + } + + if ( val == 0 ) { + reporter->InternalError("conversion error"); + return; + } + + r->Assign(j,val); + + } + valval = r; + } + + //i->tab->Assign(idxval, valval); + HashKey* k = i->tab->ComputeHash(idxval); + if ( !k ) { + reporter->InternalError("could not hash"); + return; + } + + i->tab->Assign(idxval, k, valval); + InputHash* ih = new InputHash(); + ih->idxkey = k; + ih->valhash = valhash; + + i->currDict->Insert(idxhash, ih); + +} + +void InputMgr::EndCurrentSend(const InputReader* reader) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return; + } + + // lastdict contains all deleted entries + IterCookie *c = i->lastDict->InitForIteration(); + InputHash* ih; + while ( ( ih = i->lastDict->NextEntry(c )) ) { + i->tab->Delete(ih->idxkey); + } + + i->lastDict->Clear(); + delete(i->lastDict); + + i->lastDict = i->currDict; + i->currDict = new PDict(InputHash); +} + void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { @@ -418,6 +543,95 @@ Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_ } +HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals) { + int length = 0; + + for ( int i = 0; i < num_elements; i++ ) { + const LogVal* val = vals[i]; + switch (val->type) { + case TYPE_BOOL: + case TYPE_INT: + length += sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + length += sizeof(val->val.uint_val); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + length += sizeof(val->val.double_val); + break; + + case TYPE_STRING: + { + length += val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + length += NUM_ADDR_WORDS*sizeof(uint32_t); + break; + + default: + reporter->InternalError("unsupported type for hashlogvals"); + } + + } + + int position = 0; + char *data = (char*) malloc(length); + for ( int i = 0; i < num_elements; i++ ) { + const LogVal* val = vals[i]; + switch ( val->type ) { + case TYPE_BOOL: + case TYPE_INT: + *(data+position) = val->val.int_val; + position += sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + *(data+position) = val->val.uint_val; + position += sizeof(val->val.uint_val); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + *(data+position) = val->val.double_val; + position += sizeof(val->val.double_val); + break; + + case TYPE_STRING: + { + memcpy(data+position, val->val.string_val->c_str(), val->val.string_val->length()); + position += val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + memcpy(data+position, val->val.addr_val, NUM_ADDR_WORDS*sizeof(uint32_t)); + position += NUM_ADDR_WORDS*sizeof(uint32_t); + break; + + default: + reporter->InternalError("unsupported type for hashlogvals2"); + } + + + } + + assert(position == length); + return new HashKey(data, length); + + +} + Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { if ( request_type != TYPE_ANY && request_type != val->type ) { @@ -495,3 +709,11 @@ InputMgr::ReaderInfo* InputMgr::FindReader(const EnumVal* id) } +string InputMgr::Hash(const string &input) { + unsigned char digest[16]; + hash_md5(input.length(), (const unsigned char*) input.c_str(), digest); + string out((const char*) digest, 16); + return out; +} + + diff --git a/src/InputMgr.h b/src/InputMgr.h index d5f732935c..d147fa262a 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -15,9 +15,10 @@ class InputReader; + class InputMgr { public: - InputMgr(); + InputMgr(); InputReader* CreateReader(EnumVal* id, RecordVal* description); bool ForceUpdate(EnumVal* id); @@ -32,6 +33,9 @@ protected: void Put(const InputReader* reader, const LogVal* const *vals); void Clear(const InputReader* reader); bool Delete(const InputReader* reader, const LogVal* const *vals); + + void SendEntry(const InputReader* reader, const LogVal* const *vals); + void EndCurrentSend(const InputReader* reader); private: struct ReaderInfo; @@ -40,6 +44,8 @@ private: bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); + HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); + Val* LogValToVal(const LogVal* val, TypeTag request_type = TYPE_ANY); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); @@ -49,6 +55,9 @@ private: ReaderInfo* FindReader(const EnumVal* id); vector readers; + + string Hash(const string &input); + }; extern InputMgr* input_mgr; diff --git a/src/InputReader.cc b/src/InputReader.cc index 1facc57c7f..494df3fb81 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -41,20 +41,22 @@ void InputReader::Delete(const LogVal* const *val) } -bool InputReader::Init(string arg_source, int arg_num_fields, +bool InputReader::Init(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const * arg_fields) { source = arg_source; num_fields = arg_num_fields; + index_fields = arg_idx_fields; fields = arg_fields; // disable if DoInit returns error. - disabled = !DoInit(arg_source, arg_num_fields, arg_fields); + disabled = !DoInit(arg_source, arg_num_fields, arg_idx_fields, arg_fields); return !disabled; } void InputReader::Finish() { DoFinish(); + disabled = true; } bool InputReader::Update() { @@ -91,3 +93,10 @@ const char* InputReader::Fmt(const char* format, ...) } +void InputReader::SendEntry(const LogVal* const *vals) { + input_mgr->SendEntry(this, vals); +} + +void InputReader::EndCurrentSend() { + input_mgr->EndCurrentSend(this); +} diff --git a/src/InputReader.h b/src/InputReader.h index 0e93344e1a..b547d29506 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -15,7 +15,7 @@ public: InputReader(); virtual ~InputReader(); - bool Init(string arg_source, int num_fields, const LogField* const* fields); + bool Init(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const* fields); void Finish(); @@ -23,7 +23,7 @@ public: protected: // Methods that have to be overwritten by the individual readers - virtual bool DoInit(string arg_source, int num_fields, const LogField* const * fields) = 0; + virtual bool DoInit(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const * fields) = 0; virtual void DoFinish() = 0; @@ -46,11 +46,16 @@ protected: void Clear(); void Delete(const LogVal* const *val); + void SendEntry(const LogVal* const *vals); + void EndCurrentSend(); + + private: friend class InputMgr; string source; int num_fields; + int index_fields; const LogField* const * fields; // When an error occurs, this method is called to set a flag marking the diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index e4d581c0d8..e434f7e750 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -21,17 +21,26 @@ InputReaderAscii::InputReaderAscii() { //DBG_LOG(DBG_LOGGING, "input reader initialized"); file = 0; + + //keyMap = new map(); } InputReaderAscii::~InputReaderAscii() { + DoFinish(); } void InputReaderAscii::DoFinish() { + columnMap.empty(); + if ( file != 0 ) { + file->close(); + delete(file); + file = 0; + } } -bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const * fields) +bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const LogField* const * fields) { fname = path; @@ -47,6 +56,9 @@ bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const Error("could not read first line"); return false; } + + this->num_fields = num_fields; + this->idx_fields = idx_fields; // split on tabs... istringstream splitstream(line); @@ -83,12 +95,10 @@ bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const if ( wantFields != num_fields ) { // we did not find all fields? // :( - Error("wantFields != num_fields"); + Error("One of the requested fields could not be found in the input data file"); return false; } - - this->num_fields = num_fields; // well, that seems to have worked... return true; @@ -101,6 +111,9 @@ bool InputReaderAscii::DoUpdate() { // + // new keymap + //map *newKeyMap = new map(); + string line; while ( getline(*file, line ) ) { // split on tabs @@ -109,10 +122,12 @@ bool InputReaderAscii::DoUpdate() { string s; LogVal** fields = new LogVal*[num_fields]; + //string string_fields[num_fields]; unsigned int currTab = 0; unsigned int currField = 0; while ( splitstream ) { + if ( !getline(splitstream, s, '\t') ) break; @@ -146,8 +161,11 @@ bool InputReaderAscii::DoUpdate() { case TYPE_BOOL: if ( s == "T" ) { val->val.int_val = 1; - } else { + } else if ( s == "F" ) { val->val.int_val = 0; + } else { + Error(Fmt("Invalid value for boolean: %s", s.c_str())); + return false; } break; @@ -173,9 +191,15 @@ bool InputReaderAscii::DoUpdate() { val->val.subnet_val.width = atoi(width.c_str()); string addr = s.substr(0, pos); s = addr; - // fallthrough + // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // Solve this some other time.... + val->val.subnet_val.net = dotted_to_addr(s.c_str()); + break; + } case TYPE_ADDR: { + // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // Solve this some other time.... addr_type t = dotted_to_addr(s.c_str()); #ifdef BROv6 copy_addr(t, val->val.addr_val); @@ -193,19 +217,57 @@ bool InputReaderAscii::DoUpdate() { } fields[currMapping.position] = val; + //string_fields[currMapping.position] = s; currField++; } if ( currField != num_fields ) { - Error("curr_field != num_fields in DoUpdate"); + Error("curr_field != num_fields in DoUpdate. Columns in file do not match column definition."); return false; } - // ok, now we have built our line. send it back to the input manager - Put(fields); + + SendEntry(fields); + + /* + string indexstring = ""; + string valstring = ""; + for ( unsigned int i = 0; i < idx_fields; i++ ) { + indexstring.append(string_fields[i]); + } + + for ( unsigned int i = idx_fields; i < num_fields; i++ ) { + valstring.append(string_fields[i]); + } + + string valhash = Hash(valstring); + string indexhash = Hash(indexstring); + + if ( keyMap->find(indexhash) == keyMap->end() ) { + // new key + Put(fields); + } else if ( (*keyMap)[indexhash] != valhash ) { + // changed key + Put(fields); + keyMap->erase(indexhash); + } else { + // field not changed + keyMap->erase(indexhash); + } + + + (*newKeyMap)[indexhash] = valhash; + */ + + for ( unsigned int i = 0; i < num_fields; i++ ) { + delete fields[i]; + } + delete [] fields; } + + EndCurrentSend(); return true; } diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 551a08b02e..c26a139dcd 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -30,7 +30,7 @@ public: protected: - virtual bool DoInit(string path, int num_fields, + virtual bool DoInit(string path, int arg_num_fields, int arg_idx_fields, const LogField* const * fields); virtual void DoFinish(); @@ -42,9 +42,12 @@ private: string fname; unsigned int num_fields; + unsigned int idx_fields; // map columns in the file to columns to send back to the manager vector columnMap; + + //map *keyMap; }; diff --git a/src/Val.h b/src/Val.h index d851be311b..3ae0bc3334 100644 --- a/src/Val.h +++ b/src/Val.h @@ -841,6 +841,9 @@ public: timer = 0; } + HashKey* ComputeHash(const Val* index) const + { return table_hash->ComputeHash(index, 1); } + protected: friend class Val; friend class StateAccess; @@ -851,8 +854,6 @@ protected: void CheckExpireAttr(attr_tag at); int ExpandCompoundAndInit(val_list* vl, int k, Val* new_val); int CheckAndAssign(Val* index, Val* new_val, Opcode op = OP_ASSIGN); - HashKey* ComputeHash(const Val* index) const - { return table_hash->ComputeHash(index, 1); } bool AddProperties(Properties arg_state); bool RemoveProperties(Properties arg_state); From 638976791efb3b7d9d491266d83bb30b73d4bd53 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 2 Nov 2011 14:00:19 -0700 Subject: [PATCH 028/964] hashing seems to work _correctly_ now... --- src/InputMgr.cc | 44 ++++++++++++++++++++++++++++++----------- src/InputReaderAscii.cc | 41 +++++++++++++++++++++++++++++++++----- src/InputReaderAscii.h | 3 +++ 3 files changed, 72 insertions(+), 16 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5270c83d5a..e3acfb9505 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -290,10 +290,10 @@ bool InputMgr::ForceUpdate(EnumVal* id) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { - reporter->Error("Reader not found"); + reporter->InternalError("Reader not found"); return false; } - + return i->reader->Update(); } @@ -302,8 +302,9 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo int position = 0; - if ( num_fields == 1 ) { + if ( num_fields == 1 && type->FieldType(0)->Tag() != TYPE_RECORD ) { idxval = LogValToVal(vals[0]); + position = 1; } else { ListVal *l = new ListVal(TYPE_ANY); for ( int j = 0 ; j < type->NumFields(); j++ ) { @@ -317,6 +318,7 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo idxval = l; } + //reporter->Error("Position: %d, num_fields: %d", position, num_fields); assert ( position == num_fields ); return idxval; @@ -331,8 +333,15 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { return; } + + reporter->Error("Hashing %d index fields", i->num_idx_fields); HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); + reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); + reporter->Error("Hashing %d val fields", i->num_val_fields); HashKey* valhash = HashLogVals(i->num_val_fields, vals+i->num_idx_fields); + reporter->Error("Result: %d", (uint64_t) valhash->Hash()); + + //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); InputHash *h = i->lastDict->Lookup(idxhash); if ( h != 0 ) { @@ -393,9 +402,12 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { } i->tab->Assign(idxval, k, valval); + InputHash* ih = new InputHash(); + k = i->tab->ComputeHash(idxval); ih->idxkey = k; ih->valhash = valhash; + //i->tab->Delete(k); i->currDict->Insert(idxhash, ih); @@ -407,11 +419,12 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { reporter->InternalError("Unknown reader"); return; } - - // lastdict contains all deleted entries + // lastdict contains all deleted entries and should be empty apart from that IterCookie *c = i->lastDict->InitForIteration(); InputHash* ih; - while ( ( ih = i->lastDict->NextEntry(c )) ) { + reporter->Error("ending"); + while ( ( ih = i->lastDict->NextEntry(c) ) ) { + reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); } @@ -582,28 +595,37 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals } + //reporter->Error("Length: %d", length); + int position = 0; char *data = (char*) malloc(length); + if ( data == 0 ) { + reporter->InternalError("Could not malloc?"); + } for ( int i = 0; i < num_elements; i++ ) { const LogVal* val = vals[i]; switch ( val->type ) { case TYPE_BOOL: case TYPE_INT: - *(data+position) = val->val.int_val; + //reporter->Error("Adding field content to pos %d: %lld", val->val.int_val, position); + memcpy(data+position, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); + //*(data+position) = val->val.int_val; position += sizeof(val->val.int_val); break; case TYPE_COUNT: case TYPE_COUNTER: case TYPE_PORT: - *(data+position) = val->val.uint_val; + //*(data+position) = val->val.uint_val; + memcpy(data+position, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); position += sizeof(val->val.uint_val); break; case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: - *(data+position) = val->val.double_val; + //*(data+position) = val->val.double_val; + memcpy(data+position, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); position += sizeof(val->val.double_val); break; @@ -685,7 +707,7 @@ InputMgr::ReaderInfo* InputMgr::FindReader(const InputReader* reader) { for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { - if ( (*s)->reader == reader ) + if ( (*s)->reader && (*s)->reader == reader ) { return *s; } @@ -699,7 +721,7 @@ InputMgr::ReaderInfo* InputMgr::FindReader(const EnumVal* id) { for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { - if ( (*s)->id == id ) + if ( (*s)->id && (*s)->id->AsEnum() == id->AsEnum() ) { return *s; } diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index e434f7e750..4755b2b74f 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -50,6 +50,16 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const return false; } + + this->num_fields = num_fields; + this->idx_fields = idx_fields; + this->fields = fields; + + return true; +} + + +bool InputReaderAscii::ReadHeader() { // try to read the header line... string line; if ( !getline(*file, line) ) { @@ -57,9 +67,6 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const return false; } - this->num_fields = num_fields; - this->idx_fields = idx_fields; - // split on tabs... istringstream splitstream(line); unsigned int currTab = 0; @@ -70,7 +77,7 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const break; // current found heading in s... compare if we want it - for ( int i = 0; i < num_fields; i++ ) { + for ( unsigned int i = 0; i < num_fields; i++ ) { const LogField* field = fields[i]; if ( field->name == s ) { // cool, found field. note position @@ -92,7 +99,7 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const currTab++; } - if ( wantFields != num_fields ) { + if ( wantFields != (int) num_fields ) { // we did not find all fields? // :( Error("One of the requested fields could not be found in the input data file"); @@ -106,9 +113,30 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const // read the entire file and send appropriate thingies back to InputMgr bool InputReaderAscii::DoUpdate() { + + + // dirty, fix me. (well, apparently after trying seeking, etc - this is not that bad) + if ( file && file->is_open() ) { + file->close(); + } + file = new ifstream(fname.c_str()); + if ( !file->is_open() ) { + Error(Fmt("cannot open %s", fname.c_str())); + return false; + } + // + + // file->seekg(0, ios::beg); // do not forget clear. + + + if ( ReadHeader() == false ) { + return false; + } + // TODO: all the stuff we need for a second reading. // *cough* // + // // new keymap @@ -152,6 +180,7 @@ bool InputReaderAscii::DoUpdate() { } LogVal* val = new LogVal(currMapping.type, true); + //bzero(val, sizeof(LogVal)); switch ( currMapping.type ) { case TYPE_STRING: @@ -267,6 +296,8 @@ bool InputReaderAscii::DoUpdate() { } + //file->clear(); // remove end of file evil bits + //file->seekg(0, ios::beg); // and seek to start. EndCurrentSend(); return true; diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index c26a139dcd..673a2cdae2 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -37,6 +37,8 @@ protected: virtual bool DoUpdate(); private: + + bool ReadHeader(); ifstream* file; string fname; @@ -46,6 +48,7 @@ private: // map columns in the file to columns to send back to the manager vector columnMap; + const LogField* const * fields; // raw mapping //map *keyMap; From b5a77aa77baf69b788adb7b40f4246f0692939a0 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 2 Nov 2011 14:29:58 -0700 Subject: [PATCH 029/964] reading seems to work with all atomic types + records... --- src/InputMgr.cc | 25 ++++++++++++++++++++----- src/InputReaderAscii.cc | 2 +- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index e3acfb9505..bbfd354189 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -334,12 +334,12 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { } - reporter->Error("Hashing %d index fields", i->num_idx_fields); + //reporter->Error("Hashing %d index fields", i->num_idx_fields); HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); - reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); - reporter->Error("Hashing %d val fields", i->num_val_fields); + //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); + //reporter->Error("Hashing %d val fields", i->num_val_fields); HashKey* valhash = HashLogVals(i->num_val_fields, vals+i->num_idx_fields); - reporter->Error("Result: %d", (uint64_t) valhash->Hash()); + //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); @@ -422,7 +422,6 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { // lastdict contains all deleted entries and should be empty apart from that IterCookie *c = i->lastDict->InitForIteration(); InputHash* ih; - reporter->Error("ending"); while ( ( ih = i->lastDict->NextEntry(c) ) ) { reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); @@ -589,6 +588,11 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals length += NUM_ADDR_WORDS*sizeof(uint32_t); break; + case TYPE_SUBNET: + length += sizeof(val->val.subnet_val.width); + length += sizeof(val->val.subnet_val.net); + break; + default: reporter->InternalError("unsupported type for hashlogvals"); } @@ -641,6 +645,13 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals position += NUM_ADDR_WORDS*sizeof(uint32_t); break; + case TYPE_SUBNET: + memcpy(data+position,(const char*) &(val->val.subnet_val.width), sizeof(val->val.subnet_val.width) ); + position += sizeof(val->val.subnet_val.width); + memcpy(data+position, (const char*) &(val->val.subnet_val.net), sizeof(val->val.subnet_val.net) ); + position += sizeof(val->val.subnet_val.net); + break; + default: reporter->InternalError("unsupported type for hashlogvals2"); } @@ -694,6 +705,10 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { return new AddrVal(val->val.addr_val); break; + case TYPE_SUBNET: + return new SubNetVal(val->val.subnet_val.net, val->val.subnet_val.width); + break; + default: reporter->InternalError("unsupported type for input_read"); } diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 4755b2b74f..39635de407 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -216,7 +216,7 @@ bool InputReaderAscii::DoUpdate() { case TYPE_SUBNET: { int pos = s.find("/"); - string width = s.substr(pos); + string width = s.substr(pos+1); val->val.subnet_val.width = atoi(width.c_str()); string addr = s.substr(0, pos); s = addr; From c4d6f814ffc4d984308c2600b5b7e60ee6b38f14 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 2 Nov 2011 18:09:09 -0700 Subject: [PATCH 030/964] Tuning the pretty-printed alarms output. - Now including the included time range into the subject. - With some notices, it got confused who's the orginator. --- .../frameworks/notice/actions/pp-alarms.bro | 54 ++++++++++--------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/scripts/base/frameworks/notice/actions/pp-alarms.bro b/scripts/base/frameworks/notice/actions/pp-alarms.bro index 1284d7885f..c8d8259cdc 100644 --- a/scripts/base/frameworks/notice/actions/pp-alarms.bro +++ b/scripts/base/frameworks/notice/actions/pp-alarms.bro @@ -44,34 +44,40 @@ function pp_open() pp_alarms_open = T; pp_alarms = open(pp_alarms_name); - - local dest = mail_dest_pretty_printed != "" ? mail_dest_pretty_printed - : mail_dest; - - local headers = email_headers("Alarm summary", dest); - write_file(pp_alarms, headers + "\n"); } # Closes and mails out the current output file. -function pp_send() +function pp_send(rinfo: Log::RotationInfo) { if ( ! pp_alarms_open ) return; write_file(pp_alarms, "\n\n--\n[Automatically generated]\n\n"); close(pp_alarms); - - system(fmt("/bin/cat %s | %s -t -oi && /bin/rm %s", - pp_alarms_name, sendmail, pp_alarms_name)); - pp_alarms_open = F; + + local from = strftime("%H:%M:%S", rinfo$open); + local to = strftime("%H:%M:%S", rinfo$close); + local subject = fmt("Alarm summary from %s-%s", from, to); + local dest = mail_dest_pretty_printed != "" ? mail_dest_pretty_printed + : mail_dest; + + local headers = email_headers(subject, dest); + + local header_name = pp_alarms_name + ".tmp"; + local header = open(header_name); + write_file(header, headers + "\n"); + close(header); + + system(fmt("/bin/cat %s %s | %s -t -oi && /bin/rm -f %s %s", + header_name, pp_alarms_name, sendmail, header_name, pp_alarms_name)); } # Postprocessor function that triggers the email. function pp_postprocessor(info: Log::RotationInfo): bool { if ( want_pp() ) - pp_send(); + pp_send(info); return T; } @@ -93,7 +99,7 @@ event notice(n: Notice::Info) &priority=-5 if ( ! want_pp() ) return; - if ( ACTION_LOG !in n$actions ) + if ( ACTION_ALARM !in n$actions ) return; if ( ! pp_alarms_open ) @@ -154,31 +160,27 @@ function pretty_print_alarm(out: file, n: Info) if ( n?$id ) { - orig_p = fmt(":%s", n$id$orig_p); - resp_p = fmt(":%s", n$id$resp_p); + h1 = n$id$orig_h; + h2 = n$id$resp_h; + who = fmt("%s:%s -> %s:%s", h1, n$id$orig_p, h2, n$id$resp_p); } - if ( n?$src && n?$dst ) + else if ( n?$src && n?$dst ) { h1 = n$src; h2 = n$dst; - who = fmt("%s%s -> %s%s", h1, orig_p, h2, resp_p); - - if ( n?$uid ) - who = fmt("%s (uid %s)", who, n$uid ); + who = fmt("%s -> %s", h1, h2); } else if ( n?$src ) { - local p = ""; - - if ( n?$p ) - p = fmt(":%s", n$p); - h1 = n$src; - who = fmt("%s%s", h1, p); + who = fmt("%s%s", h1, (n?$p ? fmt(":%s", n$p) : "")); } + if ( n?$uid ) + who = fmt("%s (uid %s)", who, n$uid ); + local flag = (h1 in flag_nets || h2 in flag_nets); local line1 = fmt(">%s %D %s %s", (flag ? ">" : " "), network_time(), n$note, who); From 4845c3a9a61d95dd0246f3beb6fcaa6e74c88f8c Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 3 Nov 2011 14:04:13 -0700 Subject: [PATCH 031/964] send events when input entries change --- src/InputMgr.cc | 97 +++++++++++++++++++++++++++++++++++++++++++++- src/InputMgr.h | 8 +++- src/InputReader.cc | 24 +++++++----- src/input.bif | 11 ++++++ src/types.bif | 7 ++++ 5 files changed, 133 insertions(+), 14 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index bbfd354189..e987dda81e 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -38,6 +38,8 @@ struct InputMgr::ReaderInfo { PDict(InputHash)* currDict; PDict(InputHash)* lastDict; + + list* events; // events we fire when "something" happens }; @@ -165,6 +167,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); + info->events = new list(); int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); if ( success == false ) { @@ -223,7 +226,7 @@ bool InputMgr::IsCompatibleType(BroType* t) return false; } -bool InputMgr::RemoveReader(EnumVal* id) { +bool InputMgr::RemoveReader(const EnumVal* id) { ReaderInfo *i = 0; for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { @@ -254,6 +257,43 @@ bool InputMgr::RemoveReader(EnumVal* id) { return true; } +bool InputMgr::RegisterEvent(const EnumVal* id, string eventName) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->InternalError("Reader not found"); + return false; + } + + i->events->push_back(eventName); + + return true; +} + +bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->InternalError("Reader not found"); + return false; + } + + bool erased = false; + + std::list::iterator it = i->events->begin(); + while ( it != i->events->end() ) + { + if ( *it == eventName ) { + it = i->events->erase(it); + erased = true; + } + else + ++it; + } + + + return erased; +} + + bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { for ( int i = 0; i < rec->NumFields(); i++ ) { @@ -286,7 +326,7 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec } -bool InputMgr::ForceUpdate(EnumVal* id) +bool InputMgr::ForceUpdate(const EnumVal* id) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { @@ -333,6 +373,8 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { return; } + bool updated = false; + //reporter->Error("Hashing %d index fields", i->num_idx_fields); HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); @@ -355,6 +397,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { // updated i->lastDict->Remove(idxhash); delete(h); + updated = true; } } @@ -411,8 +454,25 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { i->currDict->Insert(idxhash, ih); + std::list::iterator it = i->events->begin(); + while ( it != i->events->end() ) { + EnumVal* ev; + if ( updated ) { + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + } else { + ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + } + + + Ref(idxval); + Ref(valval); + SendEvent(*it, ev, idxval, valval); + ++it; + } + } + void InputMgr::EndCurrentSend(const InputReader* reader) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { @@ -423,6 +483,23 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { IterCookie *c = i->lastDict->InitForIteration(); InputHash* ih; while ( ( ih = i->lastDict->NextEntry(c) ) ) { + + if ( i->events->size() > 0 ) { + ListVal *idx = i->tab->RecoverIndex(ih->idxkey); + assert(idx != 0); + Val *val = i->tab->Lookup(idx); + assert(val != 0); + + std::list::iterator it = i->events->begin(); + while ( it != i->events->end() ) { + Ref(idx); + Ref(val); + EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEvent(*it, ev, idx, val); + ++it; + } + } + reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); } @@ -523,6 +600,22 @@ void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c mgr.Dispatch(new Event(handler, vl)); } +void InputMgr::SendEvent(const string& name, EnumVal* event, Val* left, Val* right) +{ + EventHandler* handler = event_registry->Lookup(name.c_str()); + if ( handler == 0 ) { + reporter->Error("Event %s not found", name.c_str()); + return; + } + + val_list* vl = new val_list; + vl->append(event); + vl->append(left); + vl->append(right); + + mgr.Dispatch(new Event(handler, vl)); +} + Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { if ( position == 0 ) { diff --git a/src/InputMgr.h b/src/InputMgr.h index d147fa262a..378838c2cf 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -21,8 +21,11 @@ public: InputMgr(); InputReader* CreateReader(EnumVal* id, RecordVal* description); - bool ForceUpdate(EnumVal* id); - bool RemoveReader(EnumVal* id); + bool ForceUpdate(const EnumVal* id); + bool RemoveReader(const EnumVal* id); + bool RegisterEvent(const EnumVal* id, string eventName); + bool UnregisterEvent(const EnumVal* id, string eventName); + protected: friend class InputReader; @@ -43,6 +46,7 @@ private: bool IsCompatibleType(BroType* t); bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); + void SendEvent(const string& name, EnumVal* event, Val* left, Val* right); HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); diff --git a/src/InputReader.cc b/src/InputReader.cc index 494df3fb81..994f8b9b97 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -1,13 +1,12 @@ +// See the file "COPYING" in the main distribution directory for copyright. #include "InputReader.h" -// #include "EventRegistry.h" -// #include "Event.h" InputReader::InputReader() { buf = 0; buf_len = 1024; - disabled = true; // disabled will be set correcty in init. + disabled = true; // disabled will be set correcty in init. } InputReader::~InputReader() @@ -21,9 +20,9 @@ void InputReader::Error(const char *msg) } void InputReader::Error(const string &msg) - { +{ input_mgr->Error(this, msg.c_str()); - } +} void InputReader::Put(const LogVal* const *val) { @@ -54,16 +53,19 @@ bool InputReader::Init(string arg_source, int arg_num_fields, int arg_idx_fields return !disabled; } -void InputReader::Finish() { +void InputReader::Finish() +{ DoFinish(); disabled = true; } -bool InputReader::Update() { +bool InputReader::Update() +{ return DoUpdate(); } -void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { +void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) +{ input_mgr->SendEvent(name, num_vals, vals); } @@ -93,10 +95,12 @@ const char* InputReader::Fmt(const char* format, ...) } -void InputReader::SendEntry(const LogVal* const *vals) { +void InputReader::SendEntry(const LogVal* const *vals) +{ input_mgr->SendEntry(this, vals); } -void InputReader::EndCurrentSend() { +void InputReader::EndCurrentSend() +{ input_mgr->EndCurrentSend(this); } diff --git a/src/input.bif b/src/input.bif index 4e2cbf07b5..90dd2386b1 100644 --- a/src/input.bif +++ b/src/input.bif @@ -21,6 +21,17 @@ function Input::__force_update%(id: ID%) : bool return new Val( res, TYPE_BOOL ); %} +function Input::__add_event%(id: ID, name: string%) : bool + %{ + bool res = input_mgr->RegisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); + return new Val( res, TYPE_BOOL ); + %} + +function Input::__remove_event%(id: ID, name: string%) : bool + %{ + bool res = input_mgr->UnregisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); + return new Val( res, TYPE_BOOL ); + %} function Input::__remove_reader%(id: ID%) : bool %{ diff --git a/src/types.bif b/src/types.bif index 7b81a7f631..f90a954224 100644 --- a/src/types.bif +++ b/src/types.bif @@ -174,6 +174,13 @@ enum Reader %{ READER_ASCII, %} +enum Event %{ + EVENT_NEW, + EVENT_CHANGED, + EVENT_REMOVED, +%} + + enum ID %{ Unknown, %} From 40f6e1e09843fe76c3f7f9fc564e8b4ef91c26b9 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 3 Nov 2011 16:53:24 -0500 Subject: [PATCH 032/964] Better adaptation of Bro website style in Broxygen docs. --- doc/scripts/conf.py.in | 32 +- doc/scripts/source/_static/bro-base.css_t | 237 ------ doc/scripts/source/_static/bro-colors.css | 34 - doc/scripts/source/_static/bro-elements.css_t | 747 ------------------ doc/scripts/source/_static/bro-framing.css_t | 482 ----------- doc/scripts/source/_static/broxygen.css | 5 + doc/scripts/source/_templates/layout.html | 20 +- 7 files changed, 35 insertions(+), 1522 deletions(-) delete mode 100644 doc/scripts/source/_static/bro-base.css_t delete mode 100644 doc/scripts/source/_static/bro-colors.css delete mode 100644 doc/scripts/source/_static/bro-elements.css_t delete mode 100644 doc/scripts/source/_static/bro-framing.css_t create mode 100644 doc/scripts/source/_static/broxygen.css diff --git a/doc/scripts/conf.py.in b/doc/scripts/conf.py.in index 503b2bb612..cee16b9239 100644 --- a/doc/scripts/conf.py.in +++ b/doc/scripts/conf.py.in @@ -90,12 +90,36 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'basic' +html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +html_theme_options = { +"rightsidebar": "true", +"stickysidebar": "true", +"externalrefs": "true", +"footerbgcolor": "#333", +"footertextcolor": "#ddd", +"sidebarbgcolor": "#ffffff", +#"sidebarbtncolor": "", +"sidebartextcolor": "#333", +"sidebarlinkcolor": "#2a85a7", +"relbarbgcolor": "#ffffff", +"relbartextcolor": "#333", +"relbarlinkcolor": "#2a85a7", +"bgcolor": "#ffffff", +"textcolor": "#333", +"linkcolor": "#2a85a7", +"visitedlinkcolor": "#2a85a7", +"headbgcolor": "#f0f0f0", +"headtextcolor": "#000", +"headlinkcolor": "#2a85a7", +"codebgcolor": "#FFFAE2", +#"codetextcolor": "", +"bodyfont": "Arial, Helvetica, sans-serif", +"headfont": "Palatino,'Palatino Linotype',Georgia,serif", +} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] @@ -130,7 +154,9 @@ html_static_path = ['source/_static'] #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +html_sidebars = { +'**': ['localtoc.html', 'sourcelink.html', 'searchbox.html'], +} # Additional templates that should be rendered to pages, maps page names to # template names. diff --git a/doc/scripts/source/_static/bro-base.css_t b/doc/scripts/source/_static/bro-base.css_t deleted file mode 100644 index 30885c9d72..0000000000 --- a/doc/scripts/source/_static/bro-base.css_t +++ /dev/null @@ -1,237 +0,0 @@ - -{% import "bro-colors.css" as color %} - -#bro-main, #bro-standalone-main { - padding: 0 0 0 0; - position:relative; - z-index:1; -} - -#bro-main { - margin-bottom: 2em; - } - -#bro-standalone-main { - margin-bottom: 0em; - padding-left: 50px; - padding-right: 50px; - } - -#bro-outer { - color: {{color.text}}; - background: {{color.bg}}; -} - -#bro-title { - font-weight: bold; - font-size: 32px; - line-height:32px; - text-align: center; - padding-top: 3px; - margin-bottom: 30px; - font-family: Palatino,'Palatino Linotype',Georgia,serif;; - color: #000; - } - -.opening:first-letter { - font-size: 24px; - font-weight: bold; - letter-spacing: 0.05em; - } - -.opening { - font-size: 17px; -} - -.version { - text-align: right; - font-size: 12px; - color: #aaa; - line-height: 0; - height: 0; -} - -.git-info-version { - position: relative; - height: 2em; - top: -1em; - color: #ccc; - float: left; - font-size: 12px; -} - -.git-info-date { - position: relative; - height: 2em; - top: -1em; - color: #ccc; - float: right; - font-size: 12px; -} - -body { - font-family:Arial, Helvetica, sans-serif; - font-size:15px; - line-height:22px; - color: {{color.text}}; - margin: 0px; -} - -h1, h2, h3, h4, h5, h6, -h1 a, h2 a, h3 a, h4 a, h5 a, h6 a { - padding:0 0 20px 0; - font-weight:bold; - text-decoration:none; -} - -h1, h2 { - font-size:27px; - letter-spacing:-1px; -} - -h3 { - margin-top: 1em; - font-size:18px; -} - -h4 { - font-size:16px; -} - -h5 { - font-size:15px; -} - -h6 { - font-size:12px; -} - -p { - padding:0 0 20px 0; -} - -hr { - background:none; - height:1px; - line-height:1px; - border:0; - margin:0 0 20px 0; -} - -ul, ol { - margin:0 20px 20px 0; - padding-left:40px; -} - -blockquote { - margin:0 0 0 40px; -} - -strong, dfn { - font-weight:bold; -} - -em, dfn { - font-style:italic; -} - -sup, sub { - line-height:0; -} - -pre { - white-space:pre; -} - -pre, code, tt { - font-family:"Courier New", Courier, mono; -} - -dl { - margin: 0 0 20px 0; -} - -dl dt { - font-weight: bold; -} - -dd { - margin:0 0 20px 20px; -} - -small { - font-size:75%; -} - -a:link, -a:visited, -a:active -{ - color: {{color.link}}; -} - -a:hover -{ - color:{{color.emphasis_text}}; -} - -h1, h2, h3, h4, h5, h6, -h1 a, h2 a, h3 a, h4 a, h5 a, h6 a -{ - color: {{color.text}}; -} - -hr { - border-bottom:1px solid #ddd; -} - -pre { - color: {{color.text}}; - background: {{color.box_bg}}; - padding: 7px 5px 3px 5px; - margin-bottom: 25px; - margin-top: 0px; -} - -ul { - padding-bottom: 5px; - } - -h1, h2 { - margin-top: 30px; - } - -h1 { - margin-bottom: 50px; - margin-bottom: 20px; - padding-bottom: 5px; - border-bottom: 1px solid; - border-color: {{color.underline}}; - } - -h2 { - font-size: 24px; - } - -pre { - -moz-box-shadow:0 0 6px #ddd; - -webkit-box-shadow:0 0 6px #ddd; - box-shadow:0 0 6px #ddd; -} - -a { - text-decoration:none; - } - -p { - padding-bottom: 15px; - } - -p, dd, li { - text-align: justify; - } - -li { - margin-bottom: 5px; - } - diff --git a/doc/scripts/source/_static/bro-colors.css b/doc/scripts/source/_static/bro-colors.css deleted file mode 100644 index 0744a721d7..0000000000 --- a/doc/scripts/source/_static/bro-colors.css +++ /dev/null @@ -1,34 +0,0 @@ - -{% set bg = "#ffffff" %} -{% set text = "#333" %} -{% set underline = "#aaa" %} -{% set link = "#2a85a7" %} -{% set emphasis_text = "#c24444" %} - -{% set topbar_bg = "#000" %} - -{% set header_bg = "#f0f0f0" %} -{% set header_text = "#000" %} - -{% set footer_bg = "#333" %} -{% set footer_text = "#ddd" %} -{% set footer_links = "#ddd" %} - -{% set footer_bottom_bg = "#2b2b2b" %} -{% set footer_bottom_text = "#555" %} - -{% set menu_bg = "#fff" %} -{% set menu_hover_bg = "#ddd" %} -{% set menu_selected_bg = "#ddd" %} -{% set menu_shadow = "#eee" %} -{% set menu_border = "#ccc" %} - -{% set highlight_bg = "#ff6" %} -{% set box_bg = "#FFFAE2" %} -{% set box_hover_bg = "#FCF3A4" %} -{% set box_border = "#eee" %} - -{% set table_hover_bg = "#FFFAE2" %} - -{% set note_bg = "#FEE0C6" %} -{% set note_border = "#aaa" %} diff --git a/doc/scripts/source/_static/bro-elements.css_t b/doc/scripts/source/_static/bro-elements.css_t deleted file mode 100644 index f2c7a30f10..0000000000 --- a/doc/scripts/source/_static/bro-elements.css_t +++ /dev/null @@ -1,747 +0,0 @@ - -{% import "bro-colors.css" as color %} - -#searchform { - position:relative; -} - -#searchform #searchsubmit { - position:absolute; - top:0; - right:4px; - border:0; - height:22px; - width:20px; - text-indent:-9999px; - cursor:pointer; -} - -#searchform #s { - width:99%; - height:20px; - border:0; - position:absolute; - top:0; - left:0; - text-indent:5px; -} - -#searchform #searchsubmit { - background:url({{docroot}}/images/searchsubmit.gif) no-repeat center center; -} - -#searchform #s { - background:#555; -} - -.postmeta, -#searchform #s { - color: {{color.footer_bottom_text}}; -} - -#breadcrumbs { - font-size:12px; - float: left; - z-index: 1; - margin-bottom: 2em; -} - -#logo { - position: relative; - top: -5px; - margin-left: auto; - margin-right: auto; - display: block; - padding: 0px; - } - -#logo h1 { - padding:0; - margin:0; -} - -#logo-menubar { - position: absolute; - padding-left: 1em; -} - -#bro-eyes { - margin-bottom: 3em; - margin-left: auto; - margin-right: auto; - } - -.post { - margin-bottom:40px; - padding-bottom:20px; -} - -.entry-title { - padding-bottom:5px; -} - -.postmeta { - font-size:11px; - text-transform:uppercase; -} - -.post:last-child { - border:0; - margin-bottom:0; - padding-bottom:20px; -} - -.postmeta { -} - -a.more-link { - font-weight:bold; - text-decoration:none; -} - -.clearfix:after { - content: "."; - display: block; - clear: both; - visibility: hidden; - line-height: 0; - height: 0; -} - -.clearfix { - display: inline-block; -} - -html[xmlns] .clearfix { - display: block; -} -* html .clearfix { - height: 1%; -} - -.rounded { - -moz-border-radius:5px; - -webkit-border-radius:5px; - border-radius:5px; -} - -.message { - padding:8px; - margin-bottom:20px; -} - -.left { - float:left; -} - -.center { - display:block; - margin-left:auto; - margin-right:auto; - text-align:center; -} - -.right { - float:right; -} - -img.left { - padding:3px 20px 20px 0; -} - -img.right { - padding:3px 0 20px 20px; -} - -.widget img.left { - padding:5px 10px 10px 0; -} - -.widget img.right { - padding:5px 0 10px 10px; -} - -.hidden { - display:none; -} - -.pad { - padding:10px; -} - -UL.clean, OL.clean { - list-style-type:none; - margin:0; - padding:0; -} - -.primary-promo-container { - margin:0 0 40px 0; -} - -.promo-container { - position:relative; -} - -.promo-navigation { - position:absolute; - bottom:20px; - right:20px; - z-index:20; -} - -.promo-navigation a { - display:block; - float:left; - width:20px; - height:20px; - text-indent:-9999px; - margin-left:5px; -} - -.promo-navigation-small a { - width:10px; - height:10px; -} - -.promo-bottom-navigation { - padding-bottom:20px; - clear:both; -} - -.promo-bottom-navigation .promo-navigation { - bottom:0; - right:0; -} - -.pagination { - margin:0 0 40px 0; - line-height:30px; - font-size:12px; -} - -.pagination a, .pagination a span { - padding:5px 10px; - margin:2px; - text-decoration:none; -} - -.pagination span.current { - font-weight:bold; - padding:3px 3px; - margin:2px; -} - -.iconlist h5 { - line-height:100%; - font-size:15px; -} - -.iconlist h5, .iconlist h5 a { - font-weight:bold; - text-decoration:none; - padding:0; -} - -.iconlist p { - font-size:12px; -} - -.iconlist img.left { - margin:0 10px 10px 0; - padding:0; -} - -ul.iconlist, ul.iconlist li { - margin:0; - padding:0; -} - -p.quotebox { - padding:14px 20px 14px 40px; - line-height:100%; - margin:0 0 40px 0; - font-size:12px; -} - -p.quotebox { - background-position:10px center; - background-repeat:no-repeat; - background-image:url({{docroot}}/images/quote_left.gif); -} - -.ads { - padding:16px 0 0 16px; -} - -.ads img { - padding:0 16px 16px 0; - display:block; - float:left; -} - -.ads .widgettitle { - margin-right:16px; -} - -.grid-item { - padding:10px 0 0 0; - margin:0 0 40px 0; -} - -.grid-item p, .grid-item h3 { - padding-bottom:5px; -} - -/* Contact Form */ -form.standard .field { - padding:0 0 15px 0; -} - -form.standard label { - display:block; - padding:0 0 2px 2px; -} - -form.standard .textbox, -form.standard .textarea, -form.standard .select, -form.standard .button, -form.standard .checkbox, -form.standard .radio { - padding:8px 5px; -} - -form.standard .checkbox, form.standard .radio { - border:0; -} - -form.standard .button { - cursor:pointer; - padding:5px 8px; - line-height:100%; -} - -form.standard .error { - padding:1px 6px; - line-height:100%; -} - -form.standard .textbox, -form.standard .textarea, -form.standard .select, -form.standard .button, -form.standard .checkbox, -form.standard .radio { - border:1px solid #ccc; -} - -form.standard .button:hover { - background:#e5e5e5; -} - -.demogrid p { - font-size:12px; - font-weight:bold; - text-align:center; - margin:10px 0; - padding:8px 0; -} - -.faq h2 { - font-size: 17px; - } - -.faq p { - margin-left: 2em; - } - - -.invisible { - visibility: hidden; -} - -.paper-block { - font-size: 12px; - margin-left: 20px; -} - -.paper-title { - font-weight: bold; -} - -.contents { - padding: 10px; - background: #FFFAE2; - margin: 20px; - } - -.topic-title { - font-size: 20px; - font-weight: bold; - padding: 0px 0px 5px 0px; - text-align: center; - padding-top: .5em; -} - -.contents li { - margin-bottom: 0px; - list-style-type: square; -} - -.contents ul ul li { - margin-left: 0px; - padding-left: 0px; - padding-top: 0em; - font-size: 90%; - list-style-type: square; - font-weight: normal; -} - -.contents ul ul ul li { - list-style-type: none; -} - -.contents ul ul ul ul li { - display:none; -} - -.contents ul li { - padding-top: 1em; - list-style-type: none; - font-weight: bold; -} - -.contents ul { - margin-left: 0px; - padding-left: 2em; - margin: 0px 0px 0px 0px; -} - -#page-title { - font-size: 30px; - font-weight: bold; - color: {{color.text}}; - background: {{color.menu_selected_bg}}; - padding: 2px 2px 2px 0px; -} - - -.widget { - margin-left: 1em; -} - -.sidebar-toc ul li { - padding-bottom: 0px; - text-align: left; - list-style-type: square; - list-style-position: inside; - padding-left: 1em; - text-indent: -1em; - } - -.sidebar-toc ul li li { - margin-left: 1em; - margin-bottom: 0px; - list-style-type: square; - } - -.sidebar-toc ul li li a { - font-size: 8pt; -} - -a.toc-top-link { - font-size: 14px; - float: right; - margin-right: 15px; - } - -.small { - font-size: 16px; -} - -.heading { - font-weight: bold; - font-size: 18px; - color:{{color.emphasis_text}}; -} - -.shadow, .download, .note, .exercise, .question, .solution, .visible_solution, .contents, .warning, .error, .success, pre { - -moz-box-shadow:0 0 6px #ddd; - -webkit-box-shadow:0 0 6px #ddd; - box-shadow:0 0 6px #ddd; -} - -.exercise, .question, .solution, .visible_solution, .note, .warning, .error, .success, pre { - margin-left: 2em; - margin-right: 2em; - overflow: auto; - } - -.exercise, .question, .solution, .visible_solution, .note, .warning, .error, .success, pre { - margin-top: .5em; - margin-bottom: 1.5em; - } - -.exercise, .question, .solution, .visible_solution, .note, .warning, .error, .success { - border: solid 0px #aaa; - padding: 1em 1em 0em 1em; - font-size: 15px; - color: {{color.text}}; -} - -.note { - background: #fcfff4; /* Old browsers */ - background: -moz-linear-gradient(top, #fcfff4 0%, #e9e9ce 100%); /* FF3.6+ */ - background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#fcfff4), color-stop(100%,#e9e9ce)); /* Chrome,Safari4+ */ - background: -webkit-linear-gradient(top, #fcfff4 0%,#e9e9ce 100%); /* Chrome10+,Safari5.1+ */ - background: -o-linear-gradient(top, #fcfff4 0%,#e9e9ce 100%); /* Opera 11.10+ */ - background: -ms-linear-gradient(top, #fcfff4 0%,#e9e9ce 100%); /* IE10+ */ - background: linear-gradient(top, #fcfff4 0%,#e9e9ce 100%); /* W3C */ - filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#fcfff4', endColorstr='#e9e9ce',GradientType=0 ); /* IE6-9 */ -} - -.exercise, .question { - background: #f0f0f0; /* Old browsers */ - background: -moz-linear-gradient(top, #f0f0f0 0%, #e9e9ce 100%); /* FF3.6+ */ - background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#f0f0f0), color-stop(100%,#e9e9ce)); /* Chrome,Safari4+ */ - background: -webkit-linear-gradient(top, #f0f0f0 0%,#e9e9ce 100%); /* Chrome10+,Safari5.1+ */ - background: -o-linear-gradient(top, #f0f0f0 0%,#e9e9ce 100%); /* Opera 11.10+ */ - background: -ms-linear-gradient(top, #f0f0f0 0%,#e9e9ce 100%); /* IE10+ */ - background: linear-gradient(top, #f0f0f0 0%,#e9e9ce 100%); /* W3C */ - filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#f0f0f0', endColorstr='#e9e9ce',GradientType=0 ); /* IE6-9 */ -} - -.warning, .error { - background: #ffc578; /* Old browsers */ - background: -moz-linear-gradient(top, #ffc578 0%, #fb9d23 100%); /* FF3.6+ */ - background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#ffc578), color-stop(100%,#fb9d23)); /* Chrome,Safari4+ */ - background: -webkit-linear-gradient(top, #ffc578 0%,#fb9d23 100%); /* Chrome10+,Safari5.1+ */ - background: -o-linear-gradient(top, #ffc578 0%,#fb9d23 100%); /* Opera 11.10+ */ - background: -ms-linear-gradient(top, #ffc578 0%,#fb9d23 100%); /* IE10+ */ - background: linear-gradient(top, #ffc578 0%,#fb9d23 100%); /* W3C */ - filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#ffc578', endColorstr='#fb9d23',GradientType=0 ); /* IE6-9 */ -} - -.solution, .visible_solution { - background: #f0c0c0; /* Old browsers */ - background: -moz-linear-gradient(top, #f0c0c0 0%, #f0c0f0 100%); /* FF3.6+ */ - background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#f0c0c0), color-stop(100%,#f0c0f0)); /* Chrome,Safari4+ */ - background: -webkit-linear-gradient(top, #f0c0c0 0%,#f0c0f0 100%); /* Chrome10+,Safari5.1+ */ - background: -o-linear-gradient(top, #f0c0c0 0%,#f0c0f0 100%); /* Opera 11.10+ */ - background: -ms-linear-gradient(top, #f0c0c0 0%,#f0c0f0 100%); /* IE10+ */ - background: linear-gradient(top, #f0c0c0 0%,#f0c0f0 100%); /* W3C */ - filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#f0c0c0', endColorstr='#f0c0f0',GradientType=0 ); /* IE6-9 */ -} - -.admonition-title { - font-weight: normal; - font-size: 16px; - color: #000; - border-bottom: solid 1px #aaa; - padding-bottom: 0em; - margin-bottom: .5em; - margin-top: 0em; - } - -.download:hover { - background-color: #FCF3A4 -} - -.download { - border: solid 0px #eee; - background: #FFFAE2 url('../images/icons/download.png') 8px center no-repeat; - color: #333; - font-size: 20px; - right: 32px; - padding-top: 10px; - padding-bottom: 5px; - padding-left: 50px; - padding-right: 20px; - } - -.download-beta { - background: #FFCC99 url('../images/icons/download.png') 8px center no-repeat; -} - -/* .hll is pygments; see pygments.css */ -.hll { - color: {{color.text}}; - background: {{color.box_bg}}; - padding: 7px 5px 3px 5px; - margin-bottom: 25px; - margin-top: 0px; -} - -.field-list tbody tr:hover td { - background: {{color.bg}}; -} - -.field-list td, .field-list th { - border-bottom: 0px; - padding: 0 6px 0 0; -} - -.large-table td, .large-table th { - font-size: 80%; -} - -.large-table ul { - padding-left: 1em; - text-indent: -1em; - list-style-position: inside; -} - -.highlight { - background-color: {{color.highlight_bg}}; -} - -.hovershadow:hover, -.pagination a:hover { - -moz-box-shadow:0 0 3px #d0d0d0; - -webkit-box-shadow:0 0 3px #d0d0d0; - box-shadow:0 0 3px #d0d0d0; -} - -.iconlist h5, -.iconlist h5 a { - color:{{color.emphasis_text}}; -} -.iconlist h5, -.iconlist h5 a { - color:{{color.emphasis_text}}; -} - -form.standard .button, -.widgettitle -{ - color: {{color.text}}; -} - -.heading { - font-weight: bold; - font-size: 18px; - color:{{color.emphasis_text}}; -} - -.post, -.widgettitle { - border-bottom:1px solid #ddd; -} - -.line-block { - margin-bottom: 1em; - } - -.bibliography-title { - padding-top: 1em; - font-size: 18px; - font-weight: bold; - } - -.line-block { - margin-bottom: 1em; - } - -.bibliography-title { - padding-top: 1em; - font-size: 18px; - font-weight: bold; - } - -.promo-navigation a, -.pagination a, -.pagination a span, -p.quotebox, -.ads, -.boxed { - background:#fff; -} - -.promo-navigation a, -.pagination a, -.pagination a span, -.ads, -.demogrid p { - border:1px solid #ddd; -} - -table.docutils { - margin-bottom: 40px; -} - -table.docutils th { - font-size: 15px; - font-weight: bold; - padding: 10px 8px; - border-bottom: 1px solid #aaa; - vertical-align: top; -} - -table.docutils td { - font-size: 14px; - border-bottom: 1px solid #aaa; - padding: 6px 8px; - vertical-align: top; -} - -table.docutils tr:first-child td { - border-top: 1px solid #aaa; -} - -table.docutils tbody tr:hover td { - background: {{color.table_hover_bg}}; -} - -div.sphinxsidebar { - top: 30px; - bottom: 0; - margin: 0; - position: fixed; - overflow: auto; - height: auto; - float: right; - right: 0; -} - -/* this is nice, but it it leads to hidden headings when jumping - to an anchor */ -/* -div.related { - position: fixed; -} - -div.documentwrapper { - margin-top: 30px; -} -*/ - -div.sphinxsidebar h3 { - font-size: 1.4em; - font-weight: normal; - margin: 0; - padding: 0; -} - - -div.sphinxsidebar h4 { - font-size: 1.3em; - font-weight: normal; - margin: 5px 0 0 0; - padding: 0; -} - -div.sphinxsidebar p.topless { - margin: 5px 10px 10px 10px; -} - -div.sphinxsidebar ul { - margin: 10px; - padding: 0; -} - -div.sphinxsidebar input { - border: 1px solid {{ color.link }}; - font-family: sans-serif; - font-size: 1em; -} diff --git a/doc/scripts/source/_static/bro-framing.css_t b/doc/scripts/source/_static/bro-framing.css_t deleted file mode 100644 index 1458c239a4..0000000000 --- a/doc/scripts/source/_static/bro-framing.css_t +++ /dev/null @@ -1,482 +0,0 @@ - -{% import "bro-colors.css" as color %} - -#header { - padding: 20px 0; - position:relative; - z-index:2; -} - -#header { - background: {{color.header_bg}}; /* old browsers */ - background: -moz-linear-gradient(top, {{color.header_bg}} 80%, {{color.bg}} 100%); /* firefox */ - background: -webkit-gradient(linear, left top, left bottom, color-stop(80%,{{color.header_bg}}), color-stop(100%,{{color.bg}})); /* webkit */ - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='{{color.header_bg}}', endColorstr='{{color.bg}}',GradientType=0 ); /* ie */ - zoom: 1; -} - -#header { - /* border-top: 20px solid #fff; - border-top:10px solid #fff; - border-bottom:1px solid #ddd; */ -} - -#header { - padding-bottom: 0em; - margin-bottom: 1em; - } - -#bro-main .widgettitle, -{ - color: {{color.text}}; -} - -#topbar p { - padding:0; - font-size:12px; - line-height:35px; -} - -#topbar a -{ - color: {{color.bg}}; -} - - -#topbar { - background: {{color.topbar_bg}}; -} - -#topbar a { - text-decoration:none; -} - -#topbar #searchform { - margin-top:6px; -} - -#topbar { - color: {{color.footer_bg}}; -} - -#footer { - padding:40px 0 0 0; -} - -#footer { - color: {{color.footer_text}}; - background: {{color.footer_bg}}; - } - -#footer .widgettitle { - color: {{color.footer_text}}; -} - -#footer-bottom { - padding:10px 0; - font-size:12px; -} - -#footer-bottom a.scroll-top { - padding-left:15px; - text-decoration:none; -} - -#footer a { - text-decoration:none; -} - -#footer .widget li { - padding-bottom:10px; -} - -#footer .widget_links li { - padding-bottom:1px; -} - -#footer .widget li:last-child { - padding-bottom:0; -} - -#footer-bottom { - color: {{color.footer_bottom_text}}; - background: {{color.footer_bottom_bg}}; -} - -#footer-bottom a:hover, -#footer .widget_links ul a:hover, -#footer .widget_links ol a:hover -{ - color:{{color.emphasis_text}}; -} - -#footer .widget_links ul a, -#footer .widget_links ol a -{ - color: {{color.footer_links}}; -} - - -#footer { - border-bottom:1px solid #444; -} - -#footer-bottom { - border-top:1px solid #222; -} - -#footer-bottom a.scroll-top { - background:url({{docroot}}/images/to-top.gif) no-repeat left center; -} - - -li.tweet_odd, li.tweet_even { - text-align: left; -} - -.line-block { - margin-bottom: 1em; - } - -.bibliography-title { - padding-top: 1em; - font-size: 18px; - font-weight: bold; - } - -.postmeta, -#footer-bottom a, -#searchform #s { - color: {{color.footer_bottom_text}}; -} -.line-block { - margin-bottom: 1em; - } - -.bibliography-title { - padding-top: 1em; - font-size: 18px; - font-weight: bold; - } - -#footer-bottom a { - color: {{color.footer_bottom_text}}; -} - -.widget { - margin:0 0 40px 0; -} - -.widget, .widgettitle { - font-size:12px; - line-height:18px; -} - -.widgettitle { - font-weight:bold; - text-transform:uppercase; - padding:0 0 10px 0; - margin:0 0 20px 0; - line-height:100%; -} - -.widget UL, .widget OL { - list-style-type:none; - margin:0; - padding:0; -} - -.widget p { - padding:0; -} - -.textwidget p { - padding:0 0 10px 0; -} - -.widget li { - padding-bottom:10px; -} - -.widget a { - text-decoration:none; -} - -.sf-menu, .sf-menu * { - margin: 0; - padding: 0; - list-style: none; - font-size:16px; -} -.sf-menu { - line-height:100%; - right:0; - bottom:0; - float: left; -} - -.sf-menu ul { - position: absolute; - top: -999em; - width: 20em; /* left offset of submenus need to match (see below) */ -} -.sf-menu ul li { - width: 100%; -} -.sf-menu li:hover { - visibility: inherit; /* fixes IE7 'sticky bug' */ -} -.sf-menu li { - float: left; - position: relative; -} -.sf-menu a { - display: block; - position: relative; - padding: .75em 1em; - text-decoration:none; - font-weight:bold; -} - -.sf-menu ul ul { - width: 100%; -} - -.sf-menu li:hover ul, -.sf-menu li.sfHover ul { -/* left: 0; */ - top: 2.5em; /* match top ul list item height */ - z-index: 99; -} -ul.sf-menu li:hover li ul, -ul.sf-menu li.sfHover li ul { - top: -999em; -} -ul.sf-menu li li:hover ul, -ul.sf-menu li li.sfHover ul { -/* left: 10em; */ /* match ul width */ - top: -11px; - margin-left: 2px; -} -ul.sf-menu li li:hover li ul, -ul.sf-menu li li.sfHover li ul { - top: -999em; -} -ul.sf-menu li li li:hover ul, -ul.sf-menu li li li.sfHover ul { - left: 10em; /* match ul width */ - top: -11px; -} - -.sf-menu ul a { - font-size:12px; -} - -.sf-menu ul li a { - font-weight: normal; - font-size: 13px; -} - -.sf-menu li ul { - padding:10px; -} - -.sf-menu a.sf-with-ul { - padding-right: 30px; - min-width: 1px; /* trigger IE7 hasLayout so spans position accurately */ -} - -.sf-menu li li -.sf-menu a { - border-bottom:1px solid #ddd; -} - -.sf-sub-indicator { - position: absolute; - display: block; - right: 10px; - top: 1.05em; /* IE6 only */ - width: 10px; - height: 10px; - text-indent: -999em; - overflow: hidden; -} - -.sf-menu a:hover, -.sfHover a, -.sf-menu ul li.sfHover > a -{ - color:{{color.emphasis_text}}; -} - -a > .sf-sub-indicator { /* give all except IE6 the correct values */ - top: 1em; - background-position: 0 -100px; /* use translucent arrow for modern browsers*/ -} - -a:focus > .sf-sub-indicator, -a:hover > .sf-sub-indicator, -a:active > .sf-sub-indicator, -li:hover > a > .sf-sub-indicator, -li.sfHover > a > .sf-sub-indicator { - background-position: -10px -100px; /* arrow hovers for modern browsers*/ -} - -.sf-menu ul .sf-sub-indicator { background-position: -10px 0; top:0.75em; } -.sf-menu ul a > .sf-sub-indicator { background-position: 0 0; } - -.sf-menu ul a:focus > .sf-sub-indicator, -.sf-menu ul a:hover > .sf-sub-indicator, -.sf-menu ul a:active > .sf-sub-indicator, -.sf-menu ul li:hover > a > .sf-sub-indicator, -.sf-menu ul li.sfHover > a > .sf-sub-indicator { - background-position: -10px 0; /* arrow hovers for modern browsers*/ -} - -.sf-menu ul a:hover { - background: {{color.menu_hover_bg}}; -} -/* -.sf-menu li li { - background: {{color.menu_bg}}; - border-left:1px solid {{color.menu_border}}; - border-right:1px solid {{color.menu_border}}; -} -.sf-menu li li:first-child { - border-top:1px solid {{color.menu_border}}; -} -.sf-menu li ul { - -moz-box-shadow:2px 2px 2px {{color.menu_shadow}}; - -webkit-box-shadow:2px 2px 2px {{color.menu_shadow}}; - box-shadow:2px 2px 2px {{color.menu_shadow}}; -} -*/ - -.sf-menu li { - border-bottom: 1px solid #000; -} - -.sf-menu ul li { - border-top: 1px solid #000; -} - -.sf-menu ul li { - border-bottom: 0px; -} - -.sf-menu li { - display: inline; -} - -.sf-menu a -{ - color: {{color.text}}; -} - -.sf-sub-indicator { - background:url('{{docroot}}/images/menu/default-submenu-sprite.png') no-repeat -10px -100px; /* 8-bit indexed alpha png. IE6 gets solid image only */ -} - -.sf-navbar { - height: 2.5em; - padding-bottom: 2.5em; - position: relative; -} -.sf-navbar li { - position: static; -} -.sf-navbar a { - border-top: none; -} -.sf-navbar li li { - position: relative; -} -.sf-navbar li li ul { - width: 13em; -} -.sf-navbar li li li { - width: 100%; -} -.sf-navbar ul li { - width: auto; - float: left; -} -.sf-navbar a, .sf-navbar a:visited { - border: none; -} -.sf-navbar li.current-menu { -} - -ul.sf-navbar ul li:hover li, -ul.sf-navbar ul li.sfHover li, -.sf-navbar ul a:focus, .sf-navbar ul a:hover, .sf-navbar ul a:active { - background: {{color.menu_hover_bg}}; -} -ul.sf-navbar li li li:hover, -ul.sf-navbar li li li.sfHover, -.sf-navbar li li.current-menu li.current-menu, -.sf-navbar ul li li a:focus, .sf-navbar ul li li a:hover, .sf-navbar ul li li a:active { - background: {{color.menu_hover_bg}}; -} -ul.sf-navbar .current-menu ul, -ul.sf-navbar ul li:hover ul, -ul.sf-navbar ul li.sfHover ul { - left: 0; - top: 2.5em; /* match top ul list item height */ -} -ul.sf-navbar .current-menu ul ul { - top: -999em; -} - -.sf-navbar li li.current-menu > a { - font-weight: bold; -} - -/*** point all arrows down ***/ -/* point right for anchors in subs */ -.sf-navbar ul .sf-sub-indicator { background-position: -10px -100px; } -.sf-navbar ul a > .sf-sub-indicator { background-position: 0 -100px; } -/* apply hovers to modern browsers */ -.sf-navbar ul a:focus > .sf-sub-indicator, -.sf-navbar ul a:hover > .sf-sub-indicator, -.sf-navbar ul a:active > .sf-sub-indicator, -.sf-navbar ul li:hover > a > .sf-sub-indicator, -.sf-navbar ul li.sfHover > a > .sf-sub-indicator { - background-position: -10px -100px; /* arrow hovers for modern browsers*/ -} - -/*** remove shadow on first submenu ***/ -.sf-navbar > li > ul { - background: transparent; - padding: 0; - -moz-border-radius-bottomleft: 0; - -moz-border-radius-topright: 0; - -webkit-border-top-right-radius: 0; - -webkit-border-bottom-left-radius: 0; - -} - -.selected-menu { -} - -#current-menu { - background: {{color.menu_selected_bg}}; -} - -#current-submenu { - font-weight: bold; - background: transparent; -} - -#current-submenu:hover { - background: {{color.menu_hover_bg}}; -} - -#current-submenu { - color: {{ color.text }}; -} - -#current-submenu:hover { - color:{{color.emphasis_text}}; -} - diff --git a/doc/scripts/source/_static/broxygen.css b/doc/scripts/source/_static/broxygen.css new file mode 100644 index 0000000000..b862bc1a20 --- /dev/null +++ b/doc/scripts/source/_static/broxygen.css @@ -0,0 +1,5 @@ +a:hover +{ + text-decoration:none; + color:#c24444; +} diff --git a/doc/scripts/source/_templates/layout.html b/doc/scripts/source/_templates/layout.html index 120c942c4c..5a481ab882 100644 --- a/doc/scripts/source/_templates/layout.html +++ b/doc/scripts/source/_templates/layout.html @@ -1,23 +1,5 @@ {% extends "!layout.html" %} -{% set css_files = css_files + ["_static/bro-base.css", "_static/bro-framing.css", "_static/bro-elements.css"] %} -{% block relbar1 %}{% endblock %} +{% set css_files = css_files + ["_static/broxygen.css"] %} {% block relbar2 %}{% endblock %} - -{% block sidebar2 %} -
- {{ super() }} -
-{% endblock %} - -{% block content %} -
- {{ relbar() }} -
-
- {{ super() }} -
-
-
-{% endblock %} From 2e3874331d0540724a47e91c8b0146b7b3fa6846 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 4 Nov 2011 12:41:10 -0700 Subject: [PATCH 033/964] support for filters and little event fix --- scripts/base/frameworks/input/main.bro | 10 ++ src/InputMgr.cc | 204 +++++++++++++++++++++---- src/InputMgr.h | 5 + src/input.bif | 22 ++- 4 files changed, 210 insertions(+), 31 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 4bb7129d03..35ca6dfa4e 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -11,6 +11,16 @@ export { destination: any; reader: Reader &default=default_reader; }; + + type Filter: record { + name: string; + ## descriptive name. for later removal + + + pred: function(typ: Input::Event, left: any, right: any): bool &optional; + ## decision function, that decides if an inserton, update or removal should really be executed + }; + } @load base/input.bif diff --git a/src/InputMgr.cc b/src/InputMgr.cc index e987dda81e..0b87a561ad 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -24,6 +24,11 @@ public: declare(PDict, InputHash); +struct InputMgr::Filter { + EnumVal* id; + string name; + Func* pred; +}; struct InputMgr::ReaderInfo { EnumVal* id; @@ -39,10 +44,17 @@ struct InputMgr::ReaderInfo { PDict(InputHash)* currDict; PDict(InputHash)* lastDict; - list* events; // events we fire when "something" happens + list events; // events we fire when "something" happens + list filters; // events we fire when "something" happens + +// ~ReaderInfo(); }; +//void InputMgr::~ReaderInfo() { +// +//} + struct InputReaderDefinition { bro_int_t type; // the type const char *name; // descriptive name for error messages @@ -167,8 +179,6 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); - info->events = new list(); - int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); if ( success == false ) { RemoveReader(id); @@ -264,7 +274,7 @@ bool InputMgr::RegisterEvent(const EnumVal* id, string eventName) { return false; } - i->events->push_back(eventName); + i->events.push_back(eventName); return true; } @@ -276,21 +286,23 @@ bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { return false; } - bool erased = false; + //bool erased = false; - std::list::iterator it = i->events->begin(); - while ( it != i->events->end() ) + std::list::iterator it = i->events.begin(); + while ( it != i->events.end() ) { if ( *it == eventName ) { - it = i->events->erase(it); - erased = true; + it = i->events.erase(it); + return true; + // erased = true; } else ++it; } - return erased; + return false; + //return erased; } @@ -337,6 +349,59 @@ bool InputMgr::ForceUpdate(const EnumVal* id) return i->reader->Update(); } +bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->InternalError("Reader not found"); + return false; + } + + RecordType* rtype = fval->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::Filter, 0) ) + { + reporter->Error("filter argument not of right type"); + return false; + } + + + Val* name = fval->Lookup(rtype->FieldOffset("name")); + Val* pred = fval->Lookup(rtype->FieldOffset("pred")); + + Filter filter; + filter.name = name->AsString()->CheckString(); + filter.id = id->Ref()->AsEnumVal(); + filter.pred = pred ? pred->AsFunc() : 0; + + i->filters.push_back(filter); + + return true; +} + +bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->InternalError("Reader not found"); + return false; + } + + + std::list::iterator it = i->filters.begin(); + while ( it != i->filters.end() ) + { + if ( (*it).name == name ) { + it = i->filters.erase(it); + return true; + break; + } + else + ++it; + } + + return false;; +} + + + Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const LogVal* const *vals) { Val* idxval; int position = 0; @@ -398,6 +463,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { i->lastDict->Remove(idxhash); delete(h); updated = true; + } } @@ -437,6 +503,48 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { valval = r; } + + Val* oldval = 0; + if ( updated == true ) { + // in that case, we need the old value to send the event (if we send an event). + oldval = i->tab->Lookup(idxval); + } + + + // call filters first do determine if we really add / change the entry + std::list::iterator it = i->filters.begin(); + while ( it != i->filters.end() ) { + if (! (*it).pred ) { + continue; + } + + EnumVal* ev; + Ref(idxval); + Ref(valval); + + if ( updated ) { + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + } else { + ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + } + + val_list vl(3); + vl.append(ev); + vl.append(idxval); + vl.append(valval); + Val* v = (*it).pred->Call(&vl); + bool result = v->AsBool(); + Unref(v); + + if ( result == false ) { + // throw away. Hence - we quit. + return; + } + + ++it; + } + + //i->tab->Assign(idxval, valval); HashKey* k = i->tab->ComputeHash(idxval); if ( !k ) { @@ -454,21 +562,28 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { i->currDict->Insert(idxhash, ih); - std::list::iterator it = i->events->begin(); - while ( it != i->events->end() ) { + // send events now that we are kind of finished. + std::list::iterator filter_iterator = i->events.begin(); + while ( filter_iterator != i->events.end() ) { EnumVal* ev; - if ( updated ) { + Ref(idxval); + + if ( updated ) { // in case of update send back the old value. ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + assert ( oldval != 0 ); + Ref(oldval); + SendEvent(*filter_iterator, ev, idxval, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + Ref(valval); + SendEvent(*filter_iterator, ev, idxval, valval); } - Ref(idxval); - Ref(valval); - SendEvent(*it, ev, idxval, valval); - ++it; + ++filter_iterator; } + + } @@ -483,24 +598,61 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { IterCookie *c = i->lastDict->InitForIteration(); InputHash* ih; while ( ( ih = i->lastDict->NextEntry(c) ) ) { + + if ( i->events.size() != 0 || i->filters.size() != 0 ) // we have a filter or an event + { - if ( i->events->size() > 0 ) { ListVal *idx = i->tab->RecoverIndex(ih->idxkey); assert(idx != 0); Val *val = i->tab->Lookup(idx); assert(val != 0); - std::list::iterator it = i->events->begin(); - while ( it != i->events->end() ) { - Ref(idx); - Ref(val); - EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEvent(*it, ev, idx, val); - ++it; + + { + // ask filter, if we want to expire this element... + std::list::iterator it = i->filters.begin(); + while ( it != i->filters.end() ) { + if (! (*it).pred ) { + continue; + } + + EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + Ref(idx); + Ref(val); + + val_list vl(3); + vl.append(ev); + vl.append(idx); + vl.append(val); + Val* v = (*it).pred->Call(&vl); + bool result = v->AsBool(); + Unref(v); + + if ( result == false ) { + // throw away. Hence - we quit and simply go to the next entry of lastDict + continue; + } + + ++it; + } } + + // + + { + std::list::iterator it = i->events.begin(); + while ( it != i->events.end() ) { + Ref(idx); + Ref(val); + EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEvent(*it, ev, idx, val); + ++it; + } + } + } - reporter->Error("Expiring element"); + //reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); } diff --git a/src/InputMgr.h b/src/InputMgr.h index 378838c2cf..74914e65ad 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -26,6 +26,9 @@ public: bool RegisterEvent(const EnumVal* id, string eventName); bool UnregisterEvent(const EnumVal* id, string eventName); + bool AddFilter(EnumVal *id, RecordVal* filter); + bool RemoveFilter(EnumVal* id, const string &name); + protected: friend class InputReader; @@ -62,6 +65,8 @@ private: string Hash(const string &input); + struct Filter; + }; extern InputMgr* input_mgr; diff --git a/src/input.bif b/src/input.bif index 90dd2386b1..2301482506 100644 --- a/src/input.bif +++ b/src/input.bif @@ -8,34 +8,46 @@ module Input; %%} type ReaderDescription: record; +type Filter: record; -function Input::__create_reader%(id: ID, description: Input::ReaderDescription%) : bool +function Input::__create_reader%(id: Log::ID, description: Input::ReaderDescription%) : bool %{ InputReader *the_reader = input_mgr->CreateReader(id->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} -function Input::__force_update%(id: ID%) : bool +function Input::__force_update%(id: Log::ID%) : bool %{ bool res = input_mgr->ForceUpdate(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__add_event%(id: ID, name: string%) : bool +function Input::__add_event%(id: Log::ID, name: string%) : bool %{ bool res = input_mgr->RegisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_event%(id: ID, name: string%) : bool +function Input::__remove_event%(id: Log::ID, name: string%) : bool %{ bool res = input_mgr->UnregisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_reader%(id: ID%) : bool +function Input::__remove_reader%(id: Log::ID%) : bool %{ bool res = input_mgr->RemoveReader(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} +function Input::__add_filter%(id: Log::ID, filter: Input::Filter%) : bool + %{ + bool res = input_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); + return new Val( res, TYPE_BOOL ); + %} + +function Input::__remove_filter%(id: Log::ID, name: string%) : bool + %{ + bool res = input_mgr->RemoveFilter(id->AsEnumVal(), name->AsString()->CheckString()); + return new Val( res, TYPE_BOOL); + %} From 5f37040c962a9f3d524ddcccfaeba982acb0d500 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 4 Nov 2011 13:59:43 -0700 Subject: [PATCH 034/964] filters really working as intented (though probably still memleaky) --- src/InputMgr.cc | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 0b87a561ad..7ca1e1b485 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -537,8 +537,14 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { Unref(v); if ( result == false ) { - // throw away. Hence - we quit. - return; + if ( !updated ) { + // throw away. Hence - we quit. + return; + } else { + // keep old one + i->currDict->Insert(idxhash, h); + return; + } } ++it; @@ -596,8 +602,11 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { } // lastdict contains all deleted entries and should be empty apart from that IterCookie *c = i->lastDict->InitForIteration(); + i->lastDict->MakeRobustCookie(c); InputHash* ih; - while ( ( ih = i->lastDict->NextEntry(c) ) ) { + HashKey *lastDictIdxKey; + //while ( ( ih = i->lastDict->NextEntry(c) ) ) { + while ( ( ih = i->lastDict->NextEntry(lastDictIdxKey, c) ) ) { if ( i->events.size() != 0 || i->filters.size() != 0 ) // we have a filter or an event { @@ -609,6 +618,7 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { { + bool doBreak = false; // ask filter, if we want to expire this element... std::list::iterator it = i->filters.begin(); while ( it != i->filters.end() ) { @@ -627,13 +637,21 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { Val* v = (*it).pred->Call(&vl); bool result = v->AsBool(); Unref(v); + + ++it; if ( result == false ) { - // throw away. Hence - we quit and simply go to the next entry of lastDict + // Keep it. Hence - we quit and simply go to the next entry of lastDict + // ah well - and we have to add the entry to currDict... + i->currDict->Insert(lastDictIdxKey, i->lastDict->RemoveEntry(lastDictIdxKey)); + doBreak = true; continue; } - ++it; + } + + if ( doBreak ) { + continue; } } @@ -654,6 +672,8 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { //reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); + i->lastDict->Remove(lastDictIdxKey); + delete(ih); } i->lastDict->Clear(); From 2aa0f6da5795aa90c83c9fa1872776737f1e7416 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 4 Nov 2011 14:33:34 -0700 Subject: [PATCH 035/964] beautify script calls, track filters --- scripts/base/frameworks/input/main.bro | 63 +++++++++++++++++++++++++- src/input.bif | 12 ++--- 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 35ca6dfa4e..e0c41f19be 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -20,7 +20,68 @@ export { pred: function(typ: Input::Event, left: any, right: any): bool &optional; ## decision function, that decides if an inserton, update or removal should really be executed }; - + + const no_filter: Filter = [$name=""]; # Sentinel. + + global create_reader: function(id: Log::ID, description: Input::ReaderDescription) : bool; + global remove_reader: function(id: Log::ID) : bool; + global force_update: function(id: Log::ID) : bool; + global add_event: function(id: Log::ID, name: string) : bool; + global remove_event: function(id: Log::ID, name: string) : bool; + global add_filter: function(id: Log::ID, filter: Input::Filter) : bool; + global remove_filter: function(id: Log::ID, name: string) : bool; + global get_filter: function(id: ID, name: string) : Filter; + } @load base/input.bif + + +module Input; + +global filters: table[ID, string] of Filter; + +function create_reader(id: Log::ID, description: Input::ReaderDescription) : bool + { + return __create_reader(id, description); + } + +function remove_reader(id: Log::ID) : bool + { + return __remove_reader(id); + } + +function force_update(id: Log::ID) : bool + { + return __force_update(id); + } + +function add_event(id: Log::ID, name: string) : bool + { + return __add_event(id, name); + } + +function remove_event(id: Log::ID, name: string) : bool + { + return __remove_event(id, name); + } + +function add_filter(id: Log::ID, filter: Input::Filter) : bool + { + filters[id, filter$name] = filter; + return __add_filter(id, filter); + } + +function remove_filter(id: Log::ID, name: string) : bool + { + delete filters[id, name]; + return __remove_filter(id, name); + } + +function get_filter(id: ID, name: string) : Filter + { + if ( [id, name] in filters ) + return filters[id, name]; + + return no_filter; + } diff --git a/src/input.bif b/src/input.bif index 2301482506..c4bf14e3ed 100644 --- a/src/input.bif +++ b/src/input.bif @@ -16,6 +16,12 @@ function Input::__create_reader%(id: Log::ID, description: Input::ReaderDescript return new Val( the_reader != 0, TYPE_BOOL ); %} +function Input::__remove_reader%(id: Log::ID%) : bool + %{ + bool res = input_mgr->RemoveReader(id->AsEnumVal()); + return new Val( res, TYPE_BOOL ); + %} + function Input::__force_update%(id: Log::ID%) : bool %{ bool res = input_mgr->ForceUpdate(id->AsEnumVal()); @@ -34,12 +40,6 @@ function Input::__remove_event%(id: Log::ID, name: string%) : bool return new Val( res, TYPE_BOOL ); %} -function Input::__remove_reader%(id: Log::ID%) : bool - %{ - bool res = input_mgr->RemoveReader(id->AsEnumVal()); - return new Val( res, TYPE_BOOL ); - %} - function Input::__add_filter%(id: Log::ID, filter: Input::Filter%) : bool %{ bool res = input_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); From 1d39eaf32dff51ef779188f3123e3d63a28f71f8 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 4 Nov 2011 15:03:40 -0700 Subject: [PATCH 036/964] small fixes, less leakiness --- scripts/base/frameworks/input/main.bro | 1 - src/InputMgr.cc | 98 +++++++++++--------------- 2 files changed, 41 insertions(+), 58 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index e0c41f19be..6e19452f7a 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -16,7 +16,6 @@ export { name: string; ## descriptive name. for later removal - pred: function(typ: Input::Event, left: any, right: any): bool &optional; ## decision function, that decides if an inserton, update or removal should really be executed }; diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 7ca1e1b485..18b17c7576 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -28,8 +28,14 @@ struct InputMgr::Filter { EnumVal* id; string name; Func* pred; + + ~Filter(); }; +InputMgr::Filter::~Filter() { + Unref(id); +} + struct InputMgr::ReaderInfo { EnumVal* id; EnumVal* type; @@ -45,15 +51,20 @@ struct InputMgr::ReaderInfo { PDict(InputHash)* lastDict; list events; // events we fire when "something" happens - list filters; // events we fire when "something" happens - -// ~ReaderInfo(); + list filters; // filters that can prevent our actions + ~ReaderInfo(); }; -//void InputMgr::~ReaderInfo() { -// -//} +InputMgr::ReaderInfo::~ReaderInfo() { + Unref(type); + Unref(tab); + Unref(itype); + Unref(rtype); + Unref(id); + + delete(reader); +} struct InputReaderDefinition { bro_int_t type; // the type @@ -71,10 +82,8 @@ InputReaderDefinition input_readers[] = { InputMgr::InputMgr() { - //DBG_LOG(DBG_LOGGING, "this has to happen"); } - // create a new input reader object to be used at whomevers leisure lateron. InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) { @@ -163,30 +172,26 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; - info->type = reader; - Ref(reader); + info->type = reader->Ref()->AsEnumVal(); info->num_idx_fields = idxfields; info->num_val_fields = valfields; - info->tab = dst; - Ref(dst); - info->rtype = val; - Ref(val); // we save a pointer of it... I really hope that this wasn't already done anywhere. - info->id = id; - Ref(id); // ditto... - info->itype = idx; - Ref(idx); - readers.push_back(info); + info->tab = dst->Ref()->AsTableVal(); + info->rtype = val->Ref()->AsRecordType(); + info->id = id->Ref()->AsEnumVal(); + info->itype = idx->Ref()->AsRecordType(); info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); + readers.push_back(info); + int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); if ( success == false ) { - RemoveReader(id); + assert( RemoveReader(id) ); return 0; } success = reader_obj->Update(); if ( success == false ) { - RemoveReader(id); + assert ( RemoveReader(id) ); return 0; } @@ -240,12 +245,12 @@ bool InputMgr::RemoveReader(const EnumVal* id) { ReaderInfo *i = 0; for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { - if ( (*s)->id == id ) - { - i = (*s); - readers.erase(s); // remove from vector - break; - } + if ( (*s)->id == id ) + { + i = (*s); + readers.erase(s); // remove from vector + break; + } } if ( i == 0 ) { @@ -254,14 +259,6 @@ bool InputMgr::RemoveReader(const EnumVal* id) { i->reader->Finish(); - - Unref(i->type); - Unref(i->tab); - Unref(i->itype); - Unref(i->rtype); - Unref(i->id); - - delete(i->reader); delete(i); return true; @@ -279,6 +276,8 @@ bool InputMgr::RegisterEvent(const EnumVal* id, string eventName) { return true; } +// remove first event with name eventName +// (though there shouldn't really be several events with the same name... bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { @@ -286,23 +285,18 @@ bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { return false; } - //bool erased = false; - std::list::iterator it = i->events.begin(); while ( it != i->events.end() ) { if ( *it == eventName ) { it = i->events.erase(it); return true; - // erased = true; } else ++it; } - return false; - //return erased; } @@ -335,14 +329,13 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec } return true; - } bool InputMgr::ForceUpdate(const EnumVal* id) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { - reporter->InternalError("Reader not found"); + reporter->Error("Reader not found"); return false; } @@ -352,7 +345,7 @@ bool InputMgr::ForceUpdate(const EnumVal* id) bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { - reporter->InternalError("Reader not found"); + reporter->Error("Reader not found"); return false; } @@ -380,7 +373,7 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { - reporter->InternalError("Reader not found"); + reporter->Error("Reader not found"); return false; } @@ -427,7 +420,6 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo assert ( position == num_fields ); return idxval; - } @@ -538,7 +530,8 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { if ( result == false ) { if ( !updated ) { - // throw away. Hence - we quit. + // throw away. Hence - we quit. And remove the entry from the current dictionary... + delete(i->currDict->RemoveEntry(idxhash)); return; } else { // keep old one @@ -588,9 +581,6 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { ++filter_iterator; } - - - } @@ -670,13 +660,12 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { } - //reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); - i->lastDict->Remove(lastDictIdxKey); + i->lastDict->Remove(lastDictIdxKey); // deletex in next line delete(ih); } - i->lastDict->Clear(); + i->lastDict->Clear(); // should be empty... but... well... who knows... delete(i->lastDict); i->lastDict = i->currDict; @@ -699,11 +688,6 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { } else { RecordVal * r = new RecordVal(i->rtype); - /* if ( i->rtype->NumFields() != (int) i->num_val_fields ) { - reporter->InternalError("Type mismatch"); - return; - } */ - for ( int j = 0; j < i->rtype->NumFields(); j++) { Val* val = 0; From eb1b0b9502521af9e74c7e6285728320311cbcbe Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 8 Nov 2011 06:32:26 -0600 Subject: [PATCH 037/964] Broxygen doc style tweaks. --- doc/scripts/source/_static/broxygen.css | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/scripts/source/_static/broxygen.css b/doc/scripts/source/_static/broxygen.css index b862bc1a20..d9c39a8506 100644 --- a/doc/scripts/source/_static/broxygen.css +++ b/doc/scripts/source/_static/broxygen.css @@ -3,3 +3,14 @@ a:hover text-decoration:none; color:#c24444; } + +div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 +{ +background-color:#ffffff; +border-bottom: 1px solid #aaa; +} + +th.field-name +{ +white-space:nowrap; +} From 5983d44d950d2b93d91c36846668b35fb24d437a Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 8 Nov 2011 15:33:32 -0800 Subject: [PATCH 038/964] read header line in bro logfile format --- src/InputReaderAscii.cc | 19 +++++++++++++++++-- src/InputReaderAscii.h | 3 ++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 39635de407..e4770421c6 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -62,7 +62,7 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const bool InputReaderAscii::ReadHeader() { // try to read the header line... string line; - if ( !getline(*file, line) ) { + if ( !GetLine(line) ) { Error("could not read first line"); return false; } @@ -111,6 +111,21 @@ bool InputReaderAscii::ReadHeader() { return true; } +bool InputReaderAscii::GetLine(string& str) { + while ( getline(*file, str) ) { + if ( str[0] != '#' ) { + return true; + } + + if ( str.compare(0,8, "#fields\t") == 0 ) { + str = str.substr(8); + return true; + } + } + + return false; +} + // read the entire file and send appropriate thingies back to InputMgr bool InputReaderAscii::DoUpdate() { @@ -143,7 +158,7 @@ bool InputReaderAscii::DoUpdate() { //map *newKeyMap = new map(); string line; - while ( getline(*file, line ) ) { + while ( GetLine(line ) ) { // split on tabs istringstream splitstream(line); diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 673a2cdae2..f86cfd0062 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -37,8 +37,9 @@ protected: virtual bool DoUpdate(); private: - bool ReadHeader(); + + bool GetLine(string& str); ifstream* file; string fname; From a92592d08d25eecd76972d6968e75228a85ff803 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 11 Nov 2011 11:18:49 -0600 Subject: [PATCH 039/964] Fix parallel make from top-level to work on more platforms --- Makefile | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index c736ecdbcb..c5f4bf67ce 100644 --- a/Makefile +++ b/Makefile @@ -11,23 +11,11 @@ VERSION_FULL=$(REPO)-`cat VERSION` VERSION_MIN=$(REPO)-`cat VERSION`-minimal HAVE_MODULES=git submodule | grep -v cmake >/dev/null -all: configured - ( cd $(BUILD) && make ) +SUBDIRS = $(BUILD) +$(SUBDIRS):: configured + $(MAKE) -C $@ $(MAKECMDGOALS) -install: configured - ( cd $(BUILD) && make install ) - -install-aux: configured - ( cd $(BUILD) && make install-aux ) - -clean: configured docclean - ( cd $(BUILD) && make clean ) - -doc: configured - ( cd $(BUILD) && make doc ) - -docclean: configured - ( cd $(BUILD) && make docclean ) +all install install-aux doc docclean clean: $(SUBDIRS) dist: @rm -rf $(VERSION_FULL) $(VERSION_FULL).tgz From d750c3ba7494aafece165d43bfd58ae0b61f4019 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 11 Nov 2011 12:39:00 -0600 Subject: [PATCH 040/964] Promote libz and libmagic to required dependencies. --- CMakeLists.txt | 32 ++++++++----------- INSTALL | 10 +++--- config.h.in | 6 ---- doc/quickstart.rst | 15 ++++----- src/CMakeLists.txt | 10 +----- src/ChunkedIO.cc | 4 --- src/ChunkedIO.h | 4 --- src/FileAnalyzer.cc | 8 ----- src/FileAnalyzer.h | 4 --- src/HTTP.cc | 8 ----- src/HTTP.h | 4 --- src/IRC.cc | 5 --- src/RemoteSerializer.cc | 16 ---------- src/ZIP.cc | 3 -- src/ZIP.h | 3 -- src/bro.bif | 4 --- .../base/protocols/http/http-mime-and-md5.bro | 1 - .../base/protocols/irc/dcc-extract.test | 1 - .../base/protocols/smtp/mime-extract.test | 1 - .../scripts/base/protocols/smtp/mime.test | 1 - 20 files changed, 26 insertions(+), 114 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 78e9344a4c..241a5b29d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,6 +53,8 @@ FindRequiredPackage(BISON) FindRequiredPackage(PCAP) FindRequiredPackage(OpenSSL) FindRequiredPackage(BIND) +FindRequiredPackage(LibMagic) +FindRequiredPackage(ZLIB) if (NOT BinPAC_ROOT_DIR AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/aux/binpac/CMakeLists.txt) @@ -72,26 +74,12 @@ include_directories(BEFORE ${OpenSSL_INCLUDE_DIR} ${BIND_INCLUDE_DIR} ${BinPAC_INCLUDE_DIR} + ${LibMagic_INCLUDE_DIR} + ${ZLIB_INCLUDE_DIR} ) # Optional Dependencies -set(HAVE_LIBMAGIC false) -find_package(LibMagic) -if (LIBMAGIC_FOUND) - set(HAVE_LIBMAGIC true) - include_directories(BEFORE ${LibMagic_INCLUDE_DIR}) - list(APPEND OPTLIBS ${LibMagic_LIBRARY}) -endif () - -set(HAVE_LIBZ false) -find_package(ZLIB) -if (ZLIB_FOUND) - set(HAVE_LIBZ true) - include_directories(BEFORE ${ZLIB_INCLUDE_DIR}) - list(APPEND OPTLIBS ${ZLIB_LIBRARY}) -endif () - set(USE_GEOIP false) find_package(LibGeoIP) if (LIBGEOIP_FOUND) @@ -110,6 +98,16 @@ if (ENABLE_PERFTOOLS) endif () endif () +set(brodeps + ${BinPAC_LIBRARY} + ${PCAP_LIBRARY} + ${OpenSSL_LIBRARIES} + ${BIND_LIBRARY} + ${LibMagic_LIBRARY} + ${ZLIB_LIBRARY} + ${OPTLIBS} +) + ######################################################################## ## System Introspection @@ -184,8 +182,6 @@ message( "\nAux. Tools: ${INSTALL_AUX_TOOLS}" "\n" "\nGeoIP: ${USE_GEOIP}" - "\nlibz: ${HAVE_LIBZ}" - "\nlibmagic: ${HAVE_LIBMAGIC}" "\nGoogle perftools: ${USE_PERFTOOLS}" "\n" "\n================================================================\n" diff --git a/INSTALL b/INSTALL index d1351a502f..7dd817ec13 100644 --- a/INSTALL +++ b/INSTALL @@ -14,16 +14,15 @@ before you begin: * OpenSSL (headers and libraries) http://www.openssl.org -Bro can make uses of some optional libraries if they are found at -installation time: - * Libmagic For identifying file types (e.g., in FTP transfers). - * LibGeoIP For geo-locating IP addresses. - * Libz For decompressing HTTP bodies by the HTTP analyzer, and for compressed Bro-to-Bro communication. +Bro can make uses of some optional libraries if they are found at +installation time: + + * LibGeoIP For geo-locating IP addresses. Bro also needs the following tools, but on most systems they will already come preinstalled: @@ -33,7 +32,6 @@ already come preinstalled: * Flex (Fast Lexical Analyzer) * Perl (Used only during the Bro build process) - Installation ============ diff --git a/config.h.in b/config.h.in index 3783a8390d..62aa34ef43 100644 --- a/config.h.in +++ b/config.h.in @@ -14,12 +14,6 @@ /* Define if you have the `getopt_long' function. */ #cmakedefine HAVE_GETOPT_LONG -/* Define if you have the `magic' library (-lmagic). */ -#cmakedefine HAVE_LIBMAGIC - -/* Define if you have the `z' library (-lz). */ -#cmakedefine HAVE_LIBZ - /* We are on a Linux system */ #cmakedefine HAVE_LINUX diff --git a/doc/quickstart.rst b/doc/quickstart.rst index 22523e1618..76a343e97f 100644 --- a/doc/quickstart.rst +++ b/doc/quickstart.rst @@ -60,13 +60,13 @@ Required Dependencies .. console:: - > sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig + > sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel file-devel * DEB/Debian-based Linux: .. console:: - > sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig + > sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev libmagic-dev * FreeBSD @@ -94,21 +94,20 @@ Required Dependencies Optional Dependencies ~~~~~~~~~~~~~~~~~~~~~ -Bro can use libmagic for identifying file types, libGeoIP for geo-locating -IP addresses, libz for (de)compression during analysis and communication, -and sendmail for sending emails. +Bro can use libGeoIP for geo-locating IP addresses and sendmail for +sending emails. * RPM/RedHat-based Linux: .. console:: - > sudo yum install zlib-devel file-devel GeoIP-devel sendmail + > sudo yum install GeoIP-devel sendmail * DEB/Debian-based Linux: .. console:: - > sudo apt-get install zlib1g-dev libmagic-dev libgeoip-dev sendmail + > sudo apt-get install libgeoip-dev sendmail * Ports-based FreeBSD @@ -116,7 +115,7 @@ and sendmail for sending emails. > sudo pkg_add -r GeoIP - libz, libmagic, and sendmail are typically already available. + sendmail is typically already available. * Mac OS X diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b4779e1557..897acc9d37 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -419,15 +419,7 @@ add_definitions(-DBRO_BUILD_PATH="${CMAKE_CURRENT_BINARY_DIR}") add_executable(bro ${bro_SRCS} ${bro_HEADERS}) -set(brolibs - ${BinPAC_LIBRARY} - ${PCAP_LIBRARY} - ${OpenSSL_LIBRARIES} - ${BIND_LIBRARY} - ${OPTLIBS} -) - -target_link_libraries(bro ${brolibs}) +target_link_libraries(bro ${brodeps}) install(TARGETS bro DESTINATION bin) install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${BRO_SCRIPT_INSTALL_PATH}/base) diff --git a/src/ChunkedIO.cc b/src/ChunkedIO.cc index ff84a343c7..f5bcb4b7c1 100644 --- a/src/ChunkedIO.cc +++ b/src/ChunkedIO.cc @@ -1170,8 +1170,6 @@ void ChunkedIOSSL::Stats(char* buffer, int length) ChunkedIO::Stats(buffer + i, length - i); } -#ifdef HAVE_LIBZ - bool CompressedChunkedIO::Init() { zin.zalloc = 0; @@ -1348,5 +1346,3 @@ void CompressedChunkedIO::Stats(char* buffer, int length) io->Stats(buffer + i, length - i); buffer[length-1] = '\0'; } - -#endif /* HAVE_LIBZ */ diff --git a/src/ChunkedIO.h b/src/ChunkedIO.h index ca95f4b40b..56b5656945 100644 --- a/src/ChunkedIO.h +++ b/src/ChunkedIO.h @@ -287,8 +287,6 @@ private: static SSL_CTX* ctx; }; -#ifdef HAVE_LIBZ - #include // Wrapper class around a another ChunkedIO which the (un-)compresses data. @@ -335,6 +333,4 @@ protected: unsigned long uncompressed_bytes_written; }; -#endif /* HAVE_LIBZ */ - #endif diff --git a/src/FileAnalyzer.cc b/src/FileAnalyzer.cc index 672d1e1e09..d4064e8144 100644 --- a/src/FileAnalyzer.cc +++ b/src/FileAnalyzer.cc @@ -3,23 +3,19 @@ #include "FileAnalyzer.h" #include "Reporter.h" -#ifdef HAVE_LIBMAGIC magic_t File_Analyzer::magic = 0; magic_t File_Analyzer::magic_mime = 0; -#endif File_Analyzer::File_Analyzer(Connection* conn) : TCP_ApplicationAnalyzer(AnalyzerTag::File, conn) { buffer_len = 0; -#ifdef HAVE_LIBMAGIC if ( ! magic ) { InitMagic(&magic, MAGIC_NONE); InitMagic(&magic_mime, MAGIC_MIME); } -#endif } void File_Analyzer::DeliverStream(int len, const u_char* data, bool orig) @@ -52,13 +48,11 @@ void File_Analyzer::Identify() const char* descr = 0; const char* mime = 0; -#ifdef HAVE_LIBMAGIC if ( magic ) descr = magic_buffer(magic, buffer, buffer_len); if ( magic_mime ) mime = magic_buffer(magic_mime, buffer, buffer_len); -#endif val_list* vl = new val_list; vl->append(BuildConnVal()); @@ -68,7 +62,6 @@ void File_Analyzer::Identify() ConnectionEvent(file_transferred, vl); } -#ifdef HAVE_LIBMAGIC void File_Analyzer::InitMagic(magic_t* magic, int flags) { *magic = magic_open(flags); @@ -83,4 +76,3 @@ void File_Analyzer::InitMagic(magic_t* magic, int flags) *magic = 0; } } -#endif diff --git a/src/FileAnalyzer.h b/src/FileAnalyzer.h index 8c1890bb85..dcf9d22e8e 100644 --- a/src/FileAnalyzer.h +++ b/src/FileAnalyzer.h @@ -5,9 +5,7 @@ #include "TCP.h" -#ifdef HAVE_LIBMAGIC #include -#endif class File_Analyzer : public TCP_ApplicationAnalyzer { public: @@ -31,12 +29,10 @@ protected: char buffer[BUFFER_SIZE]; int buffer_len; -#ifdef HAVE_LIBMAGIC static void InitMagic(magic_t* magic, int flags); static magic_t magic; static magic_t magic_mime; -#endif }; #endif diff --git a/src/HTTP.cc b/src/HTTP.cc index 71fa1a3dd0..b41933156d 100644 --- a/src/HTTP.cc +++ b/src/HTTP.cc @@ -43,9 +43,7 @@ HTTP_Entity::HTTP_Entity(HTTP_Message *arg_message, MIME_Entity* parent_entity, header_length = 0; deliver_body = (http_entity_data != 0); encoding = IDENTITY; -#ifdef HAVE_LIBZ zip = 0; -#endif } void HTTP_Entity::EndOfData() @@ -53,7 +51,6 @@ void HTTP_Entity::EndOfData() if ( DEBUG_http ) DEBUG_MSG("%.6f: end of data\n", network_time); -#ifdef HAVE_LIBZ if ( zip ) { zip->Done(); @@ -61,7 +58,6 @@ void HTTP_Entity::EndOfData() zip = 0; encoding = IDENTITY; } -#endif if ( body_length ) http_message->MyHTTP_Analyzer()-> @@ -179,7 +175,6 @@ private: void HTTP_Entity::DeliverBody(int len, const char* data, int trailing_CRLF) { -#ifdef HAVE_LIBZ if ( encoding == GZIP || encoding == DEFLATE ) { ZIP_Analyzer::Method method = @@ -198,7 +193,6 @@ void HTTP_Entity::DeliverBody(int len, const char* data, int trailing_CRLF) zip->NextStream(len, (const u_char*) data, false); } else -#endif DeliverBodyClear(len, data, trailing_CRLF); } @@ -450,9 +444,7 @@ void HTTP_Entity::SubmitAllHeaders() // content-length headers or if connection is to be closed afterwards // anyway. else if ( http_message->MyHTTP_Analyzer()->IsConnectionClose () -#ifdef HAVE_LIBZ || encoding == GZIP || encoding == DEFLATE -#endif ) { // FIXME: Using INT_MAX is kind of a hack here. Better diff --git a/src/HTTP.h b/src/HTTP.h index 13b87d219f..00524da20d 100644 --- a/src/HTTP.h +++ b/src/HTTP.h @@ -29,10 +29,8 @@ public: int expect_body); ~HTTP_Entity() { -#ifdef HAVE_LIBZ if ( zip ) { zip->Done(); delete zip; } -#endif } void EndOfData(); @@ -55,9 +53,7 @@ protected: int64_t header_length; int deliver_body; enum { IDENTITY, GZIP, COMPRESS, DEFLATE } encoding; -#ifdef HAVE_LIBZ ZIP_Analyzer* zip; -#endif MIME_Entity* NewChildEntity() { return new HTTP_Entity(http_message, this, 1); } diff --git a/src/IRC.cc b/src/IRC.cc index caf7c492b6..1918300ba2 100644 --- a/src/IRC.cc +++ b/src/IRC.cc @@ -1188,15 +1188,10 @@ void IRC_Analyzer::DeliverStream(int length, const u_char* line, bool orig) if ( orig_status == REGISTERED && resp_status == REGISTERED && orig_zip_status == ACCEPT_ZIP && resp_zip_status == ACCEPT_ZIP ) { -#ifdef HAVE_LIBZ orig_zip_status = ZIP_LOADED; resp_zip_status = ZIP_LOADED; AddSupportAnalyzer(new ZIP_Analyzer(Conn(), true)); AddSupportAnalyzer(new ZIP_Analyzer(Conn(), false)); -#else - reporter->Error("IRC analyzer lacking libz support"); - Remove(); -#endif } return; diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 324da2c92b..a21a7abc60 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -1222,10 +1222,7 @@ bool RemoteSerializer::SendCapabilities(Peer* peer) uint32 caps = 0; -#ifdef HAVE_LIBZ caps |= Peer::COMPRESSION; -#endif - caps |= Peer::PID_64BIT; caps |= Peer::NEW_CACHE_STRATEGY; @@ -2106,11 +2103,9 @@ bool RemoteSerializer::ProcessPhaseDone() bool RemoteSerializer::HandshakeDone(Peer* peer) { -#ifdef HAVE_LIBZ if ( peer->caps & Peer::COMPRESSION && peer->comp_level > 0 ) if ( ! SendToChild(MSG_COMPRESS, peer, 1, peer->comp_level) ) return false; -#endif if ( ! (peer->caps & Peer::PID_64BIT) ) Log(LogInfo, "peer does not support 64bit PIDs; using compatibility mode", peer); @@ -3699,11 +3694,6 @@ bool SocketComm::ProcessListen() bool SocketComm::ProcessParentCompress() { -#ifndef HAVE_LIBZ - InternalError("supposed to enable compression but don't have zlib"); - return false; -#else - assert(parent_args); uint32* args = (uint32*) parent_args->data; @@ -3727,7 +3717,6 @@ bool SocketComm::ProcessParentCompress() Log(fmt("enabling compression (level %d)", level), parent_peer); return true; -#endif } bool SocketComm::ProcessRemoteMessage(SocketComm::Peer* peer) @@ -3847,10 +3836,6 @@ bool SocketComm::ProcessPeerCompress(Peer* peer) { peer->state = MSG_NONE; -#ifndef HAVE_LIBZ - Error("peer compresses although we do not support it", peer); - return false; -#else if ( ! parent_peer->compressor ) { parent_peer->io = new CompressedChunkedIO(parent_peer->io); @@ -3862,7 +3847,6 @@ bool SocketComm::ProcessPeerCompress(Peer* peer) ((CompressedChunkedIO*) peer->io)->EnableDecompression(); Log("enabling decompression", peer); return true; -#endif } bool SocketComm::Connect(Peer* peer) diff --git a/src/ZIP.cc b/src/ZIP.cc index 26095d1f11..0ebe93abe6 100644 --- a/src/ZIP.cc +++ b/src/ZIP.cc @@ -2,8 +2,6 @@ #include "ZIP.h" -#ifdef HAVE_LIBZ - ZIP_Analyzer::ZIP_Analyzer(Connection* conn, bool orig, Method arg_method) : TCP_SupportAnalyzer(AnalyzerTag::Zip, conn, orig) { @@ -89,4 +87,3 @@ void ZIP_Analyzer::DeliverStream(int len, const u_char* data, bool orig) } while ( zip->avail_out == 0 ); } -#endif diff --git a/src/ZIP.h b/src/ZIP.h index ab5d2ce68b..6a8a180d1a 100644 --- a/src/ZIP.h +++ b/src/ZIP.h @@ -5,8 +5,6 @@ #include "config.h" -#ifdef HAVE_LIBZ - #include "zlib.h" #include "TCP.h" @@ -29,4 +27,3 @@ protected: }; #endif -#endif diff --git a/src/bro.bif b/src/bro.bif index 3350de2867..6110d8af1a 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -3272,18 +3272,15 @@ function enable_raw_output%(f: file%): any %} %%{ -#ifdef HAVE_LIBMAGIC extern "C" { #include } -#endif %%} function identify_data%(data: string, return_mime: bool%): string %{ const char* descr = ""; -#ifdef HAVE_LIBMAGIC static magic_t magic_mime = 0; static magic_t magic_descr = 0; @@ -3309,7 +3306,6 @@ function identify_data%(data: string, return_mime: bool%): string } descr = magic_buffer(*magic, data->Bytes(), data->Len()); -#endif return new StringVal(descr); %} diff --git a/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro b/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro index 53b340d174..dd01e62413 100644 --- a/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro +++ b/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro @@ -2,7 +2,6 @@ # will normalize mime types other than the target type to prevent sensitivity # to varying versions of libmagic. -# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h # @TEST-EXEC: bro -r $TRACES/http-pipelined-requests.trace %INPUT > output # @TEST-EXEC: btest-diff http.log diff --git a/testing/btest/scripts/base/protocols/irc/dcc-extract.test b/testing/btest/scripts/base/protocols/irc/dcc-extract.test index 0324a3f28f..b6bf43ac50 100644 --- a/testing/btest/scripts/base/protocols/irc/dcc-extract.test +++ b/testing/btest/scripts/base/protocols/irc/dcc-extract.test @@ -2,7 +2,6 @@ # correctly extracted. The mime type of the file transferred is normalized # to prevent sensitivity to libmagic version being used. -# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h # @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT # @TEST-EXEC: btest-diff irc.log # @TEST-EXEC: btest-diff irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat diff --git a/testing/btest/scripts/base/protocols/smtp/mime-extract.test b/testing/btest/scripts/base/protocols/smtp/mime-extract.test index ba0869f6d9..8dc8bc9e31 100644 --- a/testing/btest/scripts/base/protocols/smtp/mime-extract.test +++ b/testing/btest/scripts/base/protocols/smtp/mime-extract.test @@ -1,4 +1,3 @@ -# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h # @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT # @TEST-EXEC: btest-diff smtp_entities.log # @TEST-EXEC: btest-diff smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat diff --git a/testing/btest/scripts/base/protocols/smtp/mime.test b/testing/btest/scripts/base/protocols/smtp/mime.test index 6d50d5919f..2b80e148ff 100644 --- a/testing/btest/scripts/base/protocols/smtp/mime.test +++ b/testing/btest/scripts/base/protocols/smtp/mime.test @@ -1,7 +1,6 @@ # Checks logging of mime types and md5 calculation. Mime type in the log # is normalized to prevent sensitivity to libmagic version. -# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h # @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT # @TEST-EXEC: btest-diff smtp_entities.log From 5865bf38501fe892c29302916e022dbb855c94fa Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 11 Nov 2011 13:48:11 -0600 Subject: [PATCH 041/964] Add decode_base64_custom BiF to allow alternate base64 alphabets. Addresses #670 --- src/Base64.cc | 69 +++++++++++++++---- src/Base64.h | 10 ++- src/bro.bif | 12 ++++ testing/btest/Baseline/bifs.decode_base64/out | 6 ++ testing/btest/bifs/decode_base64.bro | 14 ++++ 5 files changed, 97 insertions(+), 14 deletions(-) create mode 100644 testing/btest/Baseline/bifs.decode_base64/out create mode 100644 testing/btest/bifs/decode_base64.bro diff --git a/src/Base64.cc b/src/Base64.cc index 9008837f35..080732eab2 100644 --- a/src/Base64.cc +++ b/src/Base64.cc @@ -1,14 +1,27 @@ #include "config.h" #include "Base64.h" -static int base64_table[256]; +int Base64Decoder::default_base64_table[256]; +const string Base64Decoder::default_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -static void init_base64_table() +int* Base64Decoder::InitBase64Table(const string& alphabet) { - static int table_initialized = 0; + static bool default_table_initialized = false; - if ( ++table_initialized > 1 ) - return; + if ( alphabet == default_alphabet && default_table_initialized ) + return default_base64_table; + + int* base64_table = 0; + + if ( alphabet == default_alphabet ) + { + base64_table = default_base64_table; + default_table_initialized = true; + } + else + { + base64_table = new int[256]; + } int i; for ( i = 0; i < 256; ++i ) @@ -16,28 +29,44 @@ static void init_base64_table() for ( i = 0; i < 26; ++i ) { - base64_table['A' + i] = i; - base64_table['a' + i] = i + 26; + base64_table[int(alphabet[0 + i])] = i; + base64_table[int(alphabet[26 + i])] = i + 26; } for ( i = 0; i < 10; ++i ) - base64_table['0' + i] = i + 52; + base64_table[int(alphabet[52 + i])] = i + 52; // Casts to avoid compiler warnings. - base64_table[int('+')] = 62; - base64_table[int('/')] = 63; + base64_table[int(alphabet[62])] = 62; + base64_table[int(alphabet[63])] = 63; base64_table[int('=')] = 0; + + return base64_table; } Base64Decoder::Base64Decoder(Analyzer* arg_analyzer) { - init_base64_table(); + base64_table = InitBase64Table(default_alphabet); base64_group_next = 0; base64_padding = base64_after_padding = 0; errored = 0; analyzer = arg_analyzer; } +Base64Decoder::Base64Decoder(Analyzer* arg_analyzer, const string& alphabet) + { + base64_table = InitBase64Table(alphabet); + base64_group_next = 0; + base64_padding = base64_after_padding = 0; + errored = 0; + analyzer = arg_analyzer; + } + +Base64Decoder::~Base64Decoder() + { + if ( base64_table != default_base64_table ) delete base64_table; + } + int Base64Decoder::Decode(int len, const char* data, int* pblen, char** pbuf) { int blen; @@ -142,13 +171,21 @@ int Base64Decoder::Done(int* pblen, char** pbuf) return 0; } -BroString* decode_base64(const BroString* s) + +BroString* decode_base64(const BroString* s, const BroString* a) { + if (a->Len() != 64) + { + reporter->Error("base64 decoding alphabet is not 64 characters: %s", + a->CheckString()); + return 0; + } + int buf_len = int((s->Len() + 3) / 4) * 3 + 1; int rlen2, rlen = buf_len; char* rbuf2, *rbuf = new char[rlen]; - Base64Decoder dec(0); + Base64Decoder dec(0, a->CheckString()); if ( dec.Decode(s->Len(), (const char*) s->Bytes(), &rlen, &rbuf) == -1 ) goto err; @@ -166,3 +203,9 @@ err: delete [] rbuf; return 0; } + +BroString* decode_base64(const BroString* s) + { + BroString a(Base64Decoder::default_alphabet); + return decode_base64(s, &a); + } diff --git a/src/Base64.h b/src/Base64.h index 7f351a2c2b..518ae469cd 100644 --- a/src/Base64.h +++ b/src/Base64.h @@ -17,7 +17,8 @@ public: // when the decoder is called by the built-in function // decode_base64(). Base64Decoder(Analyzer* analyzer); - ~Base64Decoder() { } + Base64Decoder(Analyzer* analyzer, const string& alphabet); + ~Base64Decoder(); // A note on Decode(): // @@ -47,6 +48,11 @@ public: reporter->Error("%s", msg); } + static int* InitBase64Table(const string& alphabet); + + static const string default_alphabet; + static int default_base64_table[256]; + protected: char error_msg[256]; @@ -57,8 +63,10 @@ protected: int base64_after_padding; int errored; // if true, we encountered an error - skip further processing Analyzer* analyzer; + int* base64_table; }; +BroString* decode_base64(const BroString* s, const BroString* a); BroString* decode_base64(const BroString* s); #endif /* base64_h */ diff --git a/src/bro.bif b/src/bro.bif index 3350de2867..e1d510f279 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1860,6 +1860,18 @@ function decode_base64%(s: string%): string } %} +function decode_base64_custom%(s: string, a: string%): string + %{ + BroString* t = decode_base64(s->AsString(), a->AsString()); + if ( t ) + return new StringVal(t); + else + { + reporter->Error("error in decoding string %s", s->CheckString()); + return new StringVal(""); + } + %} + %%{ #include "DCE_RPC.h" diff --git a/testing/btest/Baseline/bifs.decode_base64/out b/testing/btest/Baseline/bifs.decode_base64/out new file mode 100644 index 0000000000..af0d32fbb8 --- /dev/null +++ b/testing/btest/Baseline/bifs.decode_base64/out @@ -0,0 +1,6 @@ +bro +bro +bro +bro +bro +bro diff --git a/testing/btest/bifs/decode_base64.bro b/testing/btest/bifs/decode_base64.bro new file mode 100644 index 0000000000..d4cbd2f37d --- /dev/null +++ b/testing/btest/bifs/decode_base64.bro @@ -0,0 +1,14 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +global default_alphabet: string = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +global my_alphabet: string = "!#$%&/(),-.:;<>@[]^ `_{|}~abcdefghijklmnopqrstuvwxyz0123456789+?"; + +print decode_base64("YnJv"); +print decode_base64_custom("YnJv", default_alphabet); +print decode_base64_custom("}n-v", my_alphabet); + +print decode_base64("YnJv"); +print decode_base64_custom("YnJv", default_alphabet); +print decode_base64_custom("}n-v", my_alphabet); From b12d2c768e269e2c5494c380c977358228a5c41e Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 14 Nov 2011 15:24:15 -0500 Subject: [PATCH 042/964] Tiny bugfix for http file extraction along with test. --- scripts/base/protocols/http/file-extract.bro | 2 +- ...64.125:56730-125.190.109.199:80_resp_1.dat | 304 ++++++++++++++++++ .../http.log | 5 + .../protocols/http/http-extract-files.bro | 5 + 4 files changed, 315 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat create mode 100644 testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log create mode 100644 testing/btest/scripts/base/protocols/http/http-extract-files.bro diff --git a/scripts/base/protocols/http/file-extract.bro b/scripts/base/protocols/http/file-extract.bro index d36d95e475..4bf036fab8 100644 --- a/scripts/base/protocols/http/file-extract.bro +++ b/scripts/base/protocols/http/file-extract.bro @@ -33,7 +33,7 @@ export { event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=5 { # Client body extraction is not currently supported in this script. - if ( is_orig || ! c$http$first_chunk ) return; + if ( is_orig ) return; if ( c$http$first_chunk ) { diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat new file mode 100644 index 0000000000..73c369dd14 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat @@ -0,0 +1,304 @@ + +ICIR + +ICIR
+

+ICIR (The ICSI Center for Internet Research) +is a +non-profit +research institute at +ICSI +in +Berkeley, +California.
+For the three years from 1999 to 2001 we were named +ACIRI, the AT&T Center for Internet Research at ICSI, +and were funded by AT&T.
+ +The goals of ICIR are to: +

    +
  • Pursue research on the Internet architecture and related networking issues, +
  • +Participate actively in the research (SIGCOMM and IRTF) and +standards (IETF) communities, +
  • Bridge the gap between the Internet research community and commercial +interests by providing a neutral forum where topics of mutual technical +interest can be addressed. +
+

+ +

+ +
+ + + + + + + + + + +
+ +

+People +

+ + +		 + +

+Publications +

+ + +

+Projects +

+ + + +		 + +

Research

+   Transport and Congestion + + +   Traffic and Topology +
    +
  • +IDMaps +(Internet Distance Mapping). +
  • The +Internet Traffic Archive. +
  • +MINC +(Multicast-based Inference of Network-internal Characteristics). +
  • +NIMI +(National Internet Measurement Infrastructure). +
+ +

+ +Collaborators +

+ + + +
+
+ +
+

Information for visitors and local users.

+
+Last modified: June 2004. Copyright notice. + +Older versions of this web page, in its ACIRI incarnation.. +
+For more information about this server, mail www@aciri.org. +
+To report unusual activity by any of our hosts, mail abuse@aciri.org. + diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log new file mode 100644 index 0000000000..386eaf8901 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log @@ -0,0 +1,5 @@ +#separator \x09 +#path http +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file +#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file +1128727435.634189 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat diff --git a/testing/btest/scripts/base/protocols/http/http-extract-files.bro b/testing/btest/scripts/base/protocols/http/http-extract-files.bro new file mode 100644 index 0000000000..4338cddb47 --- /dev/null +++ b/testing/btest/scripts/base/protocols/http/http-extract-files.bro @@ -0,0 +1,5 @@ +# @TEST-EXEC: bro -C -r $TRACES/web.trace %INPUT +# @TEST-EXEC: btest-diff http.log +# @TEST-EXEC: btest-diff http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat + +redef HTTP::extract_file_types += /text\/html/; \ No newline at end of file From 4942767c4d3230d9edfb083f3d4f25c6ed9adc86 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 14 Nov 2011 16:12:38 -0500 Subject: [PATCH 043/964] More default "weird" tuning for the "SYN_with_data" notice. - I think the default tuning should be that anything not requiring a session to be established should use ACTION_LOG_PER_ORIG. - We need to get some tie-in with the metrics framework in place so that we can find when lots of these values are being suppressed. --- scripts/base/frameworks/notice/weird.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base/frameworks/notice/weird.bro b/scripts/base/frameworks/notice/weird.bro index 2303c97fbc..379409532c 100644 --- a/scripts/base/frameworks/notice/weird.bro +++ b/scripts/base/frameworks/notice/weird.bro @@ -174,7 +174,7 @@ export { ["SYN_after_reset"] = ACTION_LOG, ["SYN_inside_connection"] = ACTION_LOG, ["SYN_seq_jump"] = ACTION_LOG, - ["SYN_with_data"] = ACTION_LOG, + ["SYN_with_data"] = ACTION_LOG_PER_ORIG, ["TCP_christmas"] = ACTION_LOG, ["truncated_ARP"] = ACTION_LOG, ["truncated_NTP"] = ACTION_LOG, From d1787523fcb43b5382c0ff5f397b54ac49ba7205 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 14 Nov 2011 15:13:20 -0600 Subject: [PATCH 044/964] Binary packaging script tweaks. - Now requiring CMake 2.8.6 - Make moving of packages into build/ independent of package names. - Bro-all package renamed to Bro and Bro renamed to Bro-minimal which is more similar to source packages now. --- pkg/check-cmake | 2 +- pkg/make-deb-packages | 8 ++++---- pkg/make-mac-packages | 8 ++++---- pkg/make-rpm-packages | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/check-cmake b/pkg/check-cmake index 2c3ed765a6..17531af2f7 100755 --- a/pkg/check-cmake +++ b/pkg/check-cmake @@ -5,7 +5,7 @@ # version of CMake is required to obtain consistency, but can be increased # as new versions of CMake come out that also produce working packages. -CMAKE_PACK_REQ="cmake version 2.8.4" +CMAKE_PACK_REQ="cmake version 2.8.6" CMAKE_VER=`cmake -version` if [ "${CMAKE_VER}" != "${CMAKE_PACK_REQ}" ]; then diff --git a/pkg/make-deb-packages b/pkg/make-deb-packages index a9de210e52..432de8336a 100755 --- a/pkg/make-deb-packages +++ b/pkg/make-deb-packages @@ -27,21 +27,21 @@ cd .. # Minimum Bro ./configure --prefix=${prefix} --disable-broccoli --disable-broctl \ - --pkg-name-prefix=Bro --binary-package + --pkg-name-prefix=Bro-minimal --binary-package ( cd build && make package ) # Full Bro package -./configure --prefix=${prefix} --pkg-name-prefix=Bro-all --binary-package +./configure --prefix=${prefix} --pkg-name-prefix=Bro --binary-package ( cd build && make package ) # Broccoli cd aux/broccoli ./configure --prefix=${prefix} --binary-package -( cd build && make package && mv Broccoli*.deb ../../../build/ ) +( cd build && make package && mv *.deb ../../../build/ ) cd ../.. # Broctl cd aux/broctl ./configure --prefix=${prefix} --binary-package -( cd build && make package && mv Broctl*.deb ../../../build/ ) +( cd build && make package && mv *.deb ../../../build/ ) cd ../.. diff --git a/pkg/make-mac-packages b/pkg/make-mac-packages index a8f7f965c8..829a64ca25 100755 --- a/pkg/make-mac-packages +++ b/pkg/make-mac-packages @@ -35,25 +35,25 @@ cd .. # Minimum Bro CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \ - --disable-broccoli --disable-broctl --pkg-name-prefix=Bro \ + --disable-broccoli --disable-broctl --pkg-name-prefix=Bro-minimal \ --binary-package ( cd build && make package ) # Full Bro package CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \ - --pkg-name-prefix=Bro-all --binary-package + --pkg-name-prefix=Bro --binary-package ( cd build && make package ) # Broccoli cd aux/broccoli CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \ --binary-package -( cd build && make package && mv Broccoli*.dmg ../../../build/ ) +( cd build && make package && mv *.dmg ../../../build/ ) cd ../.. # Broctl cd aux/broctl CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \ --binary-package -( cd build && make package && mv Broctl*.dmg ../../../build/ ) +( cd build && make package && mv *.dmg ../../../build/ ) cd ../.. diff --git a/pkg/make-rpm-packages b/pkg/make-rpm-packages index ac8dfa97b4..9560cc80ff 100755 --- a/pkg/make-rpm-packages +++ b/pkg/make-rpm-packages @@ -20,21 +20,21 @@ cd .. # Minimum Bro ./configure --prefix=${prefix} --disable-broccoli --disable-broctl \ - --pkg-name-prefix=Bro --binary-package + --pkg-name-prefix=Bro-minimal --binary-package ( cd build && make package ) # Full Bro package -./configure --prefix=${prefix} --pkg-name-prefix=Bro-all --binary-package +./configure --prefix=${prefix} --pkg-name-prefix=Bro --binary-package ( cd build && make package ) # Broccoli cd aux/broccoli ./configure --prefix=${prefix} --binary-package -( cd build && make package && mv Broccoli*.rpm ../../../build/ ) +( cd build && make package && mv *.rpm ../../../build/ ) cd ../.. # Broctl cd aux/broctl ./configure --prefix=${prefix} --binary-package -( cd build && make package && mv Broctl*.rpm ../../../build/ ) +( cd build && make package && mv *.rpm ../../../build/ ) cd ../.. From 1a642f3568c3c8c445f76a5b7438f385b880bbe7 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 14 Nov 2011 17:18:28 -0800 Subject: [PATCH 045/964] tried enum support - doesn't yet work due to internal bro interface problems... --- src/InputMgr.cc | 6 ++++++ src/InputReaderAscii.cc | 2 +- src/input.bif | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 18b17c7576..8db5a70c66 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -828,6 +828,7 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals break; case TYPE_STRING: + case TYPE_ENUM: { length += val->val.string_val->size(); break; @@ -883,6 +884,7 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals break; case TYPE_STRING: + case TYPE_ENUM: { memcpy(data+position, val->val.string_val->c_str(), val->val.string_val->length()); position += val->val.string_val->size(); @@ -958,6 +960,10 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { return new SubNetVal(val->val.subnet_val.net, val->val.subnet_val.width); break; + case TYPE_ENUM: + reporter->InternalError("Sorry, Enums reading does not yet work, missing internal inferface"); + + default: reporter->InternalError("unsupported type for input_read"); } diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index e4770421c6..f91c55b666 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -128,7 +128,6 @@ bool InputReaderAscii::GetLine(string& str) { // read the entire file and send appropriate thingies back to InputMgr bool InputReaderAscii::DoUpdate() { - // dirty, fix me. (well, apparently after trying seeking, etc - this is not that bad) if ( file && file->is_open() ) { @@ -198,6 +197,7 @@ bool InputReaderAscii::DoUpdate() { //bzero(val, sizeof(LogVal)); switch ( currMapping.type ) { + case TYPE_ENUM: case TYPE_STRING: val->val.string_val = new string(s); break; diff --git a/src/input.bif b/src/input.bif index c4bf14e3ed..7b051fba16 100644 --- a/src/input.bif +++ b/src/input.bif @@ -51,3 +51,4 @@ function Input::__remove_filter%(id: Log::ID, name: string%) : bool bool res = input_mgr->RemoveFilter(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL); %} + From 21146abda28af68e288963cec3cb9cc9425df118 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 15 Nov 2011 07:56:48 -0800 Subject: [PATCH 046/964] Updating submodule(s). --- aux/broctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broctl b/aux/broctl index 288c8568d7..6fb4e5689d 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 288c8568d7aaa38cf7c05833c133a91cbadbfce4 +Subproject commit 6fb4e5689d2ae0d1c4ab7af0a8df80e6eaa98fb6 From 83aa4b535fb4b977f4b9a1bed8d12a7c8159f2c3 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 15 Nov 2011 10:10:30 -0600 Subject: [PATCH 047/964] Move sphinx source tree up a level. --- doc/CMakeLists.txt | 73 +++++++++++++++++- doc/{scripts => bin}/group_index_generator.py | 0 doc/{scripts => }/conf.py.in | 0 doc/scripts/CMakeLists.txt | 75 +------------------ doc/{scripts => }/source/_static/broxygen.css | 0 doc/{scripts => }/source/_static/showhide.js | 0 .../source/_templates/layout.html | 0 doc/{scripts => }/source/bifs.rst | 0 doc/{scripts => }/source/builtins.rst | 0 doc/{scripts => }/source/common.rst | 0 doc/{scripts => }/source/ext/bro.py | 0 doc/{scripts => }/source/index.rst | 0 doc/{scripts => }/source/internal.rst | 0 doc/{scripts => }/source/packages.rst | 0 doc/{scripts => }/source/scripts/index.rst | 0 15 files changed, 74 insertions(+), 74 deletions(-) rename doc/{scripts => bin}/group_index_generator.py (100%) rename doc/{scripts => }/conf.py.in (100%) rename doc/{scripts => }/source/_static/broxygen.css (100%) rename doc/{scripts => }/source/_static/showhide.js (100%) rename doc/{scripts => }/source/_templates/layout.html (100%) rename doc/{scripts => }/source/bifs.rst (100%) rename doc/{scripts => }/source/builtins.rst (100%) rename doc/{scripts => }/source/common.rst (100%) rename doc/{scripts => }/source/ext/bro.py (100%) rename doc/{scripts => }/source/index.rst (100%) rename doc/{scripts => }/source/internal.rst (100%) rename doc/{scripts => }/source/packages.rst (100%) rename doc/{scripts => }/source/scripts/index.rst (100%) diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index acef059ce8..6567faae76 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -1,4 +1,73 @@ -add_custom_target(doc) -add_custom_target(docclean) +set(BIF_SRC_DIR ${PROJECT_SOURCE_DIR}/src) +set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output) +set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out) +set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source) +set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/source) + +set(MASTER_POLICY_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/policy_index) +set(MASTER_PACKAGE_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/pkg_index) + +file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*") + +# configure the Sphinx config file (expand variables CMake might know about) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in + ${CMAKE_CURRENT_BINARY_DIR}/conf.py + @ONLY) add_subdirectory(scripts) + +# The "broxygen" target generates reST documentation for any outdated bro +# scripts and then uses Sphinx to generate HTML documentation from the reST +add_custom_target(broxygen + # copy the template documentation to the build directory + # to give as input for sphinx + COMMAND "${CMAKE_COMMAND}" -E copy_directory + ${DOC_SOURCE_DIR} + ${DOC_SOURCE_WORKDIR} + # copy generated policy script documentation into the + # working copy of the template documentation + COMMAND "${CMAKE_COMMAND}" -E copy_directory + ${RST_OUTPUT_DIR} + ${DOC_SOURCE_WORKDIR}/scripts + # append to the master index of all policy scripts + COMMAND cat ${MASTER_POLICY_INDEX} >> + ${DOC_SOURCE_WORKDIR}/scripts/index.rst + # append to the master index of all policy packages + COMMAND cat ${MASTER_PACKAGE_INDEX} >> + ${DOC_SOURCE_WORKDIR}/packages.rst + # construct a reST file for each group + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bin/group_index_generator.py + ${CMAKE_CURRENT_BINARY_DIR}/scripts/group_list + ${CMAKE_CURRENT_BINARY_DIR}/scripts + ${DOC_SOURCE_WORKDIR} + # tell sphinx to generate html + COMMAND sphinx-build + -b html + -c ${CMAKE_CURRENT_BINARY_DIR} + -d ${DOC_OUTPUT_DIR}/doctrees + ${DOC_SOURCE_WORKDIR} + ${DOC_OUTPUT_DIR}/html + # create symlink to the html output directory for convenience + COMMAND "${CMAKE_COMMAND}" -E create_symlink + ${DOC_OUTPUT_DIR}/html + ${CMAKE_BINARY_DIR}/html + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "[Sphinx] Generating HTML policy script docs" + # SOURCES just adds stuff to IDE projects as a convenience + SOURCES ${DOC_SOURCES}) + +# The "sphinxclean" target removes just the Sphinx input/output directories +# from the build directory. +add_custom_target(broxygen-clean + COMMAND "${CMAKE_COMMAND}" -E remove_directory + ${DOC_SOURCE_WORKDIR} + COMMAND "${CMAKE_COMMAND}" -E remove_directory + ${DOC_OUTPUT_DIR} + VERBATIM) + +add_dependencies(broxygen broxygen-clean restdoc) + +add_custom_target(doc) +add_custom_target(docclean) +add_dependencies(doc broxygen) +add_dependencies(docclean broxygen-clean restclean) diff --git a/doc/scripts/group_index_generator.py b/doc/bin/group_index_generator.py similarity index 100% rename from doc/scripts/group_index_generator.py rename to doc/bin/group_index_generator.py diff --git a/doc/scripts/conf.py.in b/doc/conf.py.in similarity index 100% rename from doc/scripts/conf.py.in rename to doc/conf.py.in diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt index 1cf2f768ec..377e686e6e 100644 --- a/doc/scripts/CMakeLists.txt +++ b/doc/scripts/CMakeLists.txt @@ -1,16 +1,3 @@ -set(BIF_SRC_DIR ${PROJECT_SOURCE_DIR}/src) -set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output) -set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out) -set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source) -set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/source) - -file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*") - -# configure the Sphinx config file (expand variables CMake might know about) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in - ${CMAKE_CURRENT_BINARY_DIR}/conf.py - @ONLY) - # find out what BROPATH to use when executing bro execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev OUTPUT_VARIABLE BROPATH @@ -48,7 +35,7 @@ endif () # which summary text can be extracted at build time # ${group}_doc_names: a running list of reST style document names that can be # given to a :doc: role, shared indices with ${group}_files -# + macro(REST_TARGET srcDir broInput) set(absSrcPath ${srcDir}/${broInput}) get_filename_component(basename ${broInput} NAME) @@ -143,12 +130,10 @@ endmacro(REST_TARGET) include(DocSourcesList.cmake) # create temporary list of all docs to include in the master policy/index file -set(MASTER_POLICY_INDEX ${CMAKE_CURRENT_BINARY_DIR}/policy_index) file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}") # create the temporary list of all packages to include in the master # policy/packages.rst file -set(MASTER_PACKAGE_INDEX ${CMAKE_CURRENT_BINARY_DIR}/pkg_index) set(MASTER_PKG_INDEX_TEXT "") foreach (pkg ${MASTER_PKG_LIST}) # strip of the trailing /index for the link name @@ -186,11 +171,11 @@ if (EXISTS ${RST_OUTPUT_DIR}) list(FIND ALL_REST_OUTPUTS ${_doc} _found) if (_found EQUAL -1) file(REMOVE ${_doc}) - message(STATUS "AutoDoc: remove stale reST doc: ${_doc}") + message(STATUS "Broxygen: remove stale reST doc: ${_doc}") string(REPLACE .rst .bro _brofile ${_doc}) if (EXISTS ${_brofile}) file(REMOVE ${_brofile}) - message(STATUS "AutoDoc: remove stale bro source: ${_brofile}") + message(STATUS "Broxygen: remove stale bro source: ${_brofile}") endif () endif () endforeach () @@ -211,57 +196,3 @@ add_custom_target(restclean COMMAND "${CMAKE_COMMAND}" -E remove_directory ${RST_OUTPUT_DIR} VERBATIM) - -# The "sphinxdoc" target generates reST documentation for any outdated bro -# scripts and then uses Sphinx to generate HTML documentation from the reST -add_custom_target(sphinxdoc - # copy the template documentation to the build directory - # to give as input for sphinx - COMMAND "${CMAKE_COMMAND}" -E copy_directory - ${DOC_SOURCE_DIR} - ${DOC_SOURCE_WORKDIR} - # copy generated policy script documentation into the - # working copy of the template documentation - COMMAND "${CMAKE_COMMAND}" -E copy_directory - ${RST_OUTPUT_DIR} - ${DOC_SOURCE_WORKDIR}/scripts - # append to the master index of all policy scripts - COMMAND cat ${MASTER_POLICY_INDEX} >> - ${DOC_SOURCE_WORKDIR}/scripts/index.rst - # append to the master index of all policy packages - COMMAND cat ${MASTER_PACKAGE_INDEX} >> - ${DOC_SOURCE_WORKDIR}/packages.rst - # construct a reST file for each group - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/group_index_generator.py - ${CMAKE_CURRENT_BINARY_DIR}/group_list - ${CMAKE_CURRENT_BINARY_DIR} - ${DOC_SOURCE_WORKDIR} - # tell sphinx to generate html - COMMAND sphinx-build - -b html - -c ${CMAKE_CURRENT_BINARY_DIR} - -d ${DOC_OUTPUT_DIR}/doctrees - ${DOC_SOURCE_WORKDIR} - ${DOC_OUTPUT_DIR}/html - # create symlink to the html output directory for convenience - COMMAND "${CMAKE_COMMAND}" -E create_symlink - ${DOC_OUTPUT_DIR}/html - ${CMAKE_BINARY_DIR}/html - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "[Sphinx] Generating HTML policy script docs" - # SOURCES just adds stuff to IDE projects as a convenience - SOURCES ${DOC_SOURCES}) - -# The "sphinxclean" target removes just the Sphinx input/output directories -# from the build directory. -add_custom_target(sphinxclean - COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${DOC_SOURCE_WORKDIR} - COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${DOC_OUTPUT_DIR} - VERBATIM) - -add_dependencies(sphinxdoc sphinxclean restdoc) - -add_dependencies(doc sphinxdoc) -add_dependencies(docclean sphinxclean restclean) diff --git a/doc/scripts/source/_static/broxygen.css b/doc/source/_static/broxygen.css similarity index 100% rename from doc/scripts/source/_static/broxygen.css rename to doc/source/_static/broxygen.css diff --git a/doc/scripts/source/_static/showhide.js b/doc/source/_static/showhide.js similarity index 100% rename from doc/scripts/source/_static/showhide.js rename to doc/source/_static/showhide.js diff --git a/doc/scripts/source/_templates/layout.html b/doc/source/_templates/layout.html similarity index 100% rename from doc/scripts/source/_templates/layout.html rename to doc/source/_templates/layout.html diff --git a/doc/scripts/source/bifs.rst b/doc/source/bifs.rst similarity index 100% rename from doc/scripts/source/bifs.rst rename to doc/source/bifs.rst diff --git a/doc/scripts/source/builtins.rst b/doc/source/builtins.rst similarity index 100% rename from doc/scripts/source/builtins.rst rename to doc/source/builtins.rst diff --git a/doc/scripts/source/common.rst b/doc/source/common.rst similarity index 100% rename from doc/scripts/source/common.rst rename to doc/source/common.rst diff --git a/doc/scripts/source/ext/bro.py b/doc/source/ext/bro.py similarity index 100% rename from doc/scripts/source/ext/bro.py rename to doc/source/ext/bro.py diff --git a/doc/scripts/source/index.rst b/doc/source/index.rst similarity index 100% rename from doc/scripts/source/index.rst rename to doc/source/index.rst diff --git a/doc/scripts/source/internal.rst b/doc/source/internal.rst similarity index 100% rename from doc/scripts/source/internal.rst rename to doc/source/internal.rst diff --git a/doc/scripts/source/packages.rst b/doc/source/packages.rst similarity index 100% rename from doc/scripts/source/packages.rst rename to doc/source/packages.rst diff --git a/doc/scripts/source/scripts/index.rst b/doc/source/scripts/index.rst similarity index 100% rename from doc/scripts/source/scripts/index.rst rename to doc/source/scripts/index.rst From cde8153c1868f6bd3984a2c35aeb645b1023968d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 08:36:03 -0800 Subject: [PATCH 048/964] switch to set if record or simple value is desired. --- scripts/base/frameworks/input/main.bro | 1 + src/InputMgr.cc | 14 ++++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 6e19452f7a..2b87ac980c 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -9,6 +9,7 @@ export { idx: any; val: any; destination: any; + want_record: bool &default=T; reader: Reader &default=default_reader; }; diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 8db5a70c66..5f99583c10 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -42,6 +42,7 @@ struct InputMgr::ReaderInfo { InputReader* reader; unsigned int num_idx_fields; unsigned int num_val_fields; + bool want_record; TableVal* tab; RecordType* rtype; @@ -147,6 +148,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); + Val *want_record = description->Lookup(rtype->FieldOffset("want_record")); vector fieldsV; // vector, because we don't know the length beforehands @@ -181,6 +183,11 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->itype = idx->Ref()->AsRecordType(); info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); + info->want_record = ( want_record->InternalInt() == 1 ); + + if ( valfields > 1 ) { + assert(info->want_record); + } readers.push_back(info); @@ -311,7 +318,6 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec if ( rec->FieldType(i)->Tag() == TYPE_RECORD ) { - string prep = nameprepend + rec->FieldName(i) + "."; if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep) ) @@ -464,7 +470,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { Val* valval; int position = i->num_idx_fields; - if ( i->num_val_fields == 1 ) { + if ( i->num_val_fields == 1 && !i->want_record ) { valval = LogValToVal(vals[i->num_idx_fields]); } else { RecordVal * r = new RecordVal(i->rtype); @@ -683,7 +689,7 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { Val* valval; int position = i->num_idx_fields; - if ( i->num_val_fields == 1 ) { + if ( i->num_val_fields == 1 && !i->want_record ) { valval = LogValToVal(vals[i->num_idx_fields]); } else { RecordVal * r = new RecordVal(i->rtype); @@ -961,7 +967,7 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { break; case TYPE_ENUM: - reporter->InternalError("Sorry, Enums reading does not yet work, missing internal inferface"); + reporter->InternalError("Sorry, Enum reading does not yet work, missing internal inferface"); default: From dacc019f1fa4296dd45a1be2afbbcabcfa930825 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 15 Nov 2011 08:51:48 -0800 Subject: [PATCH 049/964] Adding test for alarm mail. Can't test all the functionality, so skipping DNS lookup and the actual mailing via sendmail. --- .../alarm-mail.txt | 4 ++++ .../base/frameworks/notice/mail-alarms.bro | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.notice.mail-alarms/alarm-mail.txt create mode 100644 testing/btest/scripts/base/frameworks/notice/mail-alarms.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.mail-alarms/alarm-mail.txt b/testing/btest/Baseline/scripts.base.frameworks.notice.mail-alarms/alarm-mail.txt new file mode 100644 index 0000000000..e2cd51edd1 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.mail-alarms/alarm-mail.txt @@ -0,0 +1,4 @@ +> 2005-10-07-23:23:55 Test_Notice 141.42.64.125:56730/tcp -> 125.190.109.199:80/tcp (uid arKYeMETxOg) + test + # 141.42.64.125 = 125.190.109.199 = + diff --git a/testing/btest/scripts/base/frameworks/notice/mail-alarms.bro b/testing/btest/scripts/base/frameworks/notice/mail-alarms.bro new file mode 100644 index 0000000000..3116b1025a --- /dev/null +++ b/testing/btest/scripts/base/frameworks/notice/mail-alarms.bro @@ -0,0 +1,17 @@ +# @TEST-EXEC: bro -C -r $TRACES/web.trace %INPUT +# @TEST-EXEC: btest-diff alarm-mail.txt + +redef Notice::policy += { [$action = Notice::ACTION_ALARM, $priority = 1 ] }; +redef Notice::force_email_summaries = T; + +redef enum Notice::Type += { + Test_Notice, +}; + +event connection_established(c: connection) + { + NOTICE([$note=Test_Notice, $conn=c, $msg="test", $identifier="static"]); + } + + + From fdf01a1ba663230914615858c92c863c9899d205 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 15 Nov 2011 11:52:52 -0600 Subject: [PATCH 050/964] Finished dissolving the sphinx source directory into doc/ and doc/scripts/ --- doc/CMakeLists.txt | 8 +-- doc/{source => }/_static/broxygen.css | 0 doc/{source => }/_static/showhide.js | 0 doc/{source => }/_templates/layout.html | 0 doc/conf.py.in | 6 +- doc/{source => }/ext/bro.py | 0 doc/index.rst | 77 ++++++++++++++----------- doc/scripts/CMakeLists.txt | 2 +- doc/scripts/bifs.rst | 5 ++ doc/{source => scripts}/builtins.rst | 0 doc/{source => scripts}/common.rst | 0 doc/scripts/index.rst | 8 +++ doc/scripts/internal.rst | 5 ++ doc/{source => scripts}/packages.rst | 6 +- doc/source/bifs.rst | 5 -- doc/source/index.rst | 23 -------- doc/source/internal.rst | 5 -- doc/source/scripts/index.rst | 6 -- 18 files changed, 71 insertions(+), 85 deletions(-) rename doc/{source => }/_static/broxygen.css (100%) rename doc/{source => }/_static/showhide.js (100%) rename doc/{source => }/_templates/layout.html (100%) rename doc/{source => }/ext/bro.py (100%) create mode 100644 doc/scripts/bifs.rst rename doc/{source => scripts}/builtins.rst (100%) rename doc/{source => scripts}/common.rst (100%) create mode 100644 doc/scripts/index.rst create mode 100644 doc/scripts/internal.rst rename doc/{source => scripts}/packages.rst (78%) delete mode 100644 doc/source/bifs.rst delete mode 100644 doc/source/index.rst delete mode 100644 doc/source/internal.rst delete mode 100644 doc/source/scripts/index.rst diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 6567faae76..7685230980 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -1,8 +1,8 @@ set(BIF_SRC_DIR ${PROJECT_SOURCE_DIR}/src) set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output) set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out) -set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source) -set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/source) +set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx-sources) set(MASTER_POLICY_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/policy_index) set(MASTER_PACKAGE_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/pkg_index) @@ -34,12 +34,12 @@ add_custom_target(broxygen ${DOC_SOURCE_WORKDIR}/scripts/index.rst # append to the master index of all policy packages COMMAND cat ${MASTER_PACKAGE_INDEX} >> - ${DOC_SOURCE_WORKDIR}/packages.rst + ${DOC_SOURCE_WORKDIR}/scripts/packages.rst # construct a reST file for each group COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bin/group_index_generator.py ${CMAKE_CURRENT_BINARY_DIR}/scripts/group_list ${CMAKE_CURRENT_BINARY_DIR}/scripts - ${DOC_SOURCE_WORKDIR} + ${DOC_SOURCE_WORKDIR}/scripts # tell sphinx to generate html COMMAND sphinx-build -b html diff --git a/doc/source/_static/broxygen.css b/doc/_static/broxygen.css similarity index 100% rename from doc/source/_static/broxygen.css rename to doc/_static/broxygen.css diff --git a/doc/source/_static/showhide.js b/doc/_static/showhide.js similarity index 100% rename from doc/source/_static/showhide.js rename to doc/_static/showhide.js diff --git a/doc/source/_templates/layout.html b/doc/_templates/layout.html similarity index 100% rename from doc/source/_templates/layout.html rename to doc/_templates/layout.html diff --git a/doc/conf.py.in b/doc/conf.py.in index cee16b9239..34e92cd1f1 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -15,7 +15,7 @@ import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('source/ext')) +sys.path.insert(0, os.path.abspath('sphinx-sources/ext')) # -- General configuration ----------------------------------------------------- @@ -27,7 +27,7 @@ sys.path.insert(0, os.path.abspath('source/ext')) extensions = ['bro'] # Add any paths that contain templates here, relative to this directory. -templates_path = ['source/_templates', 'source/_static'] +templates_path = ['sphinx-sources/_templates', 'sphinx-sources/_static'] # The suffix of source filenames. source_suffix = '.rst' @@ -143,7 +143,7 @@ html_theme_options = { # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['source/_static'] +html_static_path = ['sphinx-sources/_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. diff --git a/doc/source/ext/bro.py b/doc/ext/bro.py similarity index 100% rename from doc/source/ext/bro.py rename to doc/ext/bro.py diff --git a/doc/index.rst b/doc/index.rst index c7c85bd21f..56a9e06fe8 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,50 +1,57 @@ - +.. Bro documentation master file Bro Documentation ================= -`Getting Started <{{git('bro:doc/quickstart.rst')}}>`_ - A quick introduction into using Bro 2.x. +.. toctree:: + :maxdepth: 1 -`Bro 1.5 to 2.0 Upgrade Guide <{{git('bro:doc/upgrade.rst')}}>`_ - Guidelines and notes about upgrading from Bro 1.5 to 2.x. Lots of - things have changed, so make sure to read this when upgrading. - -`BroControl <{{git('broctl:doc/broctl.rst')}}>`_ - An interactive console for managing Bro installations. - -`Script Reference <{{autodoc_bro_scripts}}/index.html>`_ - A complete reference of all policy scripts shipped with Bro. - -`FAQ <{{docroot}}/documentation/faq.html>`_ - A list with frequently asked questions. - -`How to Report a Problem <{{docroot}}/documentation/reporting-problems.html>`_ - Some advice for when you see Bro doing something you believe it - shouldn't. + quickstart + upgrade + FAQ + Reporting Problems Frameworks ---------- -Bro comes with a number of frameworks, some of which are described in -more detail here: +.. toctree:: + :maxdepth: 1 -`Notice <{{git('bro:doc/notice.rst')}}>`_ - The notice framework. - -`Logging <{{git('bro:doc/logging.rst')}}>`_ - Customizing and extensing Bro's logging. - -`Cluster <{{git('bro:doc/cluster.rst')}}>`_ - Setting up a Bro Cluster when a single box can't handle the traffic anymore. - -`Signatures <{{git('bro:doc/signatures.rst')}}>`_ - Bro has support for traditional NIDS signatures as well. + notice + logging + cluster + signatures How-Tos ------- -We also collect more specific How-Tos on specific topics: +.. toctree:: + :maxdepth: 1 -`Using GeoIP in Bro scripts <{{git('bro:doc/geoip.rst')}}>`_ - Installation and usage of the the GeoIP library. + geoip + +Script Reference +---------------- + +.. toctree:: + :maxdepth: 1 + + scripts/common + scripts/builtins + scripts/bifs + scripts/packages + scripts/index + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`search` + +Internal References +------------------- + +.. toctree:: + :maxdepth: 1 + + scripts/internal diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt index 377e686e6e..7b47424c6c 100644 --- a/doc/scripts/CMakeLists.txt +++ b/doc/scripts/CMakeLists.txt @@ -73,7 +73,7 @@ macro(REST_TARGET srcDir broInput) elseif (${extension} STREQUAL ".bif.bro") set(group bifs) elseif (relDstDir) - set(pkgIndex scripts/${relDstDir}/index) + set(pkgIndex ${relDstDir}/index) set(group ${pkgIndex}) # add package index to master package list if not already in it list(FIND MASTER_PKG_LIST ${pkgIndex} _found) diff --git a/doc/scripts/bifs.rst b/doc/scripts/bifs.rst new file mode 100644 index 0000000000..eaae0e13b8 --- /dev/null +++ b/doc/scripts/bifs.rst @@ -0,0 +1,5 @@ +.. This is a stub doc to which broxygen appends during the build process + +Built-In Functions (BIFs) +========================= + diff --git a/doc/source/builtins.rst b/doc/scripts/builtins.rst similarity index 100% rename from doc/source/builtins.rst rename to doc/scripts/builtins.rst diff --git a/doc/source/common.rst b/doc/scripts/common.rst similarity index 100% rename from doc/source/common.rst rename to doc/scripts/common.rst diff --git a/doc/scripts/index.rst b/doc/scripts/index.rst new file mode 100644 index 0000000000..5de203afe7 --- /dev/null +++ b/doc/scripts/index.rst @@ -0,0 +1,8 @@ +.. This is a stub doc to which broxygen appends during the build process + +Index of All Bro Scripts +======================== + +.. toctree:: + :maxdepth: 1 + diff --git a/doc/scripts/internal.rst b/doc/scripts/internal.rst new file mode 100644 index 0000000000..a6c10f1cfb --- /dev/null +++ b/doc/scripts/internal.rst @@ -0,0 +1,5 @@ +.. This is a stub doc to which broxygen appends during the build process + +Internal Scripts +================ + diff --git a/doc/source/packages.rst b/doc/scripts/packages.rst similarity index 78% rename from doc/source/packages.rst rename to doc/scripts/packages.rst index 35f6633fd4..47e974b0c8 100644 --- a/doc/source/packages.rst +++ b/doc/scripts/packages.rst @@ -1,7 +1,7 @@ -.. This is a stub doc to which the build process can append. +.. This is a stub doc to which broxygen appends during the build process -Bro Script Packages -=================== +Index of All Bro Script Packages +================================ Bro has the following script packages (e.g. collections of related scripts in a common directory). If the package directory contains a ``__load__.bro`` diff --git a/doc/source/bifs.rst b/doc/source/bifs.rst deleted file mode 100644 index 6a42cafafc..0000000000 --- a/doc/source/bifs.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. This is a stub doc to which the build process can append. - -Built-In Functions (BIFs) -========================= - diff --git a/doc/source/index.rst b/doc/source/index.rst deleted file mode 100644 index a144644208..0000000000 --- a/doc/source/index.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. Bro documentation master file - -Welcome to Bro's documentation! -=============================== - -Contents: - -.. toctree:: - :maxdepth: 1 - :glob: - - common - builtins - internal - bifs - packages - scripts/index - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`search` diff --git a/doc/source/internal.rst b/doc/source/internal.rst deleted file mode 100644 index 817c76e725..0000000000 --- a/doc/source/internal.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. This is a stub doc to which the build process can append. - -Internal Scripts -================ - diff --git a/doc/source/scripts/index.rst b/doc/source/scripts/index.rst deleted file mode 100644 index fd11a3924e..0000000000 --- a/doc/source/scripts/index.rst +++ /dev/null @@ -1,6 +0,0 @@ -Index of All Bro Script Documentation -===================================== - -.. toctree:: - :maxdepth: 1 - From 4a3c9923253c8c26973a24c537a3ca231961db5e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 10:57:45 -0800 Subject: [PATCH 051/964] InputReader can read Sets. --- src/InputMgr.cc | 244 +++++++++++++++++++++++++--------------- src/InputMgr.h | 2 + src/InputReaderAscii.cc | 211 ++++++++++++++++++++++------------ src/InputReaderAscii.h | 5 + src/LogMgr.cc | 6 +- src/LogMgr.h | 4 +- 6 files changed, 310 insertions(+), 162 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5f99583c10..ba17d8448f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -234,11 +234,14 @@ bool InputMgr::IsCompatibleType(BroType* t) case TYPE_TABLE: - return false; + { + return IsCompatibleType(t->AsSetType()->Indices()->PureType()); + } case TYPE_VECTOR: { - return IsCompatibleType(t->AsVectorType()->YieldType()); + return false; // do me... + //return IsCompatibleType(t->AsVectorType()->YieldType()); } default: @@ -329,6 +332,9 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec LogField* field = new LogField(); field->name = nameprepend + rec->FieldName(i); field->type = rec->FieldType(i)->Tag(); + if ( field->type == TYPE_TABLE ) { + field->set_type = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); + } fields->push_back(field); } @@ -810,49 +816,133 @@ Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_ } + +int InputMgr::GetLogValLength(const LogVal* val) { + int length = 0; + + switch (val->type) { + case TYPE_BOOL: + case TYPE_INT: + length += sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + length += sizeof(val->val.uint_val); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + length += sizeof(val->val.double_val); + break; + + case TYPE_STRING: + case TYPE_ENUM: + { + length += val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + length += NUM_ADDR_WORDS*sizeof(uint32_t); + break; + + case TYPE_SUBNET: + length += sizeof(val->val.subnet_val.width); + length += sizeof(val->val.subnet_val.net); + break; + + case TYPE_TABLE: { + for ( int i = 0; i < val->val.set_val.size; i++ ) { + length += GetLogValLength(val->val.set_val.vals[i]); + } + break; + } + + default: + reporter->InternalError("unsupported type %d for GetLogValLength", val->type); + } + + return length; + +} + +int InputMgr::CopyLogVal(char *data, const int startpos, const LogVal* val) { + switch ( val->type ) { + case TYPE_BOOL: + case TYPE_INT: + //reporter->Error("Adding field content to pos %d: %lld", val->val.int_val, startpos); + memcpy(data+startpos, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); + //*(data+startpos) = val->val.int_val; + return sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + //*(data+startpos) = val->val.uint_val; + memcpy(data+startpos, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); + return sizeof(val->val.uint_val); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + //*(data+startpos) = val->val.double_val; + memcpy(data+startpos, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); + return sizeof(val->val.double_val); + break; + + case TYPE_STRING: + case TYPE_ENUM: + { + memcpy(data+startpos, val->val.string_val->c_str(), val->val.string_val->length()); + return val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + memcpy(data+startpos, val->val.addr_val, NUM_ADDR_WORDS*sizeof(uint32_t)); + return NUM_ADDR_WORDS*sizeof(uint32_t); + break; + + case TYPE_SUBNET: { + int length = 0; + memcpy(data+startpos,(const char*) &(val->val.subnet_val.width), sizeof(val->val.subnet_val.width) ); + length += sizeof(val->val.subnet_val.width); + memcpy(data+startpos, (const char*) &(val->val.subnet_val.net), sizeof(val->val.subnet_val.net) ); + length += sizeof(val->val.subnet_val.net); + return length; + break; + } + + case TYPE_TABLE: { + int length = 0; + for ( int i = 0; i < val->val.set_val.size; i++ ) { + length += CopyLogVal(data, startpos+length, val->val.set_val.vals[i]); + } + return length; + break; + } + + default: + reporter->InternalError("unsupported type %d for CopyLogVal", val->type); + return 0; + } + + reporter->InternalError("internal error"); + return 0; + +} + HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals) { int length = 0; for ( int i = 0; i < num_elements; i++ ) { const LogVal* val = vals[i]; - switch (val->type) { - case TYPE_BOOL: - case TYPE_INT: - length += sizeof(val->val.int_val); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - length += sizeof(val->val.uint_val); - break; - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - length += sizeof(val->val.double_val); - break; - - case TYPE_STRING: - case TYPE_ENUM: - { - length += val->val.string_val->size(); - break; - } - - case TYPE_ADDR: - length += NUM_ADDR_WORDS*sizeof(uint32_t); - break; - - case TYPE_SUBNET: - length += sizeof(val->val.subnet_val.width); - length += sizeof(val->val.subnet_val.net); - break; - - default: - reporter->InternalError("unsupported type for hashlogvals"); - } - + length += GetLogValLength(val); } //reporter->Error("Length: %d", length); @@ -864,56 +954,7 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals } for ( int i = 0; i < num_elements; i++ ) { const LogVal* val = vals[i]; - switch ( val->type ) { - case TYPE_BOOL: - case TYPE_INT: - //reporter->Error("Adding field content to pos %d: %lld", val->val.int_val, position); - memcpy(data+position, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); - //*(data+position) = val->val.int_val; - position += sizeof(val->val.int_val); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - //*(data+position) = val->val.uint_val; - memcpy(data+position, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); - position += sizeof(val->val.uint_val); - break; - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - //*(data+position) = val->val.double_val; - memcpy(data+position, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); - position += sizeof(val->val.double_val); - break; - - case TYPE_STRING: - case TYPE_ENUM: - { - memcpy(data+position, val->val.string_val->c_str(), val->val.string_val->length()); - position += val->val.string_val->size(); - break; - } - - case TYPE_ADDR: - memcpy(data+position, val->val.addr_val, NUM_ADDR_WORDS*sizeof(uint32_t)); - position += NUM_ADDR_WORDS*sizeof(uint32_t); - break; - - case TYPE_SUBNET: - memcpy(data+position,(const char*) &(val->val.subnet_val.width), sizeof(val->val.subnet_val.width) ); - position += sizeof(val->val.subnet_val.width); - memcpy(data+position, (const char*) &(val->val.subnet_val.net), sizeof(val->val.subnet_val.net) ); - position += sizeof(val->val.subnet_val.net); - break; - - default: - reporter->InternalError("unsupported type for hashlogvals2"); - } - - + position += CopyLogVal(data, position, val); } assert(position == length); @@ -966,6 +1007,29 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { return new SubNetVal(val->val.subnet_val.net, val->val.subnet_val.width); break; + case TYPE_TABLE: { + if ( val->val.set_val.size == 0 ) { + // empty table + TypeList* set_index = new TypeList(base_type(TYPE_ANY)); + // iim quite sure this does not work... we probably need the internal set type for this... + reporter->InternalError("Implement me."); + return new TableVal(new SetType(set_index, 0)); + } else { + // all entries have to have the same type... + TypeTag type = val->val.set_val.vals[0]->type; + TypeList* set_index = new TypeList(base_type(type)); + set_index->Append(base_type(type)); + SetType* s = new SetType(set_index, 0); + TableVal* t = new TableVal(s); + for ( int i = 0; i < val->val.set_val.size; i++ ) { + assert( val->val.set_val.vals[i]->type == type); + t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); + } + return t; + } + break; + } + case TYPE_ENUM: reporter->InternalError("Sorry, Enum reading does not yet work, missing internal inferface"); diff --git a/src/InputMgr.h b/src/InputMgr.h index 74914e65ad..d4bfa5c355 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -52,6 +52,8 @@ private: void SendEvent(const string& name, EnumVal* event, Val* left, Val* right); HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); + int GetLogValLength(const LogVal* val); + int CopyLogVal(char *data, const int startpos, const LogVal* val); Val* LogValToVal(const LogVal* val, TypeTag request_type = TYPE_ANY); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index f91c55b666..60a8c5685a 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -11,12 +11,22 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int position = arg_position; } +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_set_type, int arg_position) + : name(arg_name), type(arg_type), set_type(arg_set_type) +{ + position = arg_position; +} + FieldMapping::FieldMapping(const FieldMapping& arg) - : name(arg.name), type(arg.type) + : name(arg.name), type(arg.type), set_type(arg.set_type) { position = arg.position; } +FieldMapping FieldMapping::setType() { + return FieldMapping(name, set_type, position); +} + InputReaderAscii::InputReaderAscii() { //DBG_LOG(DBG_LOGGING, "input reader initialized"); @@ -81,7 +91,7 @@ bool InputReaderAscii::ReadHeader() { const LogField* field = fields[i]; if ( field->name == s ) { // cool, found field. note position - FieldMapping f(field->name, field->type, i); + FieldMapping f(field->name, field->type, field->set_type, i); columnMap.push_back(f); wantFields++; break; // done with searching @@ -126,6 +136,132 @@ bool InputReaderAscii::GetLine(string& str) { return false; } + +LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { + + LogVal* val = new LogVal(field.type, true); + //bzero(val, sizeof(LogVal)); + + switch ( field.type ) { + case TYPE_ENUM: + case TYPE_STRING: + val->val.string_val = new string(s); + break; + + case TYPE_BOOL: + if ( s == "T" ) { + val->val.int_val = 1; + } else if ( s == "F" ) { + val->val.int_val = 0; + } else { + Error(Fmt("Invalid value for boolean: %s", s.c_str())); + return false; + } + break; + + case TYPE_INT: + val->val.int_val = atoi(s.c_str()); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + val->val.double_val = atof(s.c_str()); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + val->val.uint_val = atoi(s.c_str()); + break; + + case TYPE_SUBNET: { + int pos = s.find("/"); + string width = s.substr(pos+1); + val->val.subnet_val.width = atoi(width.c_str()); + string addr = s.substr(0, pos); + s = addr; + // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // Solve this some other time.... + val->val.subnet_val.net = dotted_to_addr(s.c_str()); + break; + + } + case TYPE_ADDR: { + // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // Solve this some other time.... + addr_type t = dotted_to_addr(s.c_str()); +#ifdef BROv6 + copy_addr(t, val->val.addr_val); +#else + copy_addr(&t, val->val.addr_val); +#endif + break; + } + + case TYPE_TABLE: { + // construct a table from entry... + // for the moment assume, that entries are split by ",". + + if ( s == "-" ) { + // empty + val->val.set_val.size = 0; + break; + } + + // how many entries do we have... + unsigned int length = 1; + for ( unsigned int i = 0; i < s.size(); i++ ) + if ( s[i] == ',') length++; + + unsigned int pos = 0; + LogVal** lvals = new LogVal* [length]; + val->val.set_val.vals = lvals; + val->val.set_val.size = length; + + istringstream splitstream(s); + while ( splitstream ) { + string element; + + if ( pos >= length ) { + Error(Fmt("Internal error while parsing set. pos %d > length %d", pos, length)); + break; + } + + if ( !getline(splitstream, element, ',') ) + break; + + + LogVal* newval = EntryToVal(element, field.setType()); + if ( newval == 0 ) { + Error("Error while reading set"); + return 0; + } + lvals[pos] = newval; + + pos++; + + } + + if ( pos != length ) { + Error("Internal error while parsing set: did not find all elements"); + return 0; + } + + break; + } + + + default: + Error(Fmt("unsupported field format %d for %s", field.type, + field.name.c_str())); + return 0; + } + + return val; + +} + // read the entire file and send appropriate thingies back to InputMgr bool InputReaderAscii::DoUpdate() { @@ -161,7 +297,6 @@ bool InputReaderAscii::DoUpdate() { // split on tabs istringstream splitstream(line); - string s; LogVal** fields = new LogVal*[num_fields]; //string string_fields[num_fields]; @@ -170,6 +305,7 @@ bool InputReaderAscii::DoUpdate() { unsigned int currField = 0; while ( splitstream ) { + string s; if ( !getline(splitstream, s, '\t') ) break; @@ -193,73 +329,10 @@ bool InputReaderAscii::DoUpdate() { return false; } - LogVal* val = new LogVal(currMapping.type, true); - //bzero(val, sizeof(LogVal)); - - switch ( currMapping.type ) { - case TYPE_ENUM: - case TYPE_STRING: - val->val.string_val = new string(s); - break; - - case TYPE_BOOL: - if ( s == "T" ) { - val->val.int_val = 1; - } else if ( s == "F" ) { - val->val.int_val = 0; - } else { - Error(Fmt("Invalid value for boolean: %s", s.c_str())); - return false; - } - break; - - case TYPE_INT: - val->val.int_val = atoi(s.c_str()); - break; - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - val->val.double_val = atof(s.c_str()); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - val->val.uint_val = atoi(s.c_str()); - break; - - case TYPE_SUBNET: { - int pos = s.find("/"); - string width = s.substr(pos+1); - val->val.subnet_val.width = atoi(width.c_str()); - string addr = s.substr(0, pos); - s = addr; - // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. - // Solve this some other time.... - val->val.subnet_val.net = dotted_to_addr(s.c_str()); - break; - - } - case TYPE_ADDR: { - // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. - // Solve this some other time.... - addr_type t = dotted_to_addr(s.c_str()); -#ifdef BROv6 - copy_addr(t, val->val.addr_val); -#else - copy_addr(&t, val->val.addr_val); -#endif - break; - } - - - default: - Error(Fmt("unsupported field format %d for %s", currMapping.type, - currMapping.name.c_str())); + LogVal* val = EntryToVal(s, currMapping); + if ( val == 0 ) { return false; - } - + } fields[currMapping.position] = val; //string_fields[currMapping.position] = s; diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index f86cfd0062..ab2b89339c 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -12,11 +12,15 @@ struct FieldMapping { string name; TypeTag type; + TypeTag set_type; int position; FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); + FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_set_type, int arg_position); FieldMapping(const FieldMapping& arg); FieldMapping() { position = -1; } + + FieldMapping setType(); bool IsEmpty() { return position == -1; } }; @@ -38,6 +42,7 @@ protected: private: bool ReadHeader(); + LogVal* EntryToVal(string s, FieldMapping type); bool GetLine(string& str); diff --git a/src/LogMgr.cc b/src/LogMgr.cc index f78a2a19e0..9818d9cdfb 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -81,16 +81,18 @@ struct LogMgr::Stream { bool LogField::Read(SerializationFormat* fmt) { int t; + int it; - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type")); + bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&it, "set_type") ); type = (TypeTag) t; + set_type = (TypeTag) it; return success; } bool LogField::Write(SerializationFormat* fmt) const { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type")); + return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)set_type, "set_type")); } LogVal::~LogVal() diff --git a/src/LogMgr.h b/src/LogMgr.h index 10530960cb..40dab8677b 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -15,10 +15,12 @@ class SerializationFormat; struct LogField { string name; TypeTag type; + // needed by input framework. otherwise it cannot determine the inner type of a set. + TypeTag set_type; LogField() { } LogField(const LogField& other) - : name(other.name), type(other.type) { } + : name(other.name), type(other.type), set_type(other.set_type) { } // (Un-)serialize. bool Read(SerializationFormat* fmt); From 3d0162bcdc2e2a3e4923ac22ec5597cb4d167304 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 11:18:48 -0800 Subject: [PATCH 052/964] isCompatibleType works correctly for tables. --- src/InputMgr.cc | 27 +++++++++++++++++---------- src/InputMgr.h | 2 +- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index ba17d8448f..4506501c94 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -205,7 +205,8 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) return reader_obj; } -bool InputMgr::IsCompatibleType(BroType* t) + +bool InputMgr::IsCompatibleType(BroType* t, bool atomic_only) { if ( ! t ) return false; @@ -223,24 +224,29 @@ bool InputMgr::IsCompatibleType(BroType* t) case TYPE_INTERVAL: case TYPE_ENUM: case TYPE_STRING: - case TYPE_RECORD: - // for record: check, if all elements are compatible? But... LogMgr also doesn't do this. - // ^ recursive checking is done in UnrollRecordType. return true; - - case TYPE_FILE: - case TYPE_FUNC: - return false; + case TYPE_RECORD: + return ! atomic_only; case TYPE_TABLE: { - return IsCompatibleType(t->AsSetType()->Indices()->PureType()); + if ( atomic_only ) + return false; + + if ( ! t->IsSet() ) + return false; + + return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true); } case TYPE_VECTOR: { return false; // do me... + + //if ( atomic_only ) + // return false; + // //return IsCompatibleType(t->AsVectorType()->YieldType()); } @@ -249,7 +255,8 @@ bool InputMgr::IsCompatibleType(BroType* t) } return false; - } +} + bool InputMgr::RemoveReader(const EnumVal* id) { ReaderInfo *i = 0; diff --git a/src/InputMgr.h b/src/InputMgr.h index d4bfa5c355..93c6447467 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -46,7 +46,7 @@ protected: private: struct ReaderInfo; - bool IsCompatibleType(BroType* t); + bool IsCompatibleType(BroType* t, bool atomic_only=false); bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); void SendEvent(const string& name, EnumVal* event, Val* left, Val* right); From 5e8dd4f06a74ca240a7b4ac903fde2a7cc486328 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 12:07:31 -0800 Subject: [PATCH 053/964] Fix small bug: sets that are written to logfiles may only contain atomic types. (trying to write sets of records, etc. leads to a bro segfault) --- src/LogMgr.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index f78a2a19e0..ab1706ca72 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -149,7 +149,7 @@ bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) if ( ! t->IsSet() ) return false; - return IsCompatibleType(t->AsSetType()->Indices()->PureType()); + return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true); } case TYPE_VECTOR: From ea2c89f2d72797d6c03c083cede5fec1abe70562 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 15 Nov 2011 15:39:31 -0600 Subject: [PATCH 054/964] Adapt more www niceties to broxygen (stylesheets, pygments); add more docs --- doc/INSTALL.rst | 1 + doc/README | 39 +++-- doc/_templates/layout.html | 5 + doc/cluster.rst | 4 +- doc/components/broccoli-python/README.rst | 1 + doc/components/broccoli/README.rst | 1 + doc/components/broctl/README.rst | 1 + doc/components/btest/README.rst | 1 + doc/components/capstats/README.rst | 1 + doc/components/pysubnettree/README.rst | 1 + doc/components/trace-summary/README.rst | 1 + doc/conf.py.in | 2 +- doc/ext/bro_lexer/__init__.py | 0 doc/ext/bro_lexer/__init__.pyc | Bin 0 -> 147 bytes doc/ext/bro_lexer/bro.py | 76 +++++++++ doc/ext/bro_lexer/bro.pyc | Bin 0 -> 2702 bytes doc/ext/rst_directive.py | 180 ++++++++++++++++++++ doc/faq.rst | 145 ++++++++++++++++ doc/geoip.rst | 2 +- doc/{ => images}/deployment.png | Bin doc/index.rst | 19 ++- doc/logging.rst | 2 +- doc/notice.rst | 2 +- doc/quickstart.rst | 2 +- doc/reporting-problems.rst | 194 ++++++++++++++++++++++ doc/scripts/CMakeLists.txt | 1 + doc/scripts/README | 31 +--- doc/signatures.rst | 2 +- doc/upgrade.rst | 2 +- 29 files changed, 659 insertions(+), 57 deletions(-) create mode 120000 doc/INSTALL.rst create mode 120000 doc/components/broccoli-python/README.rst create mode 120000 doc/components/broccoli/README.rst create mode 120000 doc/components/broctl/README.rst create mode 120000 doc/components/btest/README.rst create mode 120000 doc/components/capstats/README.rst create mode 120000 doc/components/pysubnettree/README.rst create mode 120000 doc/components/trace-summary/README.rst create mode 100644 doc/ext/bro_lexer/__init__.py create mode 100644 doc/ext/bro_lexer/__init__.pyc create mode 100644 doc/ext/bro_lexer/bro.py create mode 100644 doc/ext/bro_lexer/bro.pyc create mode 100644 doc/ext/rst_directive.py create mode 100644 doc/faq.rst rename doc/{ => images}/deployment.png (100%) create mode 100644 doc/reporting-problems.rst diff --git a/doc/INSTALL.rst b/doc/INSTALL.rst new file mode 120000 index 0000000000..99d491b4f8 --- /dev/null +++ b/doc/INSTALL.rst @@ -0,0 +1 @@ +../INSTALL \ No newline at end of file diff --git a/doc/README b/doc/README index 1e30d5ccc9..728afcb7d6 100644 --- a/doc/README +++ b/doc/README @@ -5,29 +5,36 @@ Documentation This directory contains Bro documentation in reStructured text format (see http://docutils.sourceforge.net/rst.html). -Please note that for now these files are primarily intended for use on -http://www.bro-ids.org. While the Bro build process will render local -versions into ``build/doc/`` (if docutils is found), the resulting -HTML is very minimalistic and some features are not supported. In -particular, some links will be broken. +It is the root of a Sphinx source tree and can be modified to add more +common/general documentation, style sheets, JavaScript, etc. The Sphinx +config file is produced from ``conf.py.in``, and can be edited to change +various Sphinx options. +There is also a custom Sphinx domain implemented in ``source/ext/bro.py`` +which adds some reST directives and roles that aid in generating useful +index entries and cross-references. Other extensions can be added in +a similar fashion. + +Either the ``make doc`` or ``make broxygen`` can be used to locally +render the reST files into HTML. Those targets depend on: + +* Python interpreter >= 2.5 +* `Sphinx `_ >= 1.0.1 + +After completion, HTML documentation is symlinked in ``build/html``. + +There's also ``make docclean`` and ``make broxygenclean`` targets to +clean the resulting documentation. Notes for Writing Documentation ------------------------------- * If you want to refer to a Bro script that's part of the - distribution, use {{'`foo.bro - <{{autodoc_bro_scripts}}/path/to/foo.html>`_'}}. For example, - ``{{'{{autodoc_bro_scripts}}/scripts/base/frameworks/notice/main.html}}'}}``. + distribution, it currently needs to be copied or otherwise symlinked + somewhere in to this Sphinx source tree. Then, it can be referenced + in a toc tree or with the :doc: role. -* If you want to refer to a page on the Bro web site, use the - ``docroot`` macro (e.g., - ``{{'href="{{docroot}}/download/index.html"'}}). Make sure to - include the ``index.html`` for the main pages, just as in the - example. - -* If you want to refer to page inside this directory, use a relative - path with HTML extension. (e.g., ``href="quickstart.html``). +* If you want to refer to a page on the Bro web site, use an HTTP URL. Guidelines ---------- diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html index 5a481ab882..90c448037e 100644 --- a/doc/_templates/layout.html +++ b/doc/_templates/layout.html @@ -2,4 +2,9 @@ {% set css_files = css_files + ["_static/broxygen.css"] %} +{% block extrahead %} + + +{% endblock %} + {% block relbar2 %}{% endblock %} diff --git a/doc/cluster.rst b/doc/cluster.rst index afd5f2fe35..9bffc35fec 100644 --- a/doc/cluster.rst +++ b/doc/cluster.rst @@ -11,9 +11,7 @@ Architecture The figure below illustrates the main components of a Bro cluster. -.. {{git_pull('bro:doc/deployment.png')}} - -.. image:: deployment.bro.png +.. image:: images/deployment.png Tap *** diff --git a/doc/components/broccoli-python/README.rst b/doc/components/broccoli-python/README.rst new file mode 120000 index 0000000000..4187e87202 --- /dev/null +++ b/doc/components/broccoli-python/README.rst @@ -0,0 +1 @@ +../../../aux/broccoli/bindings/broccoli-python/README \ No newline at end of file diff --git a/doc/components/broccoli/README.rst b/doc/components/broccoli/README.rst new file mode 120000 index 0000000000..d32c70ccd9 --- /dev/null +++ b/doc/components/broccoli/README.rst @@ -0,0 +1 @@ +../../../aux/broccoli/README \ No newline at end of file diff --git a/doc/components/broctl/README.rst b/doc/components/broctl/README.rst new file mode 120000 index 0000000000..cba305f48a --- /dev/null +++ b/doc/components/broctl/README.rst @@ -0,0 +1 @@ +../../../aux/broctl/doc/broctl.rst \ No newline at end of file diff --git a/doc/components/btest/README.rst b/doc/components/btest/README.rst new file mode 120000 index 0000000000..0da2935df1 --- /dev/null +++ b/doc/components/btest/README.rst @@ -0,0 +1 @@ +../../../aux/btest/README \ No newline at end of file diff --git a/doc/components/capstats/README.rst b/doc/components/capstats/README.rst new file mode 120000 index 0000000000..cb2380145d --- /dev/null +++ b/doc/components/capstats/README.rst @@ -0,0 +1 @@ +../../../aux/broctl/aux/capstats/README \ No newline at end of file diff --git a/doc/components/pysubnettree/README.rst b/doc/components/pysubnettree/README.rst new file mode 120000 index 0000000000..42ce17d303 --- /dev/null +++ b/doc/components/pysubnettree/README.rst @@ -0,0 +1 @@ +../../../aux/broctl/aux/pysubnettree/README \ No newline at end of file diff --git a/doc/components/trace-summary/README.rst b/doc/components/trace-summary/README.rst new file mode 120000 index 0000000000..78778364bd --- /dev/null +++ b/doc/components/trace-summary/README.rst @@ -0,0 +1 @@ +../../../aux/broctl/aux/trace-summary/README \ No newline at end of file diff --git a/doc/conf.py.in b/doc/conf.py.in index 34e92cd1f1..e997796eda 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -24,7 +24,7 @@ sys.path.insert(0, os.path.abspath('sphinx-sources/ext')) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['bro'] +extensions = ['bro', 'rst_directive'] # Add any paths that contain templates here, relative to this directory. templates_path = ['sphinx-sources/_templates', 'sphinx-sources/_static'] diff --git a/doc/ext/bro_lexer/__init__.py b/doc/ext/bro_lexer/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/doc/ext/bro_lexer/__init__.pyc b/doc/ext/bro_lexer/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1b03eaf1c181333ebdea52d9acb9162159cb5795 GIT binary patch literal 147 zcmZSn%*%Bma*kgz0~9a?~|]', Operator), + (r'([-+=&|]{2}|[+-=!><]=)', Operator), + (r'(in|match)\b', Operator.Word), + (r'[{}()\[\]$.,;]', Punctuation), + # Identfier + (r'([_a-zA-Z]\w*)(::)', bygroups(Name, Name.Namespace)), + (r'[a-zA-Z_][a-zA-Z_0-9]*', Name) + ], + 'string': [ + (r'"', String, '#pop'), + (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape), + (r'[^\\"\n]+', String), + (r'\\\n', String), + (r'\\', String) + ], + 'regex': [ + (r'/', String.Regex, '#pop'), + (r'\\[\\nt/]', String.Regex), # String.Escape is too intense. + (r'[^\\/\n]+', String.Regex), + (r'\\\n', String.Regex), + (r'\\', String.Regex) + ] + } diff --git a/doc/ext/bro_lexer/bro.pyc b/doc/ext/bro_lexer/bro.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6471e1528d8d02296dbdedc0548e86cd80a3c439 GIT binary patch literal 2702 zcmbtW>vG#f6kgeB(xhn{3h9lsG-)Z4UB`hk?GW0g<A#6G93i30@!x+Qg|&90|-3IiffxDoT3`B7t^Z$dh8UDEe?&V7$cmq>!f=EFOWS zBXCN{WmYW{7@cN(n(-ONXM{W}CO zFR<<^MH*dX{Ek3?(IpXI6Mf`uotE(i?}p z;t&<4DA&ZDy{lsPxx0mmGE}FiH=cVc*C()UY!W&R)Y)o%6t?=0T91QnQ@fD3t{*r2 z%^g2(dKS3JxZ!zjM}FSzw3=O3?dXf-{;0JVwq79!G=JlUrat-M;bvoJiQ#o1Khr2~ zfr%*jx~b=A_96HMzzyTrhH0qAwx?vcZ=+C48|9fcG@2wUrHy4O4W``66k3^;V=G_E z%vgC)(O$nPN1W%tR z=>_!Y7TYQ7Mo1!3g)z1hl^AF%EBN%oGBq|%H0>F19)e+>?^_j4g1j_EiG*g6nij_L zZH5F6>{zOSWR`hB`NiWEQt3n+Isin;vCZ@-p}E2K0calQLMN}JXZ19ThAPjni0Oe& z;ek*L)!AUZM(eoQc1={+GA?WqjRCU#ET3fLKxJFirFVGyHB9+pb-9|n!@I8yRoT`7 zXU%SVjqUA9M_4bCGkIg>@`}XX4jElO4$M_QD}L|r1KPyns&)NME@H2#*I^CbZX8bS zP}y;U>XDHh(@rWbh&TobQ==icRRix_WGGJFE{-6 zy7mBWbbFB3;Rm-q1huW!-R|N0w{E}oIKQ1Ae1F5waJz8#`z-Xq&Cg7`du-As6Zai# z!f(hR$ilG<0)E2=K~)`C5--y5+ciuR+!5N)p;GISLbFLQL}n?cK|jsW543eKWT#K_ z@fh{W_?eQ0$|JUXj$Xv&LL0n*GwZ*S(+Rq@5>5-r)IOyvDa_0+ztpLGpV5U=hSU?x z>*_GB_?>qsR;(zCW(f>&VAH=] + + Highlights the following code block according to if + given (e.g., "c", "python", etc.). + + - .. console:: + + Highlits the following code block as a shell session. + + For compatibility with the original version, "sourcecode" is + equivalent to "code". + +Original comment: + + The Pygments reStructuredText directive + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This fragment is a Docutils_ 0.5 directive that renders source code + (to HTML only, currently) via Pygments. + + To use it, adjust the options below and copy the code into a module + that you import on initialization. The code then automatically + registers a ``sourcecode`` directive that you can use instead of + normal code blocks like this:: + + .. sourcecode:: python + + My code goes here. + + If you want to have different code styles, e.g. one with line numbers + and one without, add formatters with their names in the VARIANTS dict + below. You can invoke them instead of the DEFAULT one by using a + directive option:: + + .. sourcecode:: python + :linenos: + + My code goes here. + + Look at the `directive documentation`_ to get all the gory details. + + .. _Docutils: http://docutils.sf.net/ + .. _directive documentation: + http://docutils.sourceforge.net/docs/howto/rst-directives.html + + :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +# Options +# ~~~~~~~ + +# Set to True if you want inline CSS styles instead of classes +INLINESTYLES = False + +from pygments.formatters import HtmlFormatter + +class MyHtmlFormatter(HtmlFormatter): + def format_unencoded(self, tokensource, outfile): + + # A NOP currently. + new_tokens = [] + for (i, piece) in tokensource: + new_tokens += [(i, piece)] + + return super(MyHtmlFormatter, self).format_unencoded(new_tokens, outfile) + +# The default formatter +DEFAULT = MyHtmlFormatter(noclasses=INLINESTYLES, cssclass="pygments") + +# Add name -> formatter pairs for every variant you want to use +VARIANTS = { + # 'linenos': HtmlFormatter(noclasses=INLINESTYLES, linenos=True), +} + + +import textwrap + +from docutils import nodes +from docutils.parsers.rst import directives, Directive + +from pygments import highlight +from pygments.lexers import get_lexer_by_name, guess_lexer, TextLexer +from pygments.token import Text, Keyword, Error, Operator, Name +from pygments.filter import Filter + +# Ugly hack to register the Bro lexer. I'm sure there's a better way to do it, +# but it's not obvious ... +from bro_lexer.bro import BroLexer +from pygments.lexers._mapping import LEXERS +LEXERS['BroLexer'] = ('bro_lexer.bro', BroLexer.name, BroLexer.aliases, BroLexer.filenames, ()) + +class Pygments(Directive): + """ Source code syntax hightlighting. + """ + #max_line_length = 68 + max_line_length = 0 + + required_arguments = 0 + optional_arguments = 1 + final_argument_whitespace = True + option_spec = dict([(key, directives.flag) for key in VARIANTS]) + has_content = True + + def wrapped_content(self): + content = [] + + if Console.max_line_length: + for line in self.content: + content += textwrap.wrap(line, Console.max_line_length, subsequent_indent=" ") + else: + content = self.content + + return u'\n'.join(content) + + def run(self): + self.assert_has_content() + + content = self.wrapped_content() + + if len(self.arguments) > 0: + try: + lexer = get_lexer_by_name(self.arguments[0]) + except (ValueError, IndexError): + # lexer not found, use default. + lexer = TextLexer() + else: + lexer = guess_lexer(content) + + # import sys + # print >>sys.stderr, self.arguments, lexer.__class__ + + # take an arbitrary option if more than one is given + formatter = self.options and VARIANTS[self.options.keys()[0]] or DEFAULT + parsed = highlight(content, lexer, formatter) + return [nodes.raw('', parsed, format='html')] + +class MyFilter(Filter): + def filter(self, lexer, stream): + + bol = True + + for (ttype, value) in stream: + # Color the '>' prompt sign. + if bol and ttype is Text and value == ">": + ttype = Name.Variable.Class # This gives us a nice red. + + # Discolor builtin, that can look funny. + if ttype is Name.Builtin: + ttype = Text + + bol = value.endswith("\n") + + yield (ttype, value) + +class Console(Pygments): + required_arguments = 0 + optional_arguments = 0 + + def run(self): + self.assert_has_content() + content = self.wrapped_content() + lexer = get_lexer_by_name("sh") + lexer.add_filter(MyFilter()) + parsed = highlight(content, lexer, DEFAULT) + return [nodes.raw('', parsed, format='html')] + +directives.register_directive('sourcecode', Pygments) +directives.register_directive('code', Pygments) +directives.register_directive('console', Console) diff --git a/doc/faq.rst b/doc/faq.rst new file mode 100644 index 0000000000..286f4b32cd --- /dev/null +++ b/doc/faq.rst @@ -0,0 +1,145 @@ + +========================== +Frequently Asked Questions +========================== + +.. raw:: html + +
+ +.. contents:: + +Installation and Configuration +============================== + +How can I tune my operating system for best capture performance? +---------------------------------------------------------------- + +Here are some pointers to more information: + +* Fabian Schneider's research on `high performance packet capture + `_ + +* `NSMWiki `_ has page on + *Collecting Data*. + +* An `IMC 2010 paper + `_ by + Lothar Braun et. al evaluates packet capture performance on + commodity hardware + +What does an error message like ``internal error: NB-DNS error`` mean? +--------------------------------------------------------------------------------------------------------------------------------- + +That often means that DNS is not set up correctly on the system +running Bro. Try verifying from the command line that DNS lookups +work, e.g., ``host www.google.com``. + + +Usage +===== + +How can I identify backscatter? +------------------------------- + +Identifying backscatter via connections labeled as ``OTH`` is not +a reliable means to detect backscatter. Use rather the following +procedure: + +* Enable connection history via ``redef record_state_history=T`` to + track all control/data packet types in connection logs. + +* Backscatter is now visible in terms of connections that never had an + initial ``SYN`` but started instead with a ``SYN-ACK`` or ``RST`` + (though this latter generally is just discarded). + +Is there help for understanding Bro's resource consumption? +----------------------------------------------------------- + +There are two scripts that collect statistics on resource usage: +``stats.bro`` and ``profiling.bro``. The former is quite lightweight, +while the latter should only be used for debugging. Furthermore, +there's also ``print-globals.bro``, which prints the size of all +global script variable at termination. + +How can I capture packets as an unprivileged user? +-------------------------------------------------- + +Normally, unprivileged users cannot capture packets from a network +interface, which means they would not be able to use Bro to read/analyze +live traffic. However, there are ways to enable packet capture +permission for non-root users, which is worth doing in the context of +using Bro to monitor live traffic + +With Linux Capabilities +^^^^^^^^^^^^^^^^^^^^^^^ + +Fully implemented since Linux kernel 2.6.24, capabilities are a way of +parceling superuser privileges into distinct units. Attach capabilities +required to capture packets to the ``bro`` executable file like this: + +.. console:: + + > sudo setcap cap_net_raw,cap_net_admin=eip /path/to/bro + +Now any unprivileged user should have the capability to capture packets +using Bro provided that they have the traditional file permissions to +read/execute the ``bro`` binary. + +With BPF Devices +^^^^^^^^^^^^^^^^ + +Systems using Berkeley Packet Filter (BPF) (e.g. FreeBSD & Mac OS X) +can allow users with read access to a BPF device to capture packets from +it using libpcap. + +* Example of manually changing BPF device permissions to allow users in + the ``admin`` group to capture packets: + +.. console:: + + > sudo chgrp admin /dev/bpf* + > sudo chmod g+r /dev/bpf* + +* Example of configuring devfs to set permissions of BPF devices, adding + entries to ``/etc/devfs.conf`` to grant ``admin`` group permission to + capture packets: + +.. console:: + + > sudo sh -c 'echo "own bpf root:admin" >> /etc/devfs.conf' + > sudo sh -c 'echo "perm bpf 0640" >> /etc/devfs.conf' + > sudo service devfs restart + +.. note:: As of Mac OS X 10.6, the BPF device is on devfs, but the used version + of devfs isn't capable of setting the device permissions. The permissions + can be changed manually, but they will not survive a reboot. + +Why isn't Bro producing the logs I expect? (A Note About Checksums) +------------------------------------------------------------------- + +Normally, Bro's event engine will discard packets which don't have valid +checksums. This can be a problem if one wants to analyze locally +generated/captured traffic on a system that offloads checksumming to the +network adapter. In that case, all transmitted/captured packets will have +bad checksums because they haven't yet been calculated by the NIC, thus +such packets will not undergo analysis defined in Bro policy scripts as they +normally would. Bad checksums in traces may also be a result of some packet +alteration tools. + +Bro has two options to workaround such situations and ignore bad checksums: + +1) The ``-C`` command line option to ``bro``. +2) An option called ``ignore_checksums`` that can be redefined at the policy + policy script layer (e.g. in your ``$PREFIX/share/bro/site/local/bro``): + + .. code:: bro + + redef ignore_checksums = T; + +The other alternative is to disable checksum offloading for your +network adapter, but this is not always possible or desirable. + +.. raw:: html + +
diff --git a/doc/geoip.rst b/doc/geoip.rst index 53413b5bee..bd9ae0c08d 100644 --- a/doc/geoip.rst +++ b/doc/geoip.rst @@ -3,7 +3,7 @@ GeoLocation =========== -.. class:: opening +.. rst-class:: opening During the process of creating policy scripts the need may arise to find the geographic location for an IP address. Bro has support diff --git a/doc/deployment.png b/doc/images/deployment.png similarity index 100% rename from doc/deployment.png rename to doc/images/deployment.png diff --git a/doc/index.rst b/doc/index.rst index 56a9e06fe8..022552f3d4 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -6,10 +6,11 @@ Bro Documentation .. toctree:: :maxdepth: 1 + INSTALL quickstart upgrade - FAQ - Reporting Problems + faq + reporting-problems Frameworks ---------- @@ -42,6 +43,20 @@ Script Reference scripts/packages scripts/index +Other Bro Components +-------------------- + +.. toctree:: + :maxdepth: 1 + + components/btest/README + components/broccoli/README + components/broccoli-python/README + components/broctl/README + components/capstats/README + components/pysubnettree/README + components/trace-summary/README + Indices and tables ------------------ diff --git a/doc/logging.rst b/doc/logging.rst index e7151b7f47..2817cadd45 100644 --- a/doc/logging.rst +++ b/doc/logging.rst @@ -2,7 +2,7 @@ Customizing Bro's Logging ========================== -.. class:: opening +.. rst-class:: opening Bro comes with a flexible key-value based logging interface that allows fine-grained control of what gets logged and how it is diff --git a/doc/notice.rst b/doc/notice.rst index 1322d44585..b1ffdacb75 100644 --- a/doc/notice.rst +++ b/doc/notice.rst @@ -2,7 +2,7 @@ Notice Framework ================ -.. class:: opening +.. rst-class:: opening One of the easiest ways to customize Bro is writing a local notice policy. Bro can detect a large number of potentially interesting diff --git a/doc/quickstart.rst b/doc/quickstart.rst index 22523e1618..c45db71ad9 100644 --- a/doc/quickstart.rst +++ b/doc/quickstart.rst @@ -8,7 +8,7 @@ Quick Start Guide ================= -.. class:: opening +.. rst-class:: opening The short story for getting Bro up and running in a simple configuration for analysis of either live traffic from a network interface or a packet diff --git a/doc/reporting-problems.rst b/doc/reporting-problems.rst new file mode 100644 index 0000000000..a1105708eb --- /dev/null +++ b/doc/reporting-problems.rst @@ -0,0 +1,194 @@ + +Reporting Problems +================== + +.. rst-class:: opening + + Here we summarizes some steps to follow when you see Bro doing + something it shouldn't. To provide help, it is often crucial for + us to have a way of reliably reproducing the effect you're seeing. + Unfortunately, reproducing problems can be rather tricky with Bro + because more often than not, they occur only in either very rare + situations or only after Bro has been running for some time. In + particular, getting a small trace showing a specific effect can be + a real problem. In the following, we'll summarize some strategies + to this end. + +Reporting Problems +------------------ + +Generally, when you encounter a problem with Bro, the best thing to do +is opening a new ticket in `Bro's issue tracker +`__ and include information on how to +reproduce the issue. Ideallt, your ticket should come with the +following: + +* The Bro version you're using (if working directly from the git + repository, the branch and revision number.) + +* The output you're seeing along with a description what you'd expect + Bro to do instead. + +* A *small* trace in `libpcap format `__ + demonstrating the effect (assuming the problem doesn't happen right + at startup already). + +* The exact command-line you're using to run Bro with that trace. If + you can, please try to run the Bro binary directly from the command + line rather than using BroControl. + +* Any non-standard scripts you're using (but please only those really + necessary; just a small code snippet triggering the problem would + perfect). + +* If you encounter a crash, information from the core dump, such as + the stack backtrace, can be very helpful. See below for more on + this. + + +How Do I Get a Trace File? +-------------------------- + +As Bro is usually running live, coming up with a small trace file that +reproduces a problem can turn out to be quite a challenge. Often it +works to best to start with a large trace that triggers the problem, +and then successively thin it out as much a possible. + +To get to the initial large trace, here are few things you can try: + +* Capture a trace with `tcpdump `__, either + on the same interface Bro is running on, or on another host where + you can generate traffic of the kind likely triggering the problem + (e.g., if you're seeing problems with the HTTP analyzer, record some + of your Web browsing on your desktop.) When using tcpdump, don't + forget to record *complete* packets (``tcpdump -s 0 ...``). You can + reduce the amount of traffic captured by using a suitable BPF filter + (e.g., for HTTP only, try ``port 80``). + +* Bro's command-line option ``-w `` records all packets it + processes into the given the file. You can then later run Bro + offline on this trace and it will process the packets in the same + way as it did live. This is particularly helpful with problems that + only occur after Bro has already been running for some time. For + example, sometimes a crash may be triggered by a particular kind of + traffic only occurring rarely. Running Bro live with ``-w`` and + then, after the crash, offline on the recorded trace might, with a + little bit of luck, reproduce the the problem reliably. However, be + careful with ``-w``: it can result in huge trace files, quickly + filling up your disk. (One way to mitigate the space issues is to + periodically delete the trace file by configuring + ``rotate-logs.bro`` accordingly. BroControl does that for you if you + set its ``SaveTraces`` option.) + +* Finally, you can try running Bro on a publically available trace + file, such as `anonymized FTP traffic `__, `headers-only enterprise traffic + `__, or + `Defcon traffic `__. Some of these + particularly stress certain components of Bro (e.g., the Defcon + traces contain tons of scans). + +Once you have a trace that demonstrates the effect, you will often +notice that it's pretty big, in particular if recorded from the link +you're monitoring. Therefore, the next step is to shrink its size as +much as possible. Here are a few things you can try to this end: + +* Very often, a single connection is able to demonstrate the problem. + If you can identify which one it is (e.g., from one of Bro's + ``*.log`` files) you can extract the connection's packets from the + trace usong tcpdump by filtering for the corresponding 4-tuple of + addresses and ports: + + .. console:: + + > tcpdump -r large.trace -w small.trace host and port and host and port + +* If you can't reduce the problem to a connection, try to identify + either a host pair or a single host triggering it, and filter down + the trace accordingly. + +* You can try to extract a smaller time slice from the trace using + `TCPslice `__. For example, to + extract the first 100 seconds from the trace: + + .. console:: + + # Test comment + > tcpslice +100 out + +Alternatively, tcpdump extracts the first ``n`` packets with its +option ``-c ``. + + +Getting More Information After a Crash +-------------------------------------- + +If Bro crashes, a *core dump* can be very helpful to nail down the +problem. Examining a core is not for the faint of heart but can reveal +extremely useful information. + +First, you should configure Bro with the option ``--enable-debug`` and +recompile; this will disable all compiler optimizations and thus make +the core dump more useful (don't expect great performance with this +version though; compiling Bro without optimization has a noticeable +impact on its CPU usage.). Then enable core dumps if you don't have +already (e.g., ``ulimit -c unlimited`` if you're using a bash). + +Once Bro has crashed, start gdb with the Bro binary and the file +containing the core dump. (Alternatively, you can also run Bro +directly inside gdb instead of working from a core file.) The first +helpful information to include with your tracker ticket is a stack +backtrace, which you get with gdb's ``bt`` command: + +.. console:: + + > gdb bro core + [...] + > bt + + +If the crash occurs inside Bro's script interpreter, the next thing to +do is identifying the line of script code processed just before the +abnormal termination. Look for methods in the stack backtrace which +belong to any of the script interpreter's classes. Roughly speaking, +these are all classes with names ending in ``Expr``, ``Stmt``, or +``Val``. Then climb up the stack with ``up`` until you reach the first +of these methods. The object to which ``this`` is pointing will have a +``Location`` object, which in turn contains the file name and line +number of the corresponding piece of script code. Continuing the +example from above, here's how to get that information: + +.. console:: + + [in gdb] + > up + > ... + > up + > print this->location->filename + > print this->location->first_line + + +If the crash occurs while processing input packets but you cannot +directly tell which connection is responsible (and thus not extract +its packets from the trace as suggested above), try getting the +4-tuple of the connection currently being processed from the core dump +by again examining the stack backtrace, this time looking for methods +belonging to the ``Connection`` class. That class has members +``orig_addr``/``resp_addr`` and ``orig_port``/``resp_port`` storing +(pointers to) the IP addresses and ports respectively: + +.. console:: + + [in gdb] + > up + > ... + > up + > printf "%08x:%04x %08x:%04x\n", *this->orig_addr, this->orig_port, *this->resp_addr, this->resp_port + + +Note that these values are stored in `network byte order +`__ +so you will need flip the bytes around if you are on a low-endian +machine (which is why the above example prints them in hex). For +example, if an IP address prints as ``0100007f`` , that's 127.0.0.1 . + diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt index 7b47424c6c..c14eab6d25 100644 --- a/doc/scripts/CMakeLists.txt +++ b/doc/scripts/CMakeLists.txt @@ -121,6 +121,7 @@ macro(REST_TARGET srcDir broInput) ARGS -rf .state *.log *.rst DEPENDS bro DEPENDS ${absSrcPath} + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMENT "[Bro] Generating reST docs for ${broInput}" ) diff --git a/doc/scripts/README b/doc/scripts/README index ca7f28492b..b3e44914f4 100644 --- a/doc/scripts/README +++ b/doc/scripts/README @@ -15,29 +15,10 @@ by CMake: ``build/`` directory inside ``reST`` (a symlink to ``doc/scripts/rest_output``). -``doc`` - - This target depends on a Python interpreter (>=2.5) and - `Sphinx `_ being installed. Sphinx can be - installed like:: - - > sudo easy_install sphinx - - This target will first build ``restdoc`` target and then copy the - resulting reST files as an input directory to Sphinx. - - After completion, HTML documentation can be located in the CMake - ``build/`` directory inside ``html`` (a symlink to - ``doc/scripts/out/html``) - ``restclean`` This target removes any reST documentation that has been generated so far. -``docclean`` - - This target removes Sphinx inputs and outputs from the CMake ``build/`` dir. - The ``genDocSourcesList.sh`` script can be run to automatically generate ``DocSourcesList.cmake``, which is the file CMake uses to define the list of documentation targets. This script should be run after adding new @@ -54,18 +35,10 @@ script's name to the blacklist, then append a ``rest_target()`` to the ``statictext`` variable where the first argument is the source directory containing the policy script to document, the second argument is the file name of the policy script, and the third argument is the path/name of a -pre-created reST document in the ``source/`` directory to which the +pre-created reST document in the ``../`` source directory to which the ``make doc`` process can append script documentation references. This pre-created reST document should also then be linked to from the TOC tree -in ``source/index.rst``. - -The Sphinx source tree template in ``source/`` can be modified to add more -common/general documentation, style sheets, JavaScript, etc. The Sphinx -config file is produced from ``conf.py.in``, so that can be edited to change -various Sphinx options, like setting the default HTML rendering theme. -There is also a custom Sphinx domain implemented in ``source/ext/bro.py`` -which adds some reST directives and roles that aid in generating useful -index entries and cross-references. +in ``../index.rst``. See ``example.bro`` for an example of how to document a Bro script such that ``make doc`` will be able to produce reST/HTML documentation for it. diff --git a/doc/signatures.rst b/doc/signatures.rst index 25dfc31f5e..f69b655624 100644 --- a/doc/signatures.rst +++ b/doc/signatures.rst @@ -3,7 +3,7 @@ Signatures ========== -.. class:: opening +.. rst-class:: opening Bro relies primarily on its extensive scripting language for defining and analyzing detection policies. In addition, however, diff --git a/doc/upgrade.rst b/doc/upgrade.rst index 696c64ef7f..ff426b1a77 100644 --- a/doc/upgrade.rst +++ b/doc/upgrade.rst @@ -3,7 +3,7 @@ Upgrading From Bro 1.5 to 2.0 ============================= -.. class:: opening +.. rst-class:: opening This guide details differences between Bro versions 1.5 and 2.0 that may be important for users to know as they work on updating From 72aa13968b94c0b96091a06a9f6b1dc01a124f0e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 15 Nov 2011 16:27:06 -0600 Subject: [PATCH 055/964] Fix documentation interlinkage and normalize console directive bodies. --- INSTALL | 25 ++++++-------- doc/CHANGES | 1 + doc/README | 5 +-- doc/faq.rst | 12 +++---- doc/quickstart.rst | 86 +++++++++++++++++++++------------------------- doc/signatures.rst | 4 +-- doc/upgrade.rst | 13 ++++--- 7 files changed, 69 insertions(+), 77 deletions(-) create mode 120000 doc/CHANGES diff --git a/INSTALL b/INSTALL index d1351a502f..a1b1cd99cf 100644 --- a/INSTALL +++ b/INSTALL @@ -39,9 +39,9 @@ Installation To build and install into ``/usr/local/bro``:: - > ./configure - > make - > make install + ./configure + make + make install This will first build Bro into a directory inside the distribution called ``build/``, using default build options. It then installs all @@ -50,7 +50,7 @@ required files into ``/usr/local/bro``, including the Bro binary in You can specify a different installation directory with:: - > ./configure --prefix= + ./configure --prefix= Note that ``/usr`` and ``/opt/bro`` are the standard prefixes for binary Bro packages to be installed, so those are typically not good @@ -71,11 +71,8 @@ Running Bro =========== Bro is a complex program and it takes a bit of time to get familiar -with it. A good place for newcomers to start is the quick start guide -available here: - - http://www.bro-ids.org/documentation/quickstart.html - +with it. A good place for newcomers to start is the +:doc:`quick start guide `. For developers that wish to run Bro directly from the ``build/`` directory (i.e., without performing ``make install``), they will have @@ -83,9 +80,9 @@ to first adjust ``BROPATH`` to look for scripts inside the build directory. Sourcing either ``build/bro-path-dev.sh`` or ``build/bro-path-dev.csh`` as appropriate for the current shell accomplishes this and also augments your ``PATH`` so you can use the -Bro binary directly: +Bro binary directly:: - > ./configure - > make - > source build/bro-path-dev.sh - > bro + ./configure + make + source build/bro-path-dev.sh + bro diff --git a/doc/CHANGES b/doc/CHANGES new file mode 120000 index 0000000000..3e8bc8c0c8 --- /dev/null +++ b/doc/CHANGES @@ -0,0 +1 @@ +../CHANGES \ No newline at end of file diff --git a/doc/README b/doc/README index 728afcb7d6..a9f2cdbc4e 100644 --- a/doc/README +++ b/doc/README @@ -29,10 +29,11 @@ clean the resulting documentation. Notes for Writing Documentation ------------------------------- -* If you want to refer to a Bro script that's part of the +* If you want to refer to a document that's part of the distribution, it currently needs to be copied or otherwise symlinked somewhere in to this Sphinx source tree. Then, it can be referenced - in a toc tree or with the :doc: role. + in a toc tree or with the :doc: role. Use the :download: role to + refer to static files that will not undergo sphinx rendering. * If you want to refer to a page on the Bro web site, use an HTTP URL. diff --git a/doc/faq.rst b/doc/faq.rst index 286f4b32cd..bdb1f50292 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -80,7 +80,7 @@ required to capture packets to the ``bro`` executable file like this: .. console:: - > sudo setcap cap_net_raw,cap_net_admin=eip /path/to/bro + sudo setcap cap_net_raw,cap_net_admin=eip /path/to/bro Now any unprivileged user should have the capability to capture packets using Bro provided that they have the traditional file permissions to @@ -98,8 +98,8 @@ it using libpcap. .. console:: - > sudo chgrp admin /dev/bpf* - > sudo chmod g+r /dev/bpf* + sudo chgrp admin /dev/bpf* + sudo chmod g+r /dev/bpf* * Example of configuring devfs to set permissions of BPF devices, adding entries to ``/etc/devfs.conf`` to grant ``admin`` group permission to @@ -107,9 +107,9 @@ it using libpcap. .. console:: - > sudo sh -c 'echo "own bpf root:admin" >> /etc/devfs.conf' - > sudo sh -c 'echo "perm bpf 0640" >> /etc/devfs.conf' - > sudo service devfs restart + sudo sh -c 'echo "own bpf root:admin" >> /etc/devfs.conf' + sudo sh -c 'echo "perm bpf 0640" >> /etc/devfs.conf' + sudo service devfs restart .. note:: As of Mac OS X 10.6, the BPF device is on devfs, but the used version of devfs isn't capable of setting the device permissions. The permissions diff --git a/doc/quickstart.rst b/doc/quickstart.rst index c45db71ad9..852cf7e5e2 100644 --- a/doc/quickstart.rst +++ b/doc/quickstart.rst @@ -3,6 +3,7 @@ .. _MacPorts: http://www.macports.org .. _Fink: http://www.finkproject.org .. _Homebrew: http://mxcl.github.com/homebrew +.. _bro downloads page: http://bro-ids.org/download/index.html ================= Quick Start Guide @@ -26,20 +27,19 @@ source code forms. Pre-Built Binary Release Packages --------------------------------- -See the `downloads page <{{docroot}}/download/index.html>`_ for currently -supported/targeted platforms. +See the `bro downloads page`_ for currently supported/targeted platforms. * RPM .. console:: - > sudo yum localinstall Bro-all*.rpm + sudo yum localinstall Bro-all*.rpm * DEB .. console:: - > sudo gdebi Bro-all-*.deb + sudo gdebi Bro-all-*.deb * MacOS Disk Image with Installer @@ -60,13 +60,13 @@ Required Dependencies .. console:: - > sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig + sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig * DEB/Debian-based Linux: .. console:: - > sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig + sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig * FreeBSD @@ -75,7 +75,7 @@ Required Dependencies .. console:: - > sudo pkg_add -r cmake swig bison python + sudo pkg_add -r cmake swig bison python * Mac OS X @@ -102,21 +102,21 @@ and sendmail for sending emails. .. console:: - > sudo yum install zlib-devel file-devel GeoIP-devel sendmail + sudo yum install zlib-devel file-devel GeoIP-devel sendmail * DEB/Debian-based Linux: .. console:: - > sudo apt-get install zlib1g-dev libmagic-dev libgeoip-dev sendmail + sudo apt-get install zlib1g-dev libmagic-dev libgeoip-dev sendmail * Ports-based FreeBSD .. console:: - > sudo pkg_add -r GeoIP + sudo pkg_add -r GeoIP - libz, libmagic, and sendmail are typically already available. +libz, libmagic, and sendmail are typically already available. * Mac OS X @@ -125,20 +125,20 @@ and sendmail for sending emails. Fink Homebrew), they should be automatically detected and Bro will compile against them. -Additional steps may be needed to `get the right GeoIP database -<{{git('bro:doc/geoip.rst')}}>`_. +Additional steps may be needed to :doc:`get the right GeoIP database ` Compiling Bro Source Code ~~~~~~~~~~~~~~~~~~~~~~~~~ Bro releases are bundled into source packages for convenience and -available from the `downloads page <{{docroot}}/download/index.html>`_. +available from the `bro downloads page`_. -The latest Bro development versions are obtainable through git repositories -hosted at `git.bro-ids.org `_. See our `git development -documentation <{{docroot}}/development/process.html>`_ for comprehensive -information on Bro's use of git revision control, but the short story for -downloading the full source code experience for Bro via git is: +The latest Bro development versions are obtainable through git +repositories hosted at `git.bro-ids.org `_. See +our `git development documentation +`_ for comprehensive +information on Bro's use of git revision control, but the short story +for downloading the full source code experience for Bro via git is: .. console:: @@ -157,9 +157,9 @@ desired root install path): .. console:: - > ./configure --prefix=/desired/install/path - > make - > make install + ./configure --prefix=/desired/install/path + make + make install The default installation prefix is ``/usr/local/bro``, which would typically require root privileges when doing the ``make install``. @@ -174,13 +174,13 @@ Bourne-Shell Syntax: .. console:: - > export PATH=/usr/local/bro/bin:$PATH + export PATH=/usr/local/bro/bin:$PATH C-Shell Syntax: .. console:: - > setenv PATH /usr/local/bro/bin:$PATH + setenv PATH /usr/local/bro/bin:$PATH Or substitute ``/opt/bro/bin`` instead if you installed from a binary package. @@ -211,7 +211,7 @@ Now start the BroControl shell like: .. console:: - > broctl + broctl Since this is the first-time use of the shell, perform an initial installation of the BroControl configuration: @@ -233,10 +233,9 @@ policy and output the results in ``$PREFIX/logs``. .. note:: The user starting BroControl needs permission to capture network traffic. If you are not root, you may need to grant further - privileges to the account you're using; see the `FAQ - <{{docroot}}/documentation/faq.html>`_. Also, if it - looks like Bro is not seeing any traffic, check out the FAQ entry - checksum offloading. + privileges to the account you're using; see the :doc:`FAQ `. + Also, if it looks like Bro is not seeing any traffic, check out + the FAQ entry on checksum offloading. You can leave it running for now, but to stop this Bro instance you would do: @@ -244,9 +243,7 @@ You can leave it running for now, but to stop this Bro instance you would do: [BroControl] > stop -We also recommend to insert the following entry into `crontab`: - -.. console:: +We also recommend to insert the following entry into `crontab`:: 0-59/5 * * * * $PREFIX/bin/broctl cron @@ -373,9 +370,7 @@ the variable's value may not change at run-time, but whose initial value can be modified via the ``redef`` operator at parse-time. So let's continue on our path to modify the behavior for the two SSL -and SSH notices. Looking at -`$PREFIX/share/bro/base/frameworks/notice/main.bro -<{{autodoc_bro_scripts}}/scripts/base/frameworks/notice/main.html>`_, +and SSH notices. Looking at :doc:`scripts/base/frameworks/notice/main`, we see that it advertises: .. code:: bro @@ -477,8 +472,7 @@ tweak the most basic options. Here's some suggestions on what to explore next: * Reading the code of scripts that ship with Bro is also a great way to gain understanding of the language and how you can start writing your own custom analysis. -* Review the `FAQ <{{docroot}}/documentation/faq.html>`_. -* Check out more `documentation <{{docroot}}/documentation/index.html>`_. +* Review the :doc:`FAQ `. * Continue reading below for another mini-tutorial on using Bro as a standalone command-line utility. @@ -496,7 +490,7 @@ Analyzing live traffic from an interface is simple: .. console:: - > bro -i en0 + bro -i en0 ``en0`` can be replaced by the interface of your choice and for the list of scripts, you can just use "all" for now to perform all the default analysis @@ -504,7 +498,7 @@ that's available. Bro will output log files into the working directory. -.. note:: The `FAQ <{{docroot}}/documentation/faq.html>`_ entries about +.. note:: The :doc:`FAQ ` entries about capturing as an unprivileged user and checksum offloading are particularly relevant at this point. @@ -513,7 +507,7 @@ command-line: .. console:: - > bro -i en0 local + bro -i en0 local This will cause Bro to print a warning about lacking the ``Site::local_nets`` variable being configured. You can supply this @@ -522,7 +516,7 @@ in place of the example subnets): .. console:: - > bro -r mypackets.trace local "Site::local_nets += { 1.2.3.0/24, 5.6.7.0/24 }" + bro -r mypackets.trace local "Site::local_nets += { 1.2.3.0/24, 5.6.7.0/24 }" Reading Packet Capture (pcap) Files @@ -533,7 +527,7 @@ like this: .. console:: - > sudo tcpdump -i en0 -s 0 -w mypackets.trace + sudo tcpdump -i en0 -s 0 -w mypackets.trace Where ``en0`` can be replaced by the correct interface for your system as shown by e.g. ``ifconfig``. (The ``-s 0`` argument tells it to capture @@ -544,7 +538,7 @@ and tell Bro to perform all the default analysis on the capture which primarily .. console:: - > bro -r mypackets.trace + bro -r mypackets.trace Bro will output log files into the working directory. @@ -553,7 +547,7 @@ script that we include as a suggested configuration: .. console:: - > bro -r mypackets.trace local + bro -r mypackets.trace local Telling Bro Which Scripts to Load @@ -563,7 +557,7 @@ A command-line invocation of Bro typically looks like: .. console:: - > bro + bro Where the last arguments are the specific policy scripts that this Bro instance will load. These arguments don't have to include the ``.bro`` @@ -578,7 +572,7 @@ logging) and adds SSL certificate validation. .. console:: - > bro -r mypackets.trace protocols/ssl/validate-certs + bro -r mypackets.trace protocols/ssl/validate-certs You might notice that a script you load from the command line uses the ``@load`` directive in the Bro language to declare dependence on other scripts. diff --git a/doc/signatures.rst b/doc/signatures.rst index f69b655624..e2f9a8d702 100644 --- a/doc/signatures.rst +++ b/doc/signatures.rst @@ -47,8 +47,8 @@ piece of payload which triggered the pattern match. To turn such ``signature_match`` events into actual alarms, you can load Bro's ``signature.bro`` script. This script contains a default -event handler that raises ``SensitiveSignature`` `Notices -`_ (as well as others; see the beginning of the script). +event handler that raises ``SensitiveSignature`` :doc:`Notices ` +(as well as others; see the beginning of the script). As signatures are independent of Bro's policy scripts, they are put into their own file(s). There are two ways to specify which files diff --git a/doc/upgrade.rst b/doc/upgrade.rst index ff426b1a77..71cc5e401d 100644 --- a/doc/upgrade.rst +++ b/doc/upgrade.rst @@ -39,7 +39,7 @@ version. The two rules of thumb are: if you need help. Below we summarize changes from 1.x to 2.x in more detail. This list -isn't complete, see the `CHANGES <{{git('bro:CHANGES', 'txt')}}>`_ file in the +isn't complete, see the :download:`CHANGES ` file in the distribution for the full story. Default Scripts @@ -131,8 +131,8 @@ Logging Framework endpoint. - The new logging framework makes it possible to extend, customize, - and filter logs very easily. See `the logging framework - <{{git('bro:doc/logging.rst')}}>`_ more information on usage. + and filter logs very easily. See the :doc:`logging framework ` + more information on usage. - A common pattern found in the new scripts is to store logging stream records for protocols inside the ``connection`` records so that @@ -155,8 +155,7 @@ Notice Framework The way users interact with "notices" has changed significantly in order to make it easier to define a site policy and more extensible -for adding customized actions. See the `the notice framework -<{{git('bro:doc/notice.rst')}}>`_. +for adding customized actions. See the :doc:`notice framework `. New Default Settings @@ -198,7 +197,7 @@ Variable Naming - Identifiers may have been renamed to conform to new `scripting conventions - <{{docroot}}/development/script-conventions.html>`_ + `_ BroControl @@ -240,7 +239,7 @@ Development Infrastructure Bro development has moved from using SVN to Git for revision control. Users that like to use the latest Bro developments by checking it out from the source repositories should see the `development process -<{{docroot}}/development/process.html>`_. Note that all the various +`_. Note that all the various sub-components now reside on their own repositories. However, the top-level Bro repository includes them as git submodules so it's easu to check them all out simultaneously. From fb5f26e7fcf5d3bda5c25472e2933e7ed975ddb1 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 15:23:46 -0800 Subject: [PATCH 056/964] make default values work (thanks to robin) --- src/InputMgr.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 4506501c94..a0a9d6a35c 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -97,7 +97,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) return 0; } - EnumVal* reader = description->Lookup(rtype->FieldOffset("reader"))->AsEnumVal(); + EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) @@ -148,7 +148,6 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); - Val *want_record = description->Lookup(rtype->FieldOffset("want_record")); vector fieldsV; // vector, because we don't know the length beforehands @@ -163,9 +162,11 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) if ( status ) { reporter->Error("Problem unrolling"); + Unref(reader); return 0; } + Val *want_record = description->LookupWithDefault(rtype->FieldOffset("want_record")); LogField** fields = new LogField*[fieldsV.size()]; for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { @@ -174,7 +175,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; - info->type = reader->Ref()->AsEnumVal(); + info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault info->num_idx_fields = idxfields; info->num_val_fields = valfields; info->tab = dst->Ref()->AsTableVal(); @@ -184,6 +185,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); info->want_record = ( want_record->InternalInt() == 1 ); + Unref(want_record); // ref'd by lookupwithdefault if ( valfields > 1 ) { assert(info->want_record); From 821878835a800edb794c25c32a59ef67958a87da Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 16:32:35 -0800 Subject: [PATCH 057/964] read vector. still missing: enums, empty fields for optional parameters. --- src/InputMgr.cc | 45 +++++++++++++++++++++++++++------ src/InputReaderAscii.cc | 56 ++++++++++++++++++++++++++--------------- src/InputReaderAscii.h | 6 ++--- src/LogMgr.cc | 6 ++--- src/LogMgr.h | 6 ++--- 5 files changed, 83 insertions(+), 36 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index a0a9d6a35c..4c21268c84 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -244,12 +244,10 @@ bool InputMgr::IsCompatibleType(BroType* t, bool atomic_only) case TYPE_VECTOR: { - return false; // do me... - - //if ( atomic_only ) - // return false; - // - //return IsCompatibleType(t->AsVectorType()->YieldType()); + if ( atomic_only ) + return false; + + return IsCompatibleType(t->AsVectorType()->YieldType(), true); } default: @@ -342,7 +340,9 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec field->name = nameprepend + rec->FieldName(i); field->type = rec->FieldType(i)->Tag(); if ( field->type == TYPE_TABLE ) { - field->set_type = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); + field->subtype = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); + } else if ( field->type == TYPE_VECTOR ) { + field->subtype = rec->FieldType(i)->AsVectorType()->YieldType()->Tag(); } fields->push_back(field); @@ -870,6 +870,13 @@ int InputMgr::GetLogValLength(const LogVal* val) { break; } + case TYPE_VECTOR: { + for ( int i = 0; i < val->val.vector_val.size; i++ ) { + length += GetLogValLength(val->val.vector_val.vals[i]); + } + break; + } + default: reporter->InternalError("unsupported type %d for GetLogValLength", val->type); } @@ -936,6 +943,15 @@ int InputMgr::CopyLogVal(char *data, const int startpos, const LogVal* val) { break; } + case TYPE_VECTOR: { + int length = 0; + for ( int i = 0; i < val->val.vector_val.size; i++ ) { + length += CopyLogVal(data, startpos+length, val->val.vector_val.vals[i]); + } + return length; + break; + } + default: reporter->InternalError("unsupported type %d for CopyLogVal", val->type); return 0; @@ -1039,6 +1055,21 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { break; } + case TYPE_VECTOR: { + assert ( val->val.vector_val.size > 1 ); // implement empty vector... + + // all entries have to have the same type... + TypeTag type = val->val.vector_val.vals[0]->type; + VectorType* vt = new VectorType(base_type(type)); + VectorVal* v = new VectorVal(vt); + for ( int i = 0; i < val->val.vector_val.size; i++ ) { + assert( val->val.vector_val.vals[i]->type == type); + v->Assign(i, LogValToVal( val->val.set_val.vals[i], type ), 0); + } + return v; + + } + case TYPE_ENUM: reporter->InternalError("Sorry, Enum reading does not yet work, missing internal inferface"); diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 60a8c5685a..3b4409e652 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -11,20 +11,20 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int position = arg_position; } -FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_set_type, int arg_position) - : name(arg_name), type(arg_type), set_type(arg_set_type) +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position) + : name(arg_name), type(arg_type), subtype(arg_subtype) { position = arg_position; } FieldMapping::FieldMapping(const FieldMapping& arg) - : name(arg.name), type(arg.type), set_type(arg.set_type) + : name(arg.name), type(arg.type), subtype(arg.subtype) { position = arg.position; } -FieldMapping FieldMapping::setType() { - return FieldMapping(name, set_type, position); +FieldMapping FieldMapping::subType() { + return FieldMapping(name, subtype, position); } InputReaderAscii::InputReaderAscii() @@ -91,7 +91,7 @@ bool InputReaderAscii::ReadHeader() { const LogField* field = fields[i]; if ( field->name == s ) { // cool, found field. note position - FieldMapping f(field->name, field->type, field->set_type, i); + FieldMapping f(field->name, field->type, field->subtype, i); columnMap.push_back(f); wantFields++; break; // done with searching @@ -112,7 +112,7 @@ bool InputReaderAscii::ReadHeader() { if ( wantFields != (int) num_fields ) { // we did not find all fields? // :( - Error("One of the requested fields could not be found in the input data file"); + Error(Fmt("One of the requested fields could not be found in the input data file. Found %d fields, wanted %d", wantFields, num_fields)); return false; } @@ -199,25 +199,40 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { break; } - case TYPE_TABLE: { - // construct a table from entry... - // for the moment assume, that entries are split by ",". - - if ( s == "-" ) { - // empty - val->val.set_val.size = 0; - break; - } - + case TYPE_TABLE: + case TYPE_VECTOR: + // First - common initialization + // Then - initialization for table. + // Then - initialization for vector. + // Then - common stuff + { // how many entries do we have... unsigned int length = 1; for ( unsigned int i = 0; i < s.size(); i++ ) if ( s[i] == ',') length++; unsigned int pos = 0; + LogVal** lvals = new LogVal* [length]; - val->val.set_val.vals = lvals; - val->val.set_val.size = length; + + if ( field.type == TYPE_TABLE ) { + // construct a table from entry... + // for the moment assume, that entries are split by ",". + + /* Fix support for emtyp tables if ( s == "-" ) { + // empty + val->val.set_val.size = 0; + break; + } */ + + val->val.set_val.vals = lvals; + val->val.set_val.size = length; + } else if ( field.type == TYPE_VECTOR ) { + val->val.vector_val.vals = lvals; + val->val.vector_val.size = length; + } else { + assert(false); + } istringstream splitstream(s); while ( splitstream ) { @@ -232,7 +247,7 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { break; - LogVal* newval = EntryToVal(element, field.setType()); + LogVal* newval = EntryToVal(element, field.subType()); if ( newval == 0 ) { Error("Error while reading set"); return 0; @@ -243,6 +258,7 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { } + if ( pos != length ) { Error("Internal error while parsing set: did not find all elements"); return 0; diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index ab2b89339c..56c1001acb 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -12,15 +12,15 @@ struct FieldMapping { string name; TypeTag type; - TypeTag set_type; + TypeTag subtype; int position; FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); - FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_set_type, int arg_position); + FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); FieldMapping(const FieldMapping& arg); FieldMapping() { position = -1; } - FieldMapping setType(); + FieldMapping subType(); bool IsEmpty() { return position == -1; } }; diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 9818d9cdfb..6eaace3893 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -83,16 +83,16 @@ bool LogField::Read(SerializationFormat* fmt) int t; int it; - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&it, "set_type") ); + bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&it, "subtype") ); type = (TypeTag) t; - set_type = (TypeTag) it; + subtype = (TypeTag) it; return success; } bool LogField::Write(SerializationFormat* fmt) const { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)set_type, "set_type")); + return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); } LogVal::~LogVal() diff --git a/src/LogMgr.h b/src/LogMgr.h index 40dab8677b..b8530d29ab 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -15,12 +15,12 @@ class SerializationFormat; struct LogField { string name; TypeTag type; - // needed by input framework. otherwise it cannot determine the inner type of a set. - TypeTag set_type; + // needed by input framework. otherwise it cannot determine the inner type of a set or vector. + TypeTag subtype; LogField() { } LogField(const LogField& other) - : name(other.name), type(other.type), set_type(other.set_type) { } + : name(other.name), type(other.type), subtype(other.subtype) { } // (Un-)serialize. bool Read(SerializationFormat* fmt); From c35094ea0be55edd148a6e49a6b176aa3879ed62 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 15 Nov 2011 16:42:23 -0800 Subject: [PATCH 058/964] Update missing in last commit to this branch. --- .../frameworks/notice/actions/pp-alarms.bro | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/scripts/base/frameworks/notice/actions/pp-alarms.bro b/scripts/base/frameworks/notice/actions/pp-alarms.bro index c8d8259cdc..89cf310ac5 100644 --- a/scripts/base/frameworks/notice/actions/pp-alarms.bro +++ b/scripts/base/frameworks/notice/actions/pp-alarms.bro @@ -22,6 +22,10 @@ export { ## Function that renders a single alarm. Can be overidden. global pretty_print_alarm: function(out: file, n: Info) &redef; + + ## Force generating mail file, even if reading from traces or no mail + ## destination is defined. This is mainly for testing. + global force_email_summaries = F &redef; } # We maintain an old-style file recording the pretty-printed alarms. @@ -32,6 +36,9 @@ global pp_alarms_open: bool = F; # Returns True if pretty-printed alarm summaries are activated. function want_pp() : bool { + if ( force_email_summaries ) + return T; + return (pretty_print_alarms && ! reading_traces() && (mail_dest != "" || mail_dest_pretty_printed != "")); } @@ -61,6 +68,11 @@ function pp_send(rinfo: Log::RotationInfo) local subject = fmt("Alarm summary from %s-%s", from, to); local dest = mail_dest_pretty_printed != "" ? mail_dest_pretty_printed : mail_dest; + + if ( dest == "" ) + # No mail destination configured, just leave the file alone. This is mainly for + # testing. + return; local headers = email_headers(subject, dest); @@ -193,6 +205,12 @@ function pretty_print_alarm(out: file, n: Info) return; } + if ( reading_traces() ) + { + do_msg(out, n, line1, line2, line3, h1, "", h2, ""); + return; + } + when ( local h1name = lookup_addr(h1) ) { if ( h2 == 0.0.0.0 ) From fc7ebbe8d7bc05b0129a693b0e8337a8f67fe855 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 15 Nov 2011 17:52:42 -0800 Subject: [PATCH 059/964] Updating submodule(s). [nomail] --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- cmake | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aux/binpac b/aux/binpac index 777b8a21c4..0f99acfbf6 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 777b8a21c4c74e1f62e8b9896b082e8c059b539f +Subproject commit 0f99acfbf6205830f0db699a75554262c26427f9 diff --git a/aux/bro-aux b/aux/bro-aux index 906f970df5..1a7a9357fb 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 906f970df5f708582c7002069b787d5af586b46f +Subproject commit 1a7a9357fba88a43c90a39d8d72b42fa53b89b75 diff --git a/aux/broccoli b/aux/broccoli index e02e3cc89a..a1a03c6868 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit e02e3cc89a3efb3d7ec376154e24835b4b828be8 +Subproject commit a1a03c686866bd30ee086ff933128055a20ebd56 diff --git a/aux/broctl b/aux/broctl index 6fb4e5689d..03d39aa5d4 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 6fb4e5689d2ae0d1c4ab7af0a8df80e6eaa98fb6 +Subproject commit 03d39aa5d4ab24cd9b8e404a9ceb583d5270444c diff --git a/cmake b/cmake index 704e255d7e..44f2985475 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 704e255d7ef2faf926836c1c64d16c5b8a02b063 +Subproject commit 44f2985475e5ff6cc9061683e21ef4b184bdfc7e From ab68d8400789156a23b1e6bbaf57828e7eac7c83 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 16 Nov 2011 22:13:36 -0800 Subject: [PATCH 060/964] reading of enum types (thanks, Seth) --- src/InputMgr.cc | 50 +++++++++++++++++++++++++++------------------- src/InputMgr.h | 4 ++-- src/InputReader.cc | 3 ++- src/InputReader.h | 2 +- 4 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 4c21268c84..64f3e2eb9b 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -422,7 +422,7 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo if ( num_fields == 1 && type->FieldType(0)->Tag() != TYPE_RECORD ) { - idxval = LogValToVal(vals[0]); + idxval = LogValToVal(vals[0], type->FieldType(0)); position = 1; } else { ListVal *l = new ListVal(TYPE_ANY); @@ -430,7 +430,7 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo if ( type->FieldType(j)->Tag() == TYPE_RECORD ) { l->Append(LogValToRecordVal(vals, type->FieldType(j)->AsRecordType(), &position)); } else { - l->Append(LogValToVal(vals[position], type->FieldType(j)->Tag())); + l->Append(LogValToVal(vals[position], type->FieldType(j))); position++; } } @@ -486,7 +486,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { int position = i->num_idx_fields; if ( i->num_val_fields == 1 && !i->want_record ) { - valval = LogValToVal(vals[i->num_idx_fields]); + valval = LogValToVal(vals[i->num_idx_fields], i->rtype->FieldType(i->num_idx_fields)); } else { RecordVal * r = new RecordVal(i->rtype); @@ -501,7 +501,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->rtype->FieldType(j)->Tag()); + val = LogValToVal(vals[position], i->rtype->FieldType(j)); position++; } @@ -705,7 +705,7 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { int position = i->num_idx_fields; if ( i->num_val_fields == 1 && !i->want_record ) { - valval = LogValToVal(vals[i->num_idx_fields]); + valval = LogValToVal(vals[i->num_idx_fields], i->rtype->FieldType(i->num_idx_fields)); } else { RecordVal * r = new RecordVal(i->rtype); @@ -715,7 +715,7 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->rtype->FieldType(j)->Tag()); + val = LogValToVal(vals[position], i->rtype->FieldType(j)); position++; } @@ -760,7 +760,7 @@ void InputMgr::Error(InputReader* reader, const char* msg) reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); } - +/* Does not work atm, because LogValToVal needs BroType void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { EventHandler* handler = event_registry->Lookup(name.c_str()); @@ -775,7 +775,7 @@ void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c } mgr.Dispatch(new Event(handler, vl)); -} +} */ void InputMgr::SendEvent(const string& name, EnumVal* event, Val* left, Val* right) { @@ -814,7 +814,7 @@ Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_ if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) { fieldVal = LogValToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); } else { - fieldVal = LogValToVal(vals[*position], request_type->FieldType(i)->Tag()); + fieldVal = LogValToVal(vals[*position], request_type->FieldType(i)); (*position)++; } @@ -988,10 +988,10 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals } -Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { +Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { - if ( request_type != TYPE_ANY && request_type != val->type ) { - reporter->InternalError("Typetags don't match: %d vs %d", request_type, val->type); + if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) { + reporter->InternalError("Typetags don't match: %d vs %d", request_type->Tag(), val->type); return 0; } @@ -1041,13 +1041,12 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { return new TableVal(new SetType(set_index, 0)); } else { // all entries have to have the same type... - TypeTag type = val->val.set_val.vals[0]->type; - TypeList* set_index = new TypeList(base_type(type)); - set_index->Append(base_type(type)); + BroType* type = request_type->AsTableType()->Indices()->PureType(); + TypeList* set_index = new TypeList(type->Ref()); + set_index->Append(type->Ref()); SetType* s = new SetType(set_index, 0); TableVal* t = new TableVal(s); for ( int i = 0; i < val->val.set_val.size; i++ ) { - assert( val->val.set_val.vals[i]->type == type); t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); } return t; @@ -1059,19 +1058,28 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { assert ( val->val.vector_val.size > 1 ); // implement empty vector... // all entries have to have the same type... - TypeTag type = val->val.vector_val.vals[0]->type; - VectorType* vt = new VectorType(base_type(type)); + BroType* type = request_type->AsVectorType()->YieldType(); + VectorType* vt = new VectorType(type->Ref()); VectorVal* v = new VectorVal(vt); for ( int i = 0; i < val->val.vector_val.size; i++ ) { - assert( val->val.vector_val.vals[i]->type == type); v->Assign(i, LogValToVal( val->val.set_val.vals[i], type ), 0); } return v; } - case TYPE_ENUM: - reporter->InternalError("Sorry, Enum reading does not yet work, missing internal inferface"); + case TYPE_ENUM: { + // well, this is kind of stupid, because EnumType just mangles the module name and the var name together again... + // but well + string module = extract_module_name(val->val.string_val->c_str()); + string var = extract_var_name(val->val.string_val->c_str()); + bro_int_t index = request_type->AsEnumType()->Lookup(module, var.c_str()); + if ( index == -1 ) { + reporter->InternalError("Value not found in enum mappimg. Module: %s, var: %s", module.c_str(), var.c_str()); + } + return new EnumVal(index, request_type->Ref()->AsEnumType() ); + break; + } default: diff --git a/src/InputMgr.h b/src/InputMgr.h index 93c6447467..17b7e2e804 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -55,11 +55,11 @@ private: int GetLogValLength(const LogVal* val); int CopyLogVal(char *data, const int startpos, const LogVal* val); - Val* LogValToVal(const LogVal* val, TypeTag request_type = TYPE_ANY); + Val* LogValToVal(const LogVal* val, BroType* request_type); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); - void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); diff --git a/src/InputReader.cc b/src/InputReader.cc index 994f8b9b97..7403d1f989 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -64,10 +64,11 @@ bool InputReader::Update() return DoUpdate(); } +/* void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { input_mgr->SendEvent(name, num_vals, vals); -} +} */ // stolen from logwriter const char* InputReader::Fmt(const char* format, ...) diff --git a/src/InputReader.h b/src/InputReader.h index b547d29506..41000e4c0c 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -40,7 +40,7 @@ protected: // A thread-safe version of fmt(). (stolen from logwriter) const char* Fmt(const char* format, ...); - void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); void Put(const LogVal* const *val); void Clear(); From 4fef1e3f8c5fe7005f23f91d6001cd1d79ffcdd2 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 16 Nov 2011 22:47:28 -0800 Subject: [PATCH 061/964] set & entry separator configuration (with the restriction that they have to be exactly one character long) --- .../base/frameworks/input/readers/ascii.bro | 19 +++++++++ src/InputReaderAscii.cc | 40 +++++++++++++++++-- src/InputReaderAscii.h | 15 ++++++- src/input.bif | 9 +++++ 4 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 scripts/base/frameworks/input/readers/ascii.bro diff --git a/scripts/base/frameworks/input/readers/ascii.bro b/scripts/base/frameworks/input/readers/ascii.bro new file mode 100644 index 0000000000..9f630975a2 --- /dev/null +++ b/scripts/base/frameworks/input/readers/ascii.bro @@ -0,0 +1,19 @@ +##! Interface for the ascii input reader. + +module InputAscii; + +export { + ## Separator between fields. + ## Please note that the separator has to be exactly one character long + const separator = "\t" &redef; + + ## Separator between set elements. + ## Please note that the separator has to be exactly one character long + const set_separator = "," &redef; + + ## String to use for empty fields. + const empty_field = "-" &redef; + + ## String to use for an unset &optional field. + const unset_field = "-" &redef; +} diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 3b4409e652..22cdcfdcf0 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -2,6 +2,7 @@ #include "InputReaderAscii.h" #include "DebugLogger.h" +#include "NetVar.h" #include @@ -29,15 +30,46 @@ FieldMapping FieldMapping::subType() { InputReaderAscii::InputReaderAscii() { - //DBG_LOG(DBG_LOGGING, "input reader initialized"); file = 0; //keyMap = new map(); + + separator_len = BifConst::LogAscii::separator->Len(); + separator = new char[separator_len]; + memcpy(separator, BifConst::LogAscii::separator->Bytes(), + separator_len); + if ( separator_len != 1 ) { + Error("separator length has to be 1. Separator will be truncated."); + } + + set_separator_len = BifConst::LogAscii::set_separator->Len(); + set_separator = new char[set_separator_len]; + memcpy(set_separator, BifConst::LogAscii::set_separator->Bytes(), + set_separator_len); + if ( set_separator_len != 1 ) { + Error("set_separator length has to be 1. Separator will be truncated."); + } + + empty_field_len = BifConst::LogAscii::empty_field->Len(); + empty_field = new char[empty_field_len]; + memcpy(empty_field, BifConst::LogAscii::empty_field->Bytes(), + empty_field_len); + + unset_field_len = BifConst::LogAscii::unset_field->Len(); + unset_field = new char[unset_field_len]; + memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(), + unset_field_len); + } InputReaderAscii::~InputReaderAscii() { DoFinish(); + + delete [] separator; + delete [] set_separator; + delete [] empty_field; + delete [] unset_field; } void InputReaderAscii::DoFinish() @@ -83,7 +115,7 @@ bool InputReaderAscii::ReadHeader() { int wantFields = 0; while ( splitstream ) { string s; - if ( !getline(splitstream, s, '\t')) + if ( !getline(splitstream, s, separator[0])) break; // current found heading in s... compare if we want it @@ -243,7 +275,7 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { break; } - if ( !getline(splitstream, element, ',') ) + if ( !getline(splitstream, element, set_separator[0]) ) break; @@ -322,7 +354,7 @@ bool InputReaderAscii::DoUpdate() { while ( splitstream ) { string s; - if ( !getline(splitstream, s, '\t') ) + if ( !getline(splitstream, s, separator[0]) ) break; diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 56c1001acb..d69b8c04bc 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -57,7 +57,20 @@ private: const LogField* const * fields; // raw mapping //map *keyMap; - + // + // Options set from the script-level. + char* separator; + int separator_len; + + char* set_separator; + int set_separator_len; + + char* empty_field; + int empty_field_len; + + char* unset_field; + int unset_field_len; + }; diff --git a/src/input.bif b/src/input.bif index 7b051fba16..aaef25dcc3 100644 --- a/src/input.bif +++ b/src/input.bif @@ -52,3 +52,12 @@ function Input::__remove_filter%(id: Log::ID, name: string%) : bool return new Val( res, TYPE_BOOL); %} +# Options for Ascii Reader + +module InputAscii; + +const separator: string; +const set_separator: string; +const empty_field: string; +const unset_field: string; + From 4dd95fcf3cf1efc83ec4d2c53fee765d2a800674 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 16 Nov 2011 23:51:51 -0800 Subject: [PATCH 062/964] support for uninitialized fields & empty sets and tables. The only snag is... with the default output format of the log-file writer, the input reader cannot tell if a table or set is empty or uninitialized (both cases use the same character by default). In this case, by default it is assumed that the field/vector is uninitalized. --- scripts/base/frameworks/input/__load__.bro | 2 + src/InputMgr.cc | 61 ++++++++++------------ src/InputReaderAscii.cc | 61 ++++++++-------------- src/InputReaderAscii.h | 12 ++--- 4 files changed, 57 insertions(+), 79 deletions(-) diff --git a/scripts/base/frameworks/input/__load__.bro b/scripts/base/frameworks/input/__load__.bro index a10fe855df..a3315186d5 100644 --- a/scripts/base/frameworks/input/__load__.bro +++ b/scripts/base/frameworks/input/__load__.bro @@ -1 +1,3 @@ @load ./main +@load ./readers/ascii + diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 64f3e2eb9b..9c0e9c12b5 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -505,10 +505,10 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { position++; } - if ( val == 0 ) { + /* if ( val == 0 ) { reporter->InternalError("conversion error"); return; - } + } */ r->Assign(j,val); @@ -871,7 +871,9 @@ int InputMgr::GetLogValLength(const LogVal* val) { } case TYPE_VECTOR: { - for ( int i = 0; i < val->val.vector_val.size; i++ ) { + int j = val->val.vector_val.size; + for ( int i = 0; i < j; i++ ) { + reporter->Error("size is %d", val->val.vector_val.size); length += GetLogValLength(val->val.vector_val.vals[i]); } break; @@ -945,7 +947,8 @@ int InputMgr::CopyLogVal(char *data, const int startpos, const LogVal* val) { case TYPE_VECTOR: { int length = 0; - for ( int i = 0; i < val->val.vector_val.size; i++ ) { + int j = val->val.vector_val.size; + for ( int i = 0; i < j; i++ ) { length += CopyLogVal(data, startpos+length, val->val.vector_val.vals[i]); } return length; @@ -994,6 +997,10 @@ Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { reporter->InternalError("Typetags don't match: %d vs %d", request_type->Tag(), val->type); return 0; } + + if ( !val->present ) { + return 0; // unset field + } switch ( val->type ) { @@ -1033,38 +1040,28 @@ Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { break; case TYPE_TABLE: { - if ( val->val.set_val.size == 0 ) { - // empty table - TypeList* set_index = new TypeList(base_type(TYPE_ANY)); - // iim quite sure this does not work... we probably need the internal set type for this... - reporter->InternalError("Implement me."); - return new TableVal(new SetType(set_index, 0)); - } else { - // all entries have to have the same type... - BroType* type = request_type->AsTableType()->Indices()->PureType(); - TypeList* set_index = new TypeList(type->Ref()); - set_index->Append(type->Ref()); - SetType* s = new SetType(set_index, 0); - TableVal* t = new TableVal(s); - for ( int i = 0; i < val->val.set_val.size; i++ ) { - t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); - } - return t; - } + // all entries have to have the same type... + BroType* type = request_type->AsTableType()->Indices()->PureType(); + TypeList* set_index = new TypeList(type->Ref()); + set_index->Append(type->Ref()); + SetType* s = new SetType(set_index, 0); + TableVal* t = new TableVal(s); + for ( int i = 0; i < val->val.set_val.size; i++ ) { + t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); + } + return t; break; } case TYPE_VECTOR: { - assert ( val->val.vector_val.size > 1 ); // implement empty vector... - - // all entries have to have the same type... - BroType* type = request_type->AsVectorType()->YieldType(); - VectorType* vt = new VectorType(type->Ref()); - VectorVal* v = new VectorVal(vt); - for ( int i = 0; i < val->val.vector_val.size; i++ ) { - v->Assign(i, LogValToVal( val->val.set_val.vals[i], type ), 0); - } - return v; + // all entries have to have the same type... + BroType* type = request_type->AsVectorType()->YieldType(); + VectorType* vt = new VectorType(type->Ref()); + VectorVal* v = new VectorVal(vt); + for ( int i = 0; i < val->val.vector_val.size; i++ ) { + v->Assign(i, LogValToVal( val->val.set_val.vals[i], type ), 0); + } + return v; } diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 22cdcfdcf0..4a0d4157bc 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -34,31 +34,19 @@ InputReaderAscii::InputReaderAscii() //keyMap = new map(); - separator_len = BifConst::LogAscii::separator->Len(); - separator = new char[separator_len]; - memcpy(separator, BifConst::LogAscii::separator->Bytes(), - separator_len); - if ( separator_len != 1 ) { + separator.assign( (const char*) BifConst::InputAscii::separator->Bytes(), BifConst::InputAscii::separator->Len()); + if ( separator.size() != 1 ) { Error("separator length has to be 1. Separator will be truncated."); } - set_separator_len = BifConst::LogAscii::set_separator->Len(); - set_separator = new char[set_separator_len]; - memcpy(set_separator, BifConst::LogAscii::set_separator->Bytes(), - set_separator_len); - if ( set_separator_len != 1 ) { + set_separator.assign( (const char*) BifConst::InputAscii::set_separator->Bytes(), BifConst::InputAscii::set_separator->Len()); + if ( set_separator.size() != 1 ) { Error("set_separator length has to be 1. Separator will be truncated."); } - empty_field_len = BifConst::LogAscii::empty_field->Len(); - empty_field = new char[empty_field_len]; - memcpy(empty_field, BifConst::LogAscii::empty_field->Bytes(), - empty_field_len); - - unset_field_len = BifConst::LogAscii::unset_field->Len(); - unset_field = new char[unset_field_len]; - memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(), - unset_field_len); + empty_field.assign( (const char*) BifConst::InputAscii::empty_field->Bytes(), BifConst::InputAscii::empty_field->Len()); + + unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), BifConst::InputAscii::unset_field->Len()); } @@ -66,10 +54,6 @@ InputReaderAscii::~InputReaderAscii() { DoFinish(); - delete [] separator; - delete [] set_separator; - delete [] empty_field; - delete [] unset_field; } void InputReaderAscii::DoFinish() @@ -172,7 +156,10 @@ bool InputReaderAscii::GetLine(string& str) { LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { LogVal* val = new LogVal(field.type, true); - //bzero(val, sizeof(LogVal)); + + if ( s.compare(unset_field) == 0 ) { // field is not set... + return new LogVal(field.type, false); + } switch ( field.type ) { case TYPE_ENUM: @@ -244,19 +231,13 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { if ( s[i] == ',') length++; unsigned int pos = 0; + + if ( s.compare(empty_field) == 0 ) + length = 0; LogVal** lvals = new LogVal* [length]; if ( field.type == TYPE_TABLE ) { - // construct a table from entry... - // for the moment assume, that entries are split by ",". - - /* Fix support for emtyp tables if ( s == "-" ) { - // empty - val->val.set_val.size = 0; - break; - } */ - val->val.set_val.vals = lvals; val->val.set_val.size = length; } else if ( field.type == TYPE_VECTOR ) { @@ -266,18 +247,20 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { assert(false); } + if ( length == 0 ) + break; //empty + istringstream splitstream(s); while ( splitstream ) { string element; - if ( pos >= length ) { - Error(Fmt("Internal error while parsing set. pos %d > length %d", pos, length)); - break; - } - if ( !getline(splitstream, element, set_separator[0]) ) break; - + + if ( pos >= length ) { + Error(Fmt("Internal error while parsing set. pos %d >= length %d. Element: %s", pos, length, element.c_str())); + break; + } LogVal* newval = EntryToVal(element, field.subType()); if ( newval == 0 ) { diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index d69b8c04bc..c848c17110 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -59,17 +59,13 @@ private: //map *keyMap; // // Options set from the script-level. - char* separator; - int separator_len; + string separator; - char* set_separator; - int set_separator_len; + string set_separator; - char* empty_field; - int empty_field_len; + string empty_field; - char* unset_field; - int unset_field_len; + string unset_field; }; From ec33453d2de671fbca75bfebdf348ef61f7adac0 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 17 Nov 2011 16:51:27 -0600 Subject: [PATCH 063/964] Add a "last updated date" to Broxygen docs. --- doc/conf.py.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/conf.py.in b/doc/conf.py.in index e997796eda..d33f235e14 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -59,7 +59,7 @@ release = '@VERSION@' # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -92,6 +92,8 @@ pygments_style = 'sphinx' # a list of builtin themes. html_theme = 'default' +html_last_updated_fmt = '%B %d, %Y' + # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. From 5227eb73c8854e194f620f1e8e9d10d8f9513e78 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 17 Nov 2011 16:55:51 -0600 Subject: [PATCH 064/964] Teach Broxygen :bro:see: role for referencing any identifier in Bro domain. --- doc/ext/bro.py | 41 ++++++++++++++----- doc/scripts/example.bro | 5 +++ .../doc.autogen-reST-example/example.rst | 5 +++ 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/doc/ext/bro.py b/doc/ext/bro.py index 7ec02f76a8..af2f1f888a 100644 --- a/doc/ext/bro.py +++ b/doc/ext/bro.py @@ -19,6 +19,11 @@ from docutils.parsers.rst import directives from docutils.parsers.rst.roles import set_classes class BroGeneric(ObjectDescription): + def update_type_map(self, idname): + if 'idtypes' not in self.env.domaindata['bro']: + self.env.domaindata['bro']['idtypes'] = {} + self.env.domaindata['bro']['idtypes'][idname] = self.objtype + def add_target_and_index(self, name, sig, signode): targetname = self.objtype + '-' + name if targetname not in self.state.document.ids: @@ -29,9 +34,6 @@ class BroGeneric(ObjectDescription): objects = self.env.domaindata['bro']['objects'] key = (self.objtype, name) -# this is commented out mostly just to avoid having a special directive -# for events in order to avoid the duplicate warnings in that case - """ if key in objects: self.env.warn(self.env.docname, 'duplicate description of %s %s, ' % @@ -39,8 +41,9 @@ class BroGeneric(ObjectDescription): 'other instance in ' + self.env.doc2path(objects[key]), self.lineno) - """ objects[key] = self.env.docname + self.update_type_map(name) + indextext = self.get_index_text(self.objtype, name) if indextext: self.indexnode['entries'].append(('single', indextext, @@ -65,6 +68,8 @@ class BroNamespace(BroGeneric): objects = self.env.domaindata['bro']['objects'] key = (self.objtype, name) objects[key] = self.env.docname + self.update_type_map(name) + indextext = self.get_index_text(self.objtype, name) self.indexnode['entries'].append(('single', indextext, targetname, targetname)) @@ -91,6 +96,8 @@ class BroEnum(BroGeneric): objects = self.env.domaindata['bro']['objects'] key = (self.objtype, name) objects[key] = self.env.docname + self.update_type_map(name) + indextext = self.get_index_text(self.objtype, name) #self.indexnode['entries'].append(('single', indextext, # targetname, targetname)) @@ -140,6 +147,7 @@ class BroDomain(Domain): 'id': XRefRole(), 'enum': XRefRole(), 'attr': XRefRole(), + 'see': XRefRole(), } initial_data = { @@ -154,13 +162,24 @@ class BroDomain(Domain): def resolve_xref(self, env, fromdocname, builder, typ, target, node, contnode): objects = self.data['objects'] - objtypes = self.objtypes_for_role(typ) - for objtype in objtypes: - if (objtype, target) in objects: - return make_refnode(builder, fromdocname, - objects[objtype, target], - objtype + '-' + target, - contnode, target + ' ' + objtype) + if typ == "see": + if target not in self.data['idtypes']: + self.env.warn(fromdocname, + 'unknown target for ":bro:see:`%s`' % (target)) + return [] + objtype = self.data['idtypes'][target] + return make_refnode(builder, fromdocname, + objects[objtype, target], + objtype + '-' + target, + contnode, target + ' ' + objtype) + else: + objtypes = self.objtypes_for_role(typ) + for objtype in objtypes: + if (objtype, target) in objects: + return make_refnode(builder, fromdocname, + objects[objtype, target], + objtype + '-' + target, + contnode, target + ' ' + objtype) def get_objects(self): for (typ, name), docname in self.data['objects'].iteritems(): diff --git a/doc/scripts/example.bro b/doc/scripts/example.bro index c239f2d2a2..f1b056c521 100644 --- a/doc/scripts/example.bro +++ b/doc/scripts/example.bro @@ -5,6 +5,11 @@ ##! (reST) document's summary section. ##! ##! .. tip:: You can embed directives and roles within ``##``-stylized comments. +##! +##! There's also a custom role to reference any identifier node in +##! the Bro Sphinx domain that's good for "see alsos", e.g. +##! "see also :bro:see:`Example::a_var`, :bro:see:`Example::ONE`, and +##! :bro:see:`SSH::Info`" # Comments that use a single pound sign (#) are not significant to # a script's auto-generated documentation, but ones that use a diff --git a/testing/btest/Baseline/doc.autogen-reST-example/example.rst b/testing/btest/Baseline/doc.autogen-reST-example/example.rst index b76b9af59b..5204c221c8 100644 --- a/testing/btest/Baseline/doc.autogen-reST-example/example.rst +++ b/testing/btest/Baseline/doc.autogen-reST-example/example.rst @@ -15,6 +15,11 @@ these comments are transferred directly into the auto-generated .. tip:: You can embed directives and roles within ``##``-stylized comments. +There's also a custom role to reference any identifier node in +the Bro Sphinx domain that's good for "see alsos", e.g. +"see also :bro:see:`Example::a_var`, :bro:see:`Example::ONE`, and +:bro:see:`SSH::Info`" + :Imports: :doc:`policy/frameworks/software/vulnerable ` Summary From e81cf3bb73c53eea60ae3a4e625fdc71ea590d02 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 17 Nov 2011 16:59:53 -0800 Subject: [PATCH 065/964] Adding some first doc strings to event.bif. Still trying to find the right style. --- src/event.bif | 188 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 186 insertions(+), 2 deletions(-) diff --git a/src/event.bif b/src/event.bif index d953ac78fe..1da2420083 100644 --- a/src/event.bif +++ b/src/event.bif @@ -1,35 +1,219 @@ +# Documentation conventions: +# +# - Short initial sentence (which doesn't need to be a sentence), starting +# with "Generated ..." +# +# - Use past tense for activity that has already occured. + +## Generated at Bro initialization time. The event engine generates this +## event just before normal input processing begins. It can be used to execute +## one-time initialization code at startup. At the time a handler runs, Bro will +## have executed any global initializations and statements. +## +## .. note:: When a ``bro_init`` handler executes, Bro has not yet seen any +## input packets and therefore :bro:id:`network_time` is not initialized yet. An +## artifact of that is that any timer installed in a ``bro_init`` handler will fire +## immediately with the first packet. The standard way to work around that is to +## ignore the first time the timer fires and immediately reschedule. +## +## see:: bro_done event bro_init%(%); + +## Generated at Bro termination time. The event engine generates this event when +## Bro is about to terminate, either due to having exhausted reading its input +## trace file(s), receiving a termination signal, or because Bro was run without +## a network input source and has finished executing any global statements. +## This event is generated after :bro:id:`net_done`, which gets only raised when +## processing network traffic. +## +## .. note:: If Bro terminates due to an invocation of :bro:id:`exit`, then this +## event is not generated. +## +## see:: net_done bro_init event bro_done%(%); +# TODO. event dns_mapping_valid%(dm: dns_mapping%); event dns_mapping_unverified%(dm: dns_mapping%); event dns_mapping_new_name%(dm: dns_mapping%); event dns_mapping_lost_name%(dm: dns_mapping%); -event dns_mapping_name_changed%(old_dm: dns_mapping, new_dm: dns_mapping%); +# event dns_mapping_name_changed%(old_dm: dns_mapping, new_dm: dns_mapping%); event dns_mapping_altered%(dm: dns_mapping, old_addrs: addr_set, new_addrs: addr_set%); +## Generated for every new connection. The event is raised with the first packet +## of a previously unknown connection. Bro uses a flow-based definition of +## "connection" here that includes not only TCP sessions but also UDP and ICMP +## flows. +## +## c: The connection. +## +## .. note:: Handling this event is potentially expensive. For example, during a +## SYN flooding attack, every spoofed SYN packet will lead to a new new_connection +## event. +## +## see:: *connection* event new_connection%(c: connection%); + +## Generated when reassembly starts for a TCP connection. The event is raised +## at the moment when Bro's TCP analyzer enables stream reassembly for a +## connection. +## +## c: The connection. +## +## see:: *connection* event new_connection_contents%(c: connection%); -event new_packet%(c: connection, p: pkt_hdr%); + +## Generated for an unsuccessful connection attempt. The event is raised when an +## originator unsuccessfully attempted to establish a connection. "Unsuccessful" +## is defined as at least :bro:id:`tcp_attempt_delay` seconds having elapsed since +## the originator first sent a connection establishment packet to the destination +## without seeing a reply. +## +## c: The connection. +## +## see:: connection_*, new_connection*, partial_connection. event connection_attempt%(c: connection%); + +## Generated for an established TCP connection. The event is raised when the +## initial 3-way TCP handshake has successfully finished for a connection. +## +## c: The connection. +## +## see:: *connection* event connection_established%(c: connection%); + +## Generated for a new active TCP connection if Bro did not see the initial +## handshake. The event is raised when Bro has observed traffic from each endpoint, +## but the activity did not begin with the usual connection establishment. +## +## c: The connection. +## +## see:: *connection* event partial_connection%(c: connection%); + +## TODO. +## +## see:: *connection* event connection_partial_close%(c: connection%); + +## Generated for a TCP connection that finished normally. The event is raised +## when a regular FIN handshake from both endpoints was observed. +## +## c: The connection. +## +## see:: *connection* event connection_finished%(c: connection%); + +## TODO. +## +## see:: *connection* event connection_half_finished%(c: connection%); + +## Generated for a rejected TCP connection. The event is raised when an originator +## attempted to setup a TCP connection but the responder replied with a RST packet +## denying it. +## +## c: The connection. +## +## .. note:: If the responder does not respond at all, :bro:id:`connection_attempt` +## is raised instead. If the responder initially accepts the connection but +## aborts it later, Bro first generates :bro:id:`connection_established` and then +## :bro:id:`connection_reset`. +## +## see:: *connection* event connection_rejected%(c: connection%); + +## Generated when an endpoint aborted a TCP connection. The event is raised +## when one endpoint of an established TCP connection aborted by sending a RST +## packet. +## +## c: The connection. +## +## see:: *connection* event connection_reset%(c: connection%); + +## TODO +## +## see:: *connection* event connection_pending%(c: connection%); + +## Generated when a connection's internal state is about to be removed from +## memory. Bro generates this event reliably once for every connection when it +## is about to delete the internal state. As such, the event is well-suited for +## scrip-level cleanup that needs to be performed for every connection. The +## ``connection_state_remove`` event is generated not only for TCP sessions but +## also for UDP and ICMP flows. +## +## c: The connection. +## +## see:: *connection* event connection_state_remove%(c: connection%); + +## Generated for a SYN packet. Bro raises this event for every SYN packet seen by +## its TCP analyzer. +## +## c: The connection. +## +## pkt: Information extracted from the SYN packet. +## +## .. note:: This event has quite low-level semantics and can potentially be +## expensive to generate. It should only be used if one really needs the +## specific information passed into the handler via the ``pkt`` argument. If +## not, handling one of the other ``connection_*`` events is typically the +## better approach. +## +## see:: *connection* event connection_SYN_packet%(c: connection, pkt: SYN_packet%); + +## Generated for the first ACK packet seen for a TCP connection from +## its *orginator*. +## +## c: The connection. +## +## pkt: Information extracted from the SYN packet. +## +## .. note:: This event has quite low-level semantics and should be used only +## rarely. +## +## see:: *connection* event connection_first_ACK%(c: connection%); + +## Generated when a TCP connection timed out. This event is raised when no activity +## was seen for an interval of at least :bro:id:`tcp_connection_linger`, and +## either one endpoint has already closed the connection or one side never +## never became active. +## +## c: The connection. +## +## .. note:: The precise semantics of this event can be unintuitive as it +## only covers a subset of cases where a connection times out. Often, handling +## :bro:id:`connection_state_remove` is the better option. That one will be +## generated reliably when an interval of ``tcp_inactivity_timeout`` has passed +## with out any activity seen (but also for all other ways a connection may +## terminate). +## +## see:: *connection* event connection_timeout%(c: connection%); + +## Generated when a connection 4-tuple is reused. The event is raised when Bro +## sees a new TCP session or UDP flow using a 4-tuple matching that of an earlier +## connection it still consideres active. +## +## c: The connection. +## +## see:: *connection* event connection_reused%(c: connection%); + + + event connection_status_update%(c: connection%); event connection_EOF%(c: connection, is_orig: bool%); event connection_external%(c: connection, tag: string%); + event expected_connection_seen%(c: connection, a: count%); +event new_packet%(c: connection, p: pkt_hdr%); + event protocol_confirmation%(c: connection, atype: count, aid: count%); event protocol_violation%(c: connection, atype: count, aid: count, reason: string%); From 4ee05d32db078ca2cf721e19b926359ea660c138 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 18 Nov 2011 11:20:25 -0600 Subject: [PATCH 066/964] Revert "Fix parallel make from top-level to work on more platforms" This reverts commit a92592d08d25eecd76972d6968e75228a85ff803. --- Makefile | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c5f4bf67ce..c736ecdbcb 100644 --- a/Makefile +++ b/Makefile @@ -11,11 +11,23 @@ VERSION_FULL=$(REPO)-`cat VERSION` VERSION_MIN=$(REPO)-`cat VERSION`-minimal HAVE_MODULES=git submodule | grep -v cmake >/dev/null -SUBDIRS = $(BUILD) -$(SUBDIRS):: configured - $(MAKE) -C $@ $(MAKECMDGOALS) +all: configured + ( cd $(BUILD) && make ) -all install install-aux doc docclean clean: $(SUBDIRS) +install: configured + ( cd $(BUILD) && make install ) + +install-aux: configured + ( cd $(BUILD) && make install-aux ) + +clean: configured docclean + ( cd $(BUILD) && make clean ) + +doc: configured + ( cd $(BUILD) && make doc ) + +docclean: configured + ( cd $(BUILD) && make docclean ) dist: @rm -rf $(VERSION_FULL) $(VERSION_FULL).tgz From 1ab266222ea73c9d6604505cfb5467a84159a8f5 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 18 Nov 2011 11:43:53 -0600 Subject: [PATCH 067/964] Fix parallel make portability --- Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index c736ecdbcb..e0c2860873 100644 --- a/Makefile +++ b/Makefile @@ -12,22 +12,22 @@ VERSION_MIN=$(REPO)-`cat VERSION`-minimal HAVE_MODULES=git submodule | grep -v cmake >/dev/null all: configured - ( cd $(BUILD) && make ) + $(MAKE) -C $(BUILD) $@ install: configured - ( cd $(BUILD) && make install ) + $(MAKE) -C $(BUILD) $@ install-aux: configured - ( cd $(BUILD) && make install-aux ) + $(MAKE) -C $(BUILD) $@ clean: configured docclean - ( cd $(BUILD) && make clean ) + $(MAKE) -C $(BUILD) $@ doc: configured - ( cd $(BUILD) && make doc ) + $(MAKE) -C $(BUILD) $@ docclean: configured - ( cd $(BUILD) && make docclean ) + $(MAKE) -C $(BUILD) $@ dist: @rm -rf $(VERSION_FULL) $(VERSION_FULL).tgz From e2c521fc4e6ef9eb66eb6cd431f9186468b14b70 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 18 Nov 2011 10:49:20 -0800 Subject: [PATCH 068/964] start reworking input framework... does not compile at the moment, but there are a few uncommitted changes that will be reverted in the next commit. --- scripts/base/frameworks/input/main.bro | 46 ++--- src/InputMgr.cc | 263 ++++++++++++------------- src/InputMgr.h | 18 +- src/InputReader.cc | 39 ++-- src/InputReader.h | 37 ++-- src/input.bif | 22 +-- 6 files changed, 208 insertions(+), 217 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 2b87ac980c..d9c0812498 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -4,30 +4,36 @@ module Input; export { const default_reader = READER_ASCII &redef; - type ReaderDescription: record { + type StreamDescription: record { source: string; - idx: any; - val: any; - destination: any; - want_record: bool &default=T; reader: Reader &default=default_reader; }; type Filter: record { - name: string; ## descriptive name. for later removal + name: string; + ## for tables + idx: any &optional; + val: any &optional; + destination: any &optional; + want_record: bool &default=T; + table_ev: any &optional; # event containing idx, val as values. + + ## decision function, that decides if an insertion, update or removal should really be executed. + ## or events should be thought pred: function(typ: Input::Event, left: any, right: any): bool &optional; - ## decision function, that decides if an inserton, update or removal should really be executed + + ## for "normalized" events + ev: any &optional; + ev_description: any &optional; }; const no_filter: Filter = [$name=""]; # Sentinel. - global create_reader: function(id: Log::ID, description: Input::ReaderDescription) : bool; - global remove_reader: function(id: Log::ID) : bool; + global create_stream: function(id: Log::ID, description: Input::ReaderDescription) : bool; + global remove_stream: function(id: Log::ID) : bool; global force_update: function(id: Log::ID) : bool; - global add_event: function(id: Log::ID, name: string) : bool; - global remove_event: function(id: Log::ID, name: string) : bool; global add_filter: function(id: Log::ID, filter: Input::Filter) : bool; global remove_filter: function(id: Log::ID, name: string) : bool; global get_filter: function(id: ID, name: string) : Filter; @@ -41,14 +47,14 @@ module Input; global filters: table[ID, string] of Filter; -function create_reader(id: Log::ID, description: Input::ReaderDescription) : bool +function create_stream(id: Log::ID, description: Input::ReaderDescription) : bool { - return __create_reader(id, description); + return __create_stream(id, description); } -function remove_reader(id: Log::ID) : bool +function remove_stream(id: Log::ID) : bool { - return __remove_reader(id); + return __remove_stream(id); } function force_update(id: Log::ID) : bool @@ -56,16 +62,6 @@ function force_update(id: Log::ID) : bool return __force_update(id); } -function add_event(id: Log::ID, name: string) : bool - { - return __add_event(id, name); - } - -function remove_event(id: Log::ID, name: string) : bool - { - return __remove_event(id, name); - } - function add_filter(id: Log::ID, filter: Input::Filter) : bool { filters[id, filter$name] = filter; diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 9c0e9c12b5..f9250f6f0f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -27,22 +27,11 @@ declare(PDict, InputHash); struct InputMgr::Filter { EnumVal* id; string name; - Func* pred; - ~Filter(); -}; - -InputMgr::Filter::~Filter() { - Unref(id); -} - -struct InputMgr::ReaderInfo { - EnumVal* id; - EnumVal* type; - InputReader* reader; unsigned int num_idx_fields; unsigned int num_val_fields; bool want_record; + EventHandlerPtr table_event; TableVal* tab; RecordType* rtype; @@ -50,18 +39,42 @@ struct InputMgr::ReaderInfo { PDict(InputHash)* currDict; PDict(InputHash)* lastDict; - - list events; // events we fire when "something" happens - list filters; // filters that can prevent our actions + + Func* pred; + + EventHandlerPtr event; + RecordType* event_type; + + ~Filter(); +}; + +InputMgr::Filter::~Filter() { + Unref(id); + if ( tab ) + Unref(tab); + if ( itype ) + Unref(itype); + if ( rtype ) + Unref(rtype); + if ( event_type) + Unref(event_type); +} + +struct InputMgr::ReaderInfo { + EnumVal* id; + EnumVal* type; + InputReader* reader; + + //list events; // events we fire when "something" happens + map filters; // filters that can prevent our actions ~ReaderInfo(); }; InputMgr::ReaderInfo::~ReaderInfo() { + // all the contents of filters should delete themselves automatically... + Unref(type); - Unref(tab); - Unref(itype); - Unref(rtype); Unref(id); delete(reader); @@ -86,14 +99,14 @@ InputMgr::InputMgr() } // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) +InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) { InputReaderDefinition* ir = input_readers; RecordType* rtype = description->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::ReaderDescription, 0) ) + if ( ! same_type(rtype, BifType::Record::Input::StreamDescription, 0) ) { - reporter->Error("readerDescription argument not of right type"); + reporter->Error("Streamdescription argument not of right type"); return 0; } @@ -145,55 +158,15 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); string source((const char*) bsource->Bytes(), bsource->Len()); - RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); - RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); - TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); - - - vector fieldsV; // vector, because we don't know the length beforehands - - - bool status = !UnrollRecordType(&fieldsV, idx, ""); - - int idxfields = fieldsV.size(); - - status = status || !UnrollRecordType(&fieldsV, val, ""); - int valfields = fieldsV.size() - idxfields; - - if ( status ) { - reporter->Error("Problem unrolling"); - Unref(reader); - return 0; - } - - Val *want_record = description->LookupWithDefault(rtype->FieldOffset("want_record")); - - LogField** fields = new LogField*[fieldsV.size()]; - for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { - fields[i] = fieldsV[i]; - } ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault - info->num_idx_fields = idxfields; - info->num_val_fields = valfields; - info->tab = dst->Ref()->AsTableVal(); - info->rtype = val->Ref()->AsRecordType(); info->id = id->Ref()->AsEnumVal(); - info->itype = idx->Ref()->AsRecordType(); - info->currDict = new PDict(InputHash); - info->lastDict = new PDict(InputHash); - info->want_record = ( want_record->InternalInt() == 1 ); - Unref(want_record); // ref'd by lookupwithdefault - - if ( valfields > 1 ) { - assert(info->want_record); - } readers.push_back(info); - int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); + int success = reader_obj->Init(source); if ( success == false ) { assert( RemoveReader(id) ); return 0; @@ -208,6 +181,86 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) } +bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->Error("Stream not found"); + return false; + } + + RecordType* rtype = fval->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::Filter, 0) ) + { + reporter->Error("filter argument not of right type"); + return false; + } + + + Val* name = fval->Lookup(rtype->FieldOffset("name")); + Val* pred = fval->Lookup(rtype->FieldOffset("pred")); + + RecordType *idx = fval->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); + RecordType *val = fval->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + TableVal *dst = fval->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); + + vector fieldsV; // vector, because we don't know the length beforehands + + bool status = !UnrollRecordType(&fieldsV, idx, ""); + + int idxfields = fieldsV.size(); + + status = status || !UnrollRecordType(&fieldsV, val, ""); + int valfields = fieldsV.size() - idxfields; + + if ( status ) { + reporter->Error("Problem unrolling"); + return false; + } + + Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); + + LogField** fields = new LogField*[fieldsV.size()]; + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { + fields[i] = fieldsV[i]; + } + + Filter filter; + filter.name = name->AsString()->CheckString(); + filter.id = id->Ref()->AsEnumVal(); + filter.pred = pred ? pred->AsFunc() : 0; + filter.num_idx_fields = idxfields; + filter.num_val_fields = valfields; + filter.tab = dst ? dst->Ref()->AsTableVal() : 0; + filter.rtype = rtype ? val->Ref()->AsRecordType() : 0; + filter.itype = itype ? idx->Ref()->AsRecordType() : 0; + // ya - well - we actually don't need them in every case... well, a few bytes of memory wasted + filter.currDict = new PDict(InputHash); + filter.lastDict = new PDict(InputHash); + filter.want_record = ( want_record->InternalInt() == 1 ); + Unref(want_record); // ref'd by lookupwithdefault + + if ( valfields > 1 ) { + assert(info->want_record); + } + + i->filters[id->InternalInt()] = filter; + + // ok, now we have to alert the reader of our new filter with our funky new fields + // the id is handled in a ... well, to be honest, a little bit sneaky manner. + // the "problem" is, that we can have several filters in the reader for one filter in the log manager. + // this is due to the fact, that a filter can either output it's result as a table, as an event... + // ...or as an table _and_ an event. And... if we have a table and an event, we actually need two different sets + // of filters in the reader, because the fields for the table and the event may differ and I absolutely do not want + // to build a union of these values and figure it out later. + // hence -> filter id is multiplicated with 2. + // filterId*2 -> results for table + // filterId*2+1 -> results for event + i->AddFilter( id->InternalInt() * 2, fieldsV.size(), idxfields, fields ); + + return true; +} + + bool InputMgr::IsCompatibleType(BroType* t, bool atomic_only) { if ( ! t ) @@ -258,7 +311,7 @@ bool InputMgr::IsCompatibleType(BroType* t, bool atomic_only) } -bool InputMgr::RemoveReader(const EnumVal* id) { +bool InputMgr::RemoveStream(const EnumVal* id) { ReaderInfo *i = 0; for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { @@ -281,42 +334,6 @@ bool InputMgr::RemoveReader(const EnumVal* id) { return true; } -bool InputMgr::RegisterEvent(const EnumVal* id, string eventName) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->InternalError("Reader not found"); - return false; - } - - i->events.push_back(eventName); - - return true; -} - -// remove first event with name eventName -// (though there shouldn't really be several events with the same name... -bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->InternalError("Reader not found"); - return false; - } - - std::list::iterator it = i->events.begin(); - while ( it != i->events.end() ) - { - if ( *it == eventName ) { - it = i->events.erase(it); - return true; - } - else - ++it; - } - - return false; -} - - bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { for ( int i = 0; i < rec->NumFields(); i++ ) { @@ -363,34 +380,6 @@ bool InputMgr::ForceUpdate(const EnumVal* id) return i->reader->Update(); } -bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->Error("Reader not found"); - return false; - } - - RecordType* rtype = fval->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::Filter, 0) ) - { - reporter->Error("filter argument not of right type"); - return false; - } - - - Val* name = fval->Lookup(rtype->FieldOffset("name")); - Val* pred = fval->Lookup(rtype->FieldOffset("pred")); - - Filter filter; - filter.name = name->AsString()->CheckString(); - filter.id = id->Ref()->AsEnumVal(); - filter.pred = pred ? pred->AsFunc() : 0; - - i->filters.push_back(filter); - - return true; -} - bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { @@ -398,7 +387,7 @@ bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { return false; } - +/* std::list::iterator it = i->filters.begin(); while ( it != i->filters.end() ) { @@ -410,8 +399,15 @@ bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { else ++it; } + */ - return false;; + map::iterator it = i->filters.find(id->InternalInt()); + if ( it == i->filters.end() ) { + return false; + } + + it->filters.erase(it); + return true; } @@ -444,7 +440,7 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo } -void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { +void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -605,7 +601,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { } -void InputMgr::EndCurrentSend(const InputReader* reader) { +void InputMgr::EndCurrentSend(const InputReader* reader, int id) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -693,7 +689,7 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { i->currDict = new PDict(InputHash); } -void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { +void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -733,7 +729,7 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { i->tab->Assign(idxval, valval); } -void InputMgr::Clear(const InputReader* reader) { +void InputMgr::Clear(const InputReader* reader, int id) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -873,7 +869,6 @@ int InputMgr::GetLogValLength(const LogVal* val) { case TYPE_VECTOR: { int j = val->val.vector_val.size; for ( int i = 0; i < j; i++ ) { - reporter->Error("size is %d", val->val.vector_val.size); length += GetLogValLength(val->val.vector_val.vals[i]); } break; diff --git a/src/InputMgr.h b/src/InputMgr.h index 17b7e2e804..1cacf89143 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -20,11 +20,9 @@ class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* id, RecordVal* description); + InputReader* CreateStream(EnumVal* id, RecordVal* description); bool ForceUpdate(const EnumVal* id); - bool RemoveReader(const EnumVal* id); - bool RegisterEvent(const EnumVal* id, string eventName); - bool UnregisterEvent(const EnumVal* id, string eventName); + bool RemoveStream(const EnumVal* id); bool AddFilter(EnumVal *id, RecordVal* filter); bool RemoveFilter(EnumVal* id, const string &name); @@ -36,12 +34,14 @@ protected: // Reports an error for the given reader. void Error(InputReader* reader, const char* msg); - void Put(const InputReader* reader, const LogVal* const *vals); - void Clear(const InputReader* reader); - bool Delete(const InputReader* reader, const LogVal* const *vals); + // for readers to write to input stream in direct mode (reporting new/deleted values directly) + void Put(const InputReader* reader, int id. const LogVal* const *vals); + void Clear(const InputReader* reader, int id); + bool Delete(const InputReader* reader, int id, const LogVal* const *vals); - void SendEntry(const InputReader* reader, const LogVal* const *vals); - void EndCurrentSend(const InputReader* reader); + // for readers to write to input stream in indirect mode (manager is monitoring new/deleted values) + void SendEntry(const InputReader* reader, int id, const LogVal* const *vals); + void EndCurrentSend(const InputReader* reader, int id); private: struct ReaderInfo; diff --git a/src/InputReader.cc b/src/InputReader.cc index 7403d1f989..1008cf1b67 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -24,35 +24,42 @@ void InputReader::Error(const string &msg) input_mgr->Error(this, msg.c_str()); } -void InputReader::Put(const LogVal* const *val) +void InputReader::Put(int id, const LogVal* const *val) { - input_mgr->Put(this, val); + input_mgr->Put(this, int id, val); } -void InputReader::Clear() +void InputReader::Clear(int id) { - input_mgr->Clear(this); + input_mgr->Clear(this, int id); } -void InputReader::Delete(const LogVal* const *val) +void InputReader::Delete(int id, const LogVal* const *val) { - input_mgr->Delete(this, val); + input_mgr->Delete(this, int id, val); } -bool InputReader::Init(string arg_source, int arg_num_fields, int arg_idx_fields, - const LogField* const * arg_fields) +bool InputReader::Init(string arg_source) { source = arg_source; - num_fields = arg_num_fields; - index_fields = arg_idx_fields; - fields = arg_fields; // disable if DoInit returns error. - disabled = !DoInit(arg_source, arg_num_fields, arg_idx_fields, arg_fields); + disabled = !DoInit(arg_source); return !disabled; } +bool InputReader::AddFilter(int id, int arg_num_fields, + const LogField* const * arg_fields) +{ + return DoAddFilter(int id, arg_num_fields, arg_fields); +} + +bool InputReader::RemoveFilter(int id) +{ + return DoRemoveFilter(int id); +} + void InputReader::Finish() { DoFinish(); @@ -96,12 +103,12 @@ const char* InputReader::Fmt(const char* format, ...) } -void InputReader::SendEntry(const LogVal* const *vals) +void InputReader::SendEntry(int id, const LogVal* const *vals) { - input_mgr->SendEntry(this, vals); + input_mgr->SendEntry(this, int id, vals); } -void InputReader::EndCurrentSend() +void InputReader::EndCurrentSend(int id) { - input_mgr->EndCurrentSend(this); + input_mgr->EndCurrentSend(this, int id); } diff --git a/src/InputReader.h b/src/InputReader.h index 41000e4c0c..12f0bc9db4 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -15,7 +15,11 @@ public: InputReader(); virtual ~InputReader(); - bool Init(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const* fields); + bool Init(string arg_source); + + bool AddFilter( int id, int arg_num_fields, const LogField* const* fields ); + + bool RemoveFilter ( int id ); void Finish(); @@ -23,8 +27,11 @@ public: protected: // Methods that have to be overwritten by the individual readers - virtual bool DoInit(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const * fields) = 0; - + virtual bool DoInit(string arg_sources) = 0; + + virtual bool DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ) = 0; + virtual bool DoRemoveFilter( int id ); + virtual void DoFinish() = 0; // update file contents to logmgr @@ -42,28 +49,26 @@ protected: //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); - void Put(const LogVal* const *val); - void Clear(); - void Delete(const LogVal* const *val); + // Content-sendinf-functions (simple mode). Including table-specific stuff that simply is not used if we have no table + void Put(int id, const LogVal* const *val); + void Delete(int id, const LogVal* const *val); + void Clear(int id); - void SendEntry(const LogVal* const *vals); - void EndCurrentSend(); + // Table-functions (tracking mode): Only changed lines are propagated. + void SendEntry(int id, const LogVal* const *vals); + void EndCurrentSend(int id); private: friend class InputMgr; string source; - int num_fields; - int index_fields; - const LogField* const * fields; - // When an error occurs, this method is called to set a flag marking the - // writer as disabled. + // When an error occurs, this method is called to set a flag marking the + // writer as disabled. - bool disabled; - - bool Disabled() { return disabled; } + bool disabled; + bool Disabled() { return disabled; } // For implementing Fmt(). char* buf; diff --git a/src/input.bif b/src/input.bif index aaef25dcc3..ef069316ab 100644 --- a/src/input.bif +++ b/src/input.bif @@ -7,18 +7,18 @@ module Input; #include "NetVar.h" %%} -type ReaderDescription: record; +type StreamDescription: record; type Filter: record; -function Input::__create_reader%(id: Log::ID, description: Input::ReaderDescription%) : bool +function Input::__create_stream%(id: Log::ID, description: Input::StreamDescription%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(id->AsEnumVal(), description->AsRecordVal()); + InputReader *the_reader = input_mgr->CreateStream(id->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} -function Input::__remove_reader%(id: Log::ID%) : bool +function Input::__remove_stream%(id: Log::ID%) : bool %{ - bool res = input_mgr->RemoveReader(id->AsEnumVal()); + bool res = input_mgr->RemoveStream(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} @@ -28,18 +28,6 @@ function Input::__force_update%(id: Log::ID%) : bool return new Val( res, TYPE_BOOL ); %} -function Input::__add_event%(id: Log::ID, name: string%) : bool - %{ - bool res = input_mgr->RegisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); - return new Val( res, TYPE_BOOL ); - %} - -function Input::__remove_event%(id: Log::ID, name: string%) : bool - %{ - bool res = input_mgr->UnregisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); - return new Val( res, TYPE_BOOL ); - %} - function Input::__add_filter%(id: Log::ID, filter: Input::Filter%) : bool %{ bool res = input_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); From 8d8f5dd0061839d5c6c8deb0ca86d50f06b3edd8 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 18 Nov 2011 13:58:23 -0600 Subject: [PATCH 069/964] Teach Broxygen to generate an index of Bro notices --- doc/ext/bro.py | 29 +++++++++++++++++++++++++++++ doc/index.rst | 7 ++++--- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/doc/ext/bro.py b/doc/ext/bro.py index af2f1f888a..5818038a4c 100644 --- a/doc/ext/bro.py +++ b/doc/ext/bro.py @@ -102,6 +102,11 @@ class BroEnum(BroGeneric): #self.indexnode['entries'].append(('single', indextext, # targetname, targetname)) m = sig.split() + if m[1] == "Notice::Type": + if 'notices' not in self.env.domaindata['bro']: + self.env.domaindata['bro']['notices'] = [] + self.env.domaindata['bro']['notices'].append( + (m[0], self.env.docname, targetname)) self.indexnode['entries'].append(('single', "%s (enum values); %s" % (m[1], m[0]), targetname, targetname)) @@ -120,6 +125,26 @@ class BroAttribute(BroGeneric): def get_index_text(self, objectname, name): return _('%s (attribute)') % (name) +class BroNotices(Index): + """ + Index subclass to provide the Bro notices index. + """ + + name = 'noticeindex' + localname = l_('Bro Notice Index') + shortname = l_('notices') + + def generate(self, docnames=None): + content = {} + for n in self.domain.env.domaindata['bro']['notices']: + modname = n[0].split("::")[0] + entries = content.setdefault(modname, []) + entries.append([n[0], 0, n[1], n[2], '', '', '']) + + content = sorted(content.iteritems()) + + return content, False + class BroDomain(Domain): """Bro domain.""" name = 'bro' @@ -150,6 +175,10 @@ class BroDomain(Domain): 'see': XRefRole(), } + indices = [ + BroNotices, + ] + initial_data = { 'objects': {}, # fullname -> docname, objtype } diff --git a/doc/index.rst b/doc/index.rst index 022552f3d4..ba3df81e7d 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -57,10 +57,11 @@ Other Bro Components components/pysubnettree/README components/trace-summary/README -Indices and tables ------------------- +Other Indices and References +---------------------------- -* :ref:`genindex` +* :ref:`General Index ` +* `Notice Index `_ * :ref:`search` Internal References From 05d107db17f04ec5fd90d315c246cd511cfb2782 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 18 Nov 2011 14:06:44 -0600 Subject: [PATCH 070/964] Change external Broxygen links to render same as internals. --- doc/conf.py.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/conf.py.in b/doc/conf.py.in index d33f235e14..72337b930f 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -100,7 +100,7 @@ html_last_updated_fmt = '%B %d, %Y' html_theme_options = { "rightsidebar": "true", "stickysidebar": "true", -"externalrefs": "true", +"externalrefs": "false", "footerbgcolor": "#333", "footertextcolor": "#ddd", "sidebarbgcolor": "#ffffff", From d3d09b798a710271bb99eb58ee0902467ab5af0f Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 18 Nov 2011 16:06:21 -0600 Subject: [PATCH 071/964] More Broxygen stylesheet tweaks. --- doc/_static/broxygen-extra.css | 3 + doc/_static/broxygen.css | 16 -- doc/_static/default.css_t | 309 +++++++++++++++++++++++++++++++++ doc/_static/download.js | 3 + doc/_templates/layout.html | 6 +- 5 files changed, 318 insertions(+), 19 deletions(-) create mode 100644 doc/_static/broxygen-extra.css delete mode 100644 doc/_static/broxygen.css create mode 100644 doc/_static/default.css_t create mode 100644 doc/_static/download.js diff --git a/doc/_static/broxygen-extra.css b/doc/_static/broxygen-extra.css new file mode 100644 index 0000000000..dd56416783 --- /dev/null +++ b/doc/_static/broxygen-extra.css @@ -0,0 +1,3 @@ +.highlight { + background-color: #ffffff; +} diff --git a/doc/_static/broxygen.css b/doc/_static/broxygen.css deleted file mode 100644 index d9c39a8506..0000000000 --- a/doc/_static/broxygen.css +++ /dev/null @@ -1,16 +0,0 @@ -a:hover -{ - text-decoration:none; - color:#c24444; -} - -div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 -{ -background-color:#ffffff; -border-bottom: 1px solid #aaa; -} - -th.field-name -{ -white-space:nowrap; -} diff --git a/doc/_static/default.css_t b/doc/_static/default.css_t new file mode 100644 index 0000000000..34c2157b25 --- /dev/null +++ b/doc/_static/default.css_t @@ -0,0 +1,309 @@ +/* + * default.css_t + * ~~~~~~~~~~~~~ + * + * Sphinx stylesheet -- default theme. + * + * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. + * :license: BSD, see LICENSE for details. + * + */ + +@import url("basic.css"); + +/* -- page layout ----------------------------------------------------------- */ + +body { + font-family: {{ theme_bodyfont }}; + font-size: 100%; + background-color: {{ theme_footerbgcolor }}; + color: #000; + margin: 0; + padding: 0; +} + +div.document { + background-color: {{ theme_sidebarbgcolor }}; +} + +div.documentwrapper { + float: left; + width: 100%; +} + +div.bodywrapper { + margin: 0 0 0 {{ theme_sidebarwidth|toint }}px; +} + +div.body { + background-color: {{ theme_bgcolor }}; + color: {{ theme_textcolor }}; + padding: 0 20px 30px 20px; +} + +{%- if theme_rightsidebar|tobool %} +div.bodywrapper { + margin: 0 {{ theme_sidebarwidth|toint }}px 0 0; +} +{%- endif %} + +div.footer { + color: {{ theme_footertextcolor }}; + background-color: {{ theme_footerbgcolor }}; + width: 100%; + padding: 9px 0 9px 0; + text-align: center; + font-size: 75%; +} + +div.footer a { + color: {{ theme_footertextcolor }}; + text-decoration: underline; +} + +div.related { + background-color: {{ theme_relbarbgcolor }}; + line-height: 30px; + color: {{ theme_relbartextcolor }}; +} + +div.related a { + color: {{ theme_relbarlinkcolor }}; +} + +div.sphinxsidebar { + {%- if theme_stickysidebar|tobool %} + top: 30px; + bottom: 0; + margin: 0; + position: fixed; + overflow: auto; + height: auto; + {%- endif %} + {%- if theme_rightsidebar|tobool %} + float: right; + {%- if theme_stickysidebar|tobool %} + right: 0; + {%- endif %} + {%- endif %} +} + +{%- if theme_stickysidebar|tobool %} +/* this is nice, but it it leads to hidden headings when jumping + to an anchor */ +/* +div.related { + position: fixed; +} + +div.documentwrapper { + margin-top: 30px; +} +*/ +{%- endif %} + +div.sphinxsidebar h3 { + font-family: {{ theme_bodyfont }}; + color: {{ theme_sidebartextcolor }}; + font-size: 1.4em; + font-weight: normal; + margin: 0; + padding: 0; +} + +div.sphinxsidebar h3 a { + color: {{ theme_sidebartextcolor }}; +} + +div.sphinxsidebar h4 { + font-family: {{ theme_bodyfont }}; + color: {{ theme_sidebartextcolor }}; + font-size: 1.3em; + font-weight: normal; + margin: 5px 0 0 0; + padding: 0; +} + +div.sphinxsidebar p { + color: {{ theme_sidebartextcolor }}; +} + +div.sphinxsidebar p.topless { + margin: 5px 10px 10px 10px; +} + +div.sphinxsidebar ul { + margin: 10px; + padding: 0; + color: {{ theme_sidebartextcolor }}; +} + +div.sphinxsidebar a { + color: {{ theme_sidebarlinkcolor }}; +} + +div.sphinxsidebar input { + border: 1px solid {{ theme_sidebarlinkcolor }}; + font-family: sans-serif; + font-size: 1em; +} + +{% if theme_collapsiblesidebar|tobool %} +/* for collapsible sidebar */ +div#sidebarbutton { + background-color: {{ theme_sidebarbtncolor }}; +} +{% endif %} + +/* -- hyperlink styles ------------------------------------------------------ */ + +a { + color: {{ theme_linkcolor }}; + text-decoration: none; +} + +a:visited { + color: {{ theme_visitedlinkcolor }}; + text-decoration: none; +} + +{% if theme_externalrefs|tobool %} +a.external { + text-decoration: none; + border-bottom: 1px dashed {{ theme_linkcolor }}; +} + +a.external:hover { + text-decoration: none; + border-bottom: none; +} + +a.external:visited { + text-decoration: none; + border-bottom: 1px dashed {{ theme_visitedlinkcolor }}; +} +{% endif %} + +/* -- body styles ----------------------------------------------------------- */ + +div.body h1, +div.body h2, +div.body h3, +div.body h4, +div.body h5, +div.body h6 { + font-family: {{ theme_bodyfont }}; + background-color: #ffffff; + font-weight: normal; + color: {{ theme_headtextcolor }}; + border-bottom: 1px solid #aaa; + margin: 20px -20px 10px -20px; + padding: 3px 0 3px 10px; +} + +div.body h1 { + font-family: {{ theme_headfont }}; + text-align: center; + border-bottom: none; +} + +div.body h1 { margin-top: 0; font-size: 200%; } +div.body h2 { font-size: 160%; } +div.body h3 { font-size: 140%; } +div.body h4 { font-size: 120%; } +div.body h5 { font-size: 110%; } +div.body h6 { font-size: 100%; } + +a.headerlink { + color: {{ theme_headlinkcolor }}; + font-size: 0.8em; + padding: 0 4px 0 4px; + text-decoration: none; +} + +a.headerlink:hover { + background-color: {{ theme_headlinkcolor }}; + color: white; +} + +div.admonition p.admonition-title + p { + display: inline; +} + +div.admonition p { + margin-bottom: 5px; +} + +div.admonition pre { + margin-bottom: 5px; +} + +div.admonition ul, div.admonition ol { + margin-bottom: 5px; +} + +div.note { + background-color: #eee; + border: 1px solid #ccc; +} + +div.seealso { + background-color: #ffc; + border: 1px solid #ff6; +} + +div.warning { + background-color: #ffe4e4; + border: 1px solid #f66; +} + +p.admonition-title { + display: inline; +} + +p.admonition-title:after { + content: ":"; +} + +pre { + padding: 5px; + background-color: {{ theme_codebgcolor }}; + color: {{ theme_codetextcolor }}; + line-height: 120%; + border: 1px solid #ac9; + border-left: none; + border-right: none; +} + +tt { + background-color: #ecf0f3; + padding: 0 1px 0 1px; + font-size: 0.95em; +} + +th { + background-color: #ede; +} + +.warning tt { + background: #efc2c2; +} + +.note tt { + background: #d6d6d6; +} + +.viewcode-back { + font-family: {{ theme_bodyfont }}; +} + +div.viewcode-block:target { + background-color: #f4debf; + border-top: 1px solid #ac9; + border-bottom: 1px solid #ac9; +} + +th.field-name +{ + white-space:nowrap; +} diff --git a/doc/_static/download.js b/doc/_static/download.js new file mode 100644 index 0000000000..82bfe502cb --- /dev/null +++ b/doc/_static/download.js @@ -0,0 +1,3 @@ +$(document).ready(function() { + $('.docutils.download').removeClass('download'); +}); diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html index 90c448037e..518718c55c 100644 --- a/doc/_templates/layout.html +++ b/doc/_templates/layout.html @@ -1,10 +1,10 @@ {% extends "!layout.html" %} -{% set css_files = css_files + ["_static/broxygen.css"] %} - {% block extrahead %} - + + + {% endblock %} {% block relbar2 %}{% endblock %} From ab30a3d866b3c178c2c164fd79c9404702645738 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 18 Nov 2011 14:47:28 -0800 Subject: [PATCH 072/964] Checkpointing event.bif Slowly working through ... --- src/event.bif | 379 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 336 insertions(+), 43 deletions(-) diff --git a/src/event.bif b/src/event.bif index 1da2420083..d565690811 100644 --- a/src/event.bif +++ b/src/event.bif @@ -1,22 +1,41 @@ # Documentation conventions: # -# - Short initial sentence (which doesn't need to be a sentence), starting -# with "Generated ..." -# # - Use past tense for activity that has already occured. +# +# - List parameters with an empty line in between. +# +# - Within the description, reference other parameters of teh same events +# with *xyz*. +# +# - Order: +# +# - Short initial sentence (which doesn't need to be a sentence), +# starting with "Generated ..." +# +# - Description. +# +# - Parameters +# +# - .. see:: +# +# - .. note:: + ## Generated at Bro initialization time. The event engine generates this ## event just before normal input processing begins. It can be used to execute ## one-time initialization code at startup. At the time a handler runs, Bro will ## have executed any global initializations and statements. ## -## .. note:: When a ``bro_init`` handler executes, Bro has not yet seen any -## input packets and therefore :bro:id:`network_time` is not initialized yet. An -## artifact of that is that any timer installed in a ``bro_init`` handler will fire +## see:: bro_done +## +## .. note:: +## +## When a ``bro_init`` handler executes, Bro has not yet seen any input packets +## and therefore :bro:id:`network_time` is not initialized yet. An artifact +## of that is that any timer installed in a ``bro_init`` handler will fire ## immediately with the first packet. The standard way to work around that is to ## ignore the first time the timer fires and immediately reschedule. ## -## see:: bro_done event bro_init%(%); ## Generated at Bro termination time. The event engine generates this event when @@ -26,10 +45,12 @@ event bro_init%(%); ## This event is generated after :bro:id:`net_done`, which gets only raised when ## processing network traffic. ## -## .. note:: If Bro terminates due to an invocation of :bro:id:`exit`, then this -## event is not generated. -## ## see:: net_done bro_init +## +## .. note:: +## +## If Bro terminates due to an invocation of :bro:id:`exit`, then this event is +## not generated. event bro_done%(%); # TODO. @@ -46,12 +67,14 @@ event dns_mapping_altered%(dm: dns_mapping, old_addrs: addr_set, new_addrs: addr ## flows. ## ## c: The connection. -## -## .. note:: Handling this event is potentially expensive. For example, during a -## SYN flooding attack, every spoofed SYN packet will lead to a new new_connection -## event. ## ## see:: *connection* +## +## .. note:: +## +## Handling this event is potentially expensive. For example, during a SYN +## flooding attack, every spoofed SYN packet will lead to a new new_connection +## event. event new_connection%(c: connection%); ## Generated when reassembly starts for a TCP connection. The event is raised @@ -112,15 +135,17 @@ event connection_half_finished%(c: connection%); ## Generated for a rejected TCP connection. The event is raised when an originator ## attempted to setup a TCP connection but the responder replied with a RST packet ## denying it. +## +## see:: *connection* ## ## c: The connection. ## -## .. note:: If the responder does not respond at all, :bro:id:`connection_attempt` -## is raised instead. If the responder initially accepts the connection but -## aborts it later, Bro first generates :bro:id:`connection_established` and then +## .. note:: +## +## If the responder does not respond at all, :bro:id:`connection_attempt` is +## raised instead. If the responder initially accepts the connection but aborts +## it later, Bro first generates :bro:id:`connection_established` and then ## :bro:id:`connection_reset`. -## -## see:: *connection* event connection_rejected%(c: connection%); ## Generated when an endpoint aborted a TCP connection. The event is raised @@ -156,13 +181,14 @@ event connection_state_remove%(c: connection%); ## ## pkt: Information extracted from the SYN packet. ## -## .. note:: This event has quite low-level semantics and can potentially be -## expensive to generate. It should only be used if one really needs the -## specific information passed into the handler via the ``pkt`` argument. If -## not, handling one of the other ``connection_*`` events is typically the -## better approach. -## ## see:: *connection* +## +## .. note:: +## +## This event has quite low-level semantics and can potentially be expensive to +## generate. It should only be used if one really needs the specific information +## passed into the handler via the ``pkt`` argument. If not, handling one of the +## other ``connection_*`` events is typically the better approach. event connection_SYN_packet%(c: connection, pkt: SYN_packet%); ## Generated for the first ACK packet seen for a TCP connection from @@ -171,11 +197,12 @@ event connection_SYN_packet%(c: connection, pkt: SYN_packet%); ## c: The connection. ## ## pkt: Information extracted from the SYN packet. -## -## .. note:: This event has quite low-level semantics and should be used only -## rarely. ## ## see:: *connection* +## +## .. note:: +## +## This event has quite low-level semantics and should be used only rarely. event connection_first_ACK%(c: connection%); ## Generated when a TCP connection timed out. This event is raised when no activity @@ -184,15 +211,17 @@ event connection_first_ACK%(c: connection%); ## never became active. ## ## c: The connection. +## +## see:: *connection* ## -## .. note:: The precise semantics of this event can be unintuitive as it -## only covers a subset of cases where a connection times out. Often, handling +## .. note:: +## +## The precise semantics of this event can be unintuitive as it only +## covers a subset of cases where a connection times out. Often, handling ## :bro:id:`connection_state_remove` is the better option. That one will be ## generated reliably when an interval of ``tcp_inactivity_timeout`` has passed ## with out any activity seen (but also for all other ways a connection may ## terminate). -## -## see:: *connection* event connection_timeout%(c: connection%); ## Generated when a connection 4-tuple is reused. The event is raised when Bro @@ -204,24 +233,292 @@ event connection_timeout%(c: connection%); ## see:: *connection* event connection_reused%(c: connection%); - - +## Generated in regular intervals during the life time of a connection. The +## events is raised each :bro:id:`connection_status_update_interval` seconds +## and can be used to check conditions on a regular basis. +## +## c: The connection. +## +## see:: *connection* event connection_status_update%(c: connection%); + +## Generated at the end of reassembled TCP connections. The TCP reassembler +## raised the event once for each endpoint of a connection when it finished +## reassembling the corresponding side of the communication. +## +## c: The connection. +## +## is_orig: True if the event is raised for the originator side. +## +## see:: *connection* event connection_EOF%(c: connection, is_orig: bool%); + +## Generated for a new connection received from the communication subsystem. Remote +## peers can inject packets into Bro's packet loop, for example via TODO_Broccoli. +## The communication systems raises this event with the first packet of a +## connection coming in this way. event connection_external%(c: connection, tag: string%); +## Generated when a connected is seen that has previously marked as being expected. +## The function :bro:id:`expect_connection` tells Bro to expect a particular +## connection to come up, and which analyzer to associate with it. Once the +## first packet of such a connection is indeed seen, this event is raised. +## +## c: The connection. +## +## a: The analyzer that was scheduled for the connection with the +## :bro:id:`expect_connection` call. When the event is raised, that +## analyzer will already have been activated to process the connection. The +## ``count`` is one of the ``ANALYZER_*`` constants, see TODO. +## +## see:: *connection* event expected_connection_seen%(c: connection, a: count%); +## Generated for every packet Bro sees. This is a very low-level and expensive +## event that should be avoided when at all possible. Is's usually infeasible to +## handle when processing even medium volumes of traffic in real-time. That +## said, if you work from a trace and want to do some packet-level analysis, +## it may come in handy. +## +## c: The connection the packet is part of. +## +## p: Informattion from the header of the packet that triggered the event. +## +## see:: tcp_packet packet_contents event new_packet%(c: connection, p: pkt_hdr%); +## Generated for every packet that has non-empty transport-layer payload. This is a +## very low-level and expensive event that should be avoided when at all possible. +## It's usually infeasible to handle when processing even medium volumes of +## traffic in real-time. It's even worse than :bro:id:`new_packet`. That said, if +## you work from a trace and want to do some packet-level analysis, it may come in +## handy. +## +## c: The connection the packet is part of. +## +## contants: The raw transport-layer payload. +## +## see:: new_packet tcp_packet +event packet_contents%(c: connection, contents: string%); + +## Generated for every TCP packet. This is a very low-level and expensive event +## that should be avoided when at all possible. It's usually infeasible to handle +## when processing even medium volumes of traffic in real-time. It's slightly +## better than :bro:id:`new_packet` because it affects only TCP, but not much. That +## said, if you work from a trace and want to do some packet-level analysis, it may +## come in handy. +## +## c: The connection the packet is part of. +## +## is_orig: True if the packet was sent by the connection's originator. +## +## flags: A string with the packet's TCP flags. In the string, each character +## corresponds to one set flag, as follows: ``S`` -> SYN; ``F`` -> FIN; +## ``R`` -> RST; ``A`` -> ACK; ``P`` -> PUSH. +## +## seq: The packet's TCP sequence number. +## +## ack: The packet's ACK number. +## +## len: The length of the TCP payload, as specified in the packet header. +## +## payload: The raw TCP payload. Note that this may less than *len* if the packet +## was not fully captured. +## +## see:: new_packet packet_content tcp_option tcp_contents tcp_rexmit +event tcp_packet%(c: connection, is_orig: bool, flags: string, seq: count, ack: count, len: count, payload: string%); + +## Generated for each option found in a TCP header. Like many of the ``tcp_*`` +## events, this is a very low-level event and potentially expensive as it may +## be raised very often. +## +## c: The connection the packet is part of. +## +## is_orig: True if the packet was sent by the connection's originator. +## +## opt: The numerical option number, as found in the TCP header. +## +## optlen: The length of the options value. +## +## see:: tcp_packet tcp_contents tcp_rexmit +## +## .. note:: There is currently no way to get the actual option value, if any. +event tcp_option%(c: connection, is_orig: bool, opt: count, optlen: count%); + +## Generated for each chunk of reassembled TCP payload. When reassembly is enabled +## for a TCP connection (as it is automatically whenever there is TCP protocol +## analyzer parsing it), this event is raised for each chunk of in-order payload +## reconstructed from the packet stream. Note that this event is potentially +## expensive if many connections carry signficant amounts of data as then all +## that needs to be passed on to the scripting layer. +## +## c: The connection the payload is part of. +## +## is_orig: True if the packet was sent by the connection's originator. +## +## seq: The sequence number corresponding to the first byte of the payload +## chunk. +## +## payload: The raw payload, which will be non-empty. +## +## see:: tcp_packet tcp_option tcp_rexmit +## +## .. note:: +## +## The payload received by this event is the same that is also passed into +## application-layer protocol analyzers internally. Subsequent invocations of +## this event for the same connection receive non-overlapping in-order chunks +## of its TCP payload stream. It is however undefined what size each chunk +## has; while Bro passes the data on as soon as possible, specifics depend +## on network-level effects such as latency, acknowledgements, reordering, +## etc. Also note that Bro's reassembler holds on to any data until it has seen +## the other side's acknowledgement. +event tcp_contents%(c: connection, is_orig: bool, seq: count, contents: string%); + +## Generated +event tcp_rexmit%(c: connection, is_orig: bool, seq: count, len: count, data_in_flight: count, window: count%); + +## Generated when Bro detects a TCP retransmission inconsistency. When +## reassemling TCP stream, Bro buffers all payload until it seens the responder +## acking it. If during time, the sender resends a chunk of payload but with +## content than originally, this event will be raised. +## +## c: The connection showing the inconsistency. +## +## t1: The original payload. +## +## t2: The new payload. +## +## see:: tcp_rexmit tcp_contents. +event rexmit_inconsistency%(c: connection, t1: string, t2: string%); + +## Generated when a TCP endpoint acknowledges payload that Bro did never see. +## +## c: The connection. +## +## see:: content_gap +## +## .. note:: +## +## Seeing an acknowledgment indicates that the responder of the connection +## says it has received the corresponding data. If Bro did not, it must have +## either missed one or more packets, or the responder's TCP stack is broken +## (which isn't unheard of). In practice, one will always see a few of these +## events in any larger volume of network traffic. If there are lots of them, +## however, that typically means that there is a problem with the monitoring +## infrastructure such as a tap dropping packets, split routing on the path, or +## reordering at the tap. +## +## This event reports similar situations as :bro:id:`content_gap`, though their +## specifics differ slightly. Often, however, both will be raised for the same +## connection if some of its data is missing. We should eventually merge +## the two. +event ack_above_hole%(c: connection%); + +## Generated when Bro detects a gap in a reassembled TCP payload stream. This event +## is raised when Bro, while reassemling a payload stream, determines that a chunk +## of payload is missing (e.g., because the responder has already acknowledged it, +## even though Bro didn't see it). +## +## c: The connection. +## +## is_orig: True if the gap is on the originator's side. +## +## seq: The sequence number where the gap starts. +## +## length: The number of bytes missing. +## +## see:: ack_above_hole +## +## .. note:: +## +## Content gaps tend to occur occasionally for various reasons, including broken +## TCP stacks. If, however, one finds lots of them, that typically means that +## there is a problem with the monitoring infrastructure such as a tap dropping +## packets, split routing on the path, or reordering at the tap. +## +## This event reports similar situations as :bro:id:`ack_above_hole`, though +## their specifics differ slightly. Often, however, both will be raised for +## connection if some of its data is missing. We should eventually merge the +## two. +event content_gap%(c: connection, is_orig: bool, seq: count, length: count%); + +## Summarizes the amount of missing TCP payload at regular intervals. Internally, +## Bro tracks (1) the number of :bro:id:`ack_above_hole` events, including the +## numer of bytes missing; and (2) the total number of TCP acks seen, with the +## total volume of bytes that have been acked. This event reports these statistics +## in :bro:id:`gap_report_freq` intervals for the purpose of determining packet +## loss. +## +## dt: The time that has past since the last ``gap_report`` interval. +## +## info: The gap statistics. +## +## see:: content_gap ack_above_hole +## +## .. note:: +## +## Bro comes with a script :doc:`/scripts/policy/misc/capture-loss.bro` that +## uses this event to estimate packet loss and report when a predefined +## threshold is exceeded. +event gap_report%(dt: interval, info: gap_info%); + + +## Generated when a protocol analyzer confirms that a connection is indeed +## using that protocol. Bro's TODO_dynamic protocol detection heuristically +## activates analyzers as soon as it believe a connection *could* be using a +## particular protocol. It is then left to the corresponding analyzer to +## verify whether that is indeed the case; if so, this event will be +## generated. +## +## c: The connection. +## +## atype: The type of the analyzer confirming that its protocol is in +## use. The value is one of the ``ANALYZER_*`` constants. For example, +## :bro:id:`ANALYZER_HTTP` means the HTTP analyzers determined that it's indeed +## parsing an HTTP connection. +## +## aid: A unique integer ID identifying the specific *instance* of the +## analyzer *atype* that is analyzing the connection ``c``. The ID can +## be used to reference the analyzer when using builtin functions like +## :bro:id:`disable_analyzer`. +## +## see:: protocol_violation +## +## .. note:: +## +## Bro's default scripts use this event to determine the ``service`` column of +## TODO``conn.log``: once confirmed, the protocol will be listed there. event protocol_confirmation%(c: connection, atype: count, aid: count%); + +## Generated when a protocol analyzer determines that a connection it is parsing is +## not conforming to the protocol it expects. Bro's TODO_dynamic protocol detection +## heuristically activates analyzers as soon as it believe a connection *could* be +## using a particular protocol. It is then left to the corresponding analyzer to +## verify whether that is indeed the case; if not, the analyzer will +## trigger this event. +## +## c: The connection. +## +## atype: The type of the analyzer confirming that its protocol is in +## use. The value is one of the ``ANALYZER_*`` constants. For example, +## :bro:id:`ANALYZER_HTTP` means the HTTP analyzers determined that it's indeed +## parsing an HTTP connection. +## +## aid: A unique integer ID identifying the specific *instance* of the +## analyzer *atype* that is analyzing the connection ``c``. The ID can +## be used to reference the analyzer when using builtin functions like +## :bro:id:`disable_analyzer`. +## +## see:: protocol_confirmation +## +## .. note:: +## +## Bro's default scripts use this event to disable an analyzer via +## :bro:id:`disable_analyzer` if it's parsing the wrong protocol. That's however +## a script-level decision and not done automatically by the event eninge. event protocol_violation%(c: connection, atype: count, aid: count, reason: string%); -event packet_contents%(c: connection, contents: string%); -event tcp_packet%(c: connection, is_orig: bool, flags: string, seq: count, ack: count, len: count, payload: string%); -event tcp_option%(c: connection, is_orig: bool, opt: count, optlen: count%); -event tcp_contents%(c: connection, is_orig: bool, seq: count, contents: string%); -event tcp_rexmit%(c: connection, is_orig: bool, seq: count, len: count, data_in_flight: count, window: count%); event udp_request%(u: connection%); event udp_reply%(u: connection%); event udp_contents%(u: connection, is_orig: bool, contents: string%); @@ -237,10 +534,6 @@ event conn_weird%(name: string, c: connection, addl: string%); event flow_weird%(name: string, src: addr, dst: addr%); event net_weird%(name: string%); event load_sample%(samples: load_sample_info, CPU: interval, dmem: int%); -event rexmit_inconsistency%(c: connection, t1: string, t2: string%); -event ack_above_hole%(c: connection%); -event content_gap%(c: connection, is_orig: bool, seq: count, length: count%); -event gap_report%(dt: interval, info: gap_info%); event inconsistent_option%(c: connection%); event bad_option%(c: connection%); event bad_option_termination%(c: connection%); From 305621672f679d98fd9a7ba055a301718d4481fe Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Sat, 19 Nov 2011 07:19:57 -0600 Subject: [PATCH 073/964] Teach Broxygen the .. bro:see:: directive --- doc/ext/bro.py | 50 ++++++++++++++++++- doc/scripts/example.bro | 9 +++- .../doc.autogen-reST-example/example.rst | 9 +++- 3 files changed, 63 insertions(+), 5 deletions(-) diff --git a/doc/ext/bro.py b/doc/ext/bro.py index 5818038a4c..a4f0142ce3 100644 --- a/doc/ext/bro.py +++ b/doc/ext/bro.py @@ -4,6 +4,9 @@ def setup(Sphinx): Sphinx.add_domain(BroDomain) + Sphinx.add_node(see) + Sphinx.add_directive_to_domain('bro', 'see', SeeDirective) + Sphinx.connect('doctree-resolved', process_see_nodes) from sphinx import addnodes from sphinx.domains import Domain, ObjType, Index @@ -18,6 +21,51 @@ from docutils.parsers.rst import Directive from docutils.parsers.rst import directives from docutils.parsers.rst.roles import set_classes +class see(nodes.General, nodes.Element): + refs = [] + +class SeeDirective(Directive): + has_content = True + + def run(self): + n = see('') + n.refs = string.split(string.join(self.content)) + return [n] + +def process_see_nodes(app, doctree, fromdocname): + for node in doctree.traverse(see): + content = [] + para = nodes.paragraph() + para += nodes.Text("See also:", "See also:") + for name in node.refs: + join_str = " " + if name != node.refs[0]: + join_str = ", " + link_txt = join_str + name; + + if name not in app.env.domaindata['bro']['idtypes']: + # Just create the text and issue warning + app.env.warn(fromdocname, + 'unknown target for ".. bro:see:: %s"' % (name)) + para += nodes.Text(link_txt, link_txt) + else: + # Create a reference + typ = app.env.domaindata['bro']['idtypes'][name] + todocname = app.env.domaindata['bro']['objects'][(typ, name)] + + newnode = nodes.reference('', '') + innernode = nodes.literal(_(name), _(name)) + newnode['refdocname'] = todocname + newnode['refuri'] = app.builder.get_relative_uri( + fromdocname, todocname) + newnode['refuri'] += '#' + typ + '-' + name + newnode.append(innernode) + para += nodes.Text(join_str, join_str) + para += newnode + + content.append(para) + node.replace_self(content) + class BroGeneric(ObjectDescription): def update_type_map(self, idname): if 'idtypes' not in self.env.domaindata['bro']: @@ -194,7 +242,7 @@ class BroDomain(Domain): if typ == "see": if target not in self.data['idtypes']: self.env.warn(fromdocname, - 'unknown target for ":bro:see:`%s`' % (target)) + 'unknown target for ":bro:see:`%s`"' % (target)) return [] objtype = self.data['idtypes'][target] return make_refnode(builder, fromdocname, diff --git a/doc/scripts/example.bro b/doc/scripts/example.bro index f1b056c521..d2d0ec6879 100644 --- a/doc/scripts/example.bro +++ b/doc/scripts/example.bro @@ -8,8 +8,13 @@ ##! ##! There's also a custom role to reference any identifier node in ##! the Bro Sphinx domain that's good for "see alsos", e.g. -##! "see also :bro:see:`Example::a_var`, :bro:see:`Example::ONE`, and -##! :bro:see:`SSH::Info`" +##! +##! See also: :bro:see:`Example::a_var`, :bro:see:`Example::ONE`, +##! :bro:see:`SSH::Info` +##! +##! And a custom directive does the equivalent references: +##! +##! .. bro:see:: Example::a_var Example::ONE SSH::Info # Comments that use a single pound sign (#) are not significant to # a script's auto-generated documentation, but ones that use a diff --git a/testing/btest/Baseline/doc.autogen-reST-example/example.rst b/testing/btest/Baseline/doc.autogen-reST-example/example.rst index 5204c221c8..880fa0e171 100644 --- a/testing/btest/Baseline/doc.autogen-reST-example/example.rst +++ b/testing/btest/Baseline/doc.autogen-reST-example/example.rst @@ -17,8 +17,13 @@ these comments are transferred directly into the auto-generated There's also a custom role to reference any identifier node in the Bro Sphinx domain that's good for "see alsos", e.g. -"see also :bro:see:`Example::a_var`, :bro:see:`Example::ONE`, and -:bro:see:`SSH::Info`" + +See also: :bro:see:`Example::a_var`, :bro:see:`Example::ONE`, +:bro:see:`SSH::Info` + +And a custom directive does the equivalent references: + +.. bro:see:: Example::a_var Example::ONE SSH::Info :Imports: :doc:`policy/frameworks/software/vulnerable ` From c8839da069339c9e32f859c801de8fa42ca4ae73 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sat, 19 Nov 2011 14:58:27 -0800 Subject: [PATCH 074/964] Updating submodule(s). [nomail] --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- cmake | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aux/binpac b/aux/binpac index 0f99acfbf6..34d9043740 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 0f99acfbf6205830f0db699a75554262c26427f9 +Subproject commit 34d90437403e4129468f89acce0bd1a99813a2f4 diff --git a/aux/bro-aux b/aux/bro-aux index 1a7a9357fb..7ea5837b4b 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 1a7a9357fba88a43c90a39d8d72b42fa53b89b75 +Subproject commit 7ea5837b4ba8403731ca4a9875616c0ab501342f diff --git a/aux/broccoli b/aux/broccoli index a1a03c6868..d281350dbc 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit a1a03c686866bd30ee086ff933128055a20ebd56 +Subproject commit d281350dbcc19c24aa6b6d89a4edc08a5c74a790 diff --git a/aux/broctl b/aux/broctl index 03d39aa5d4..ed4d4ce1ad 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 03d39aa5d4ab24cd9b8e404a9ceb583d5270444c +Subproject commit ed4d4ce1add51f0e08e6e8d2f5f247c2cbb422da diff --git a/cmake b/cmake index 44f2985475..f0f7958639 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 44f2985475e5ff6cc9061683e21ef4b184bdfc7e +Subproject commit f0f7958639bb921985c1f58f1186da4b49b5d54d From b3f01915fbca99ddda434ae791b143150d27fcf7 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 20 Nov 2011 12:07:50 -0800 Subject: [PATCH 075/964] compiles with basic new filter framework - but crashes on use. --- scripts/base/frameworks/input/main.bro | 16 +- src/InputMgr.cc | 255 +++++++++++------------- src/InputMgr.h | 2 +- src/InputReader.cc | 14 +- src/InputReader.h | 3 +- src/InputReaderAscii.cc | 265 ++++++++++++------------- src/InputReaderAscii.h | 30 ++- 7 files changed, 283 insertions(+), 302 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index d9c0812498..9d83d73ec6 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -14,9 +14,9 @@ export { name: string; ## for tables - idx: any &optional; - val: any &optional; - destination: any &optional; + idx: any; + val: any; + destination: any; want_record: bool &default=T; table_ev: any &optional; # event containing idx, val as values. @@ -25,13 +25,13 @@ export { pred: function(typ: Input::Event, left: any, right: any): bool &optional; ## for "normalized" events - ev: any &optional; - ev_description: any &optional; + # ev: any &optional; + # ev_description: any &optional; }; - const no_filter: Filter = [$name=""]; # Sentinel. + const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. - global create_stream: function(id: Log::ID, description: Input::ReaderDescription) : bool; + global create_stream: function(id: Log::ID, description: Input::StreamDescription) : bool; global remove_stream: function(id: Log::ID) : bool; global force_update: function(id: Log::ID) : bool; global add_filter: function(id: Log::ID, filter: Input::Filter) : bool; @@ -47,7 +47,7 @@ module Input; global filters: table[ID, string] of Filter; -function create_stream(id: Log::ID, description: Input::ReaderDescription) : bool +function create_stream(id: Log::ID, description: Input::StreamDescription) : bool { return __create_stream(id, description); } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index f9250f6f0f..403d656140 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -16,18 +16,20 @@ #include "CompHash.h" -class InputHash { -public: +struct InputHash { HashKey* valhash; HashKey* idxkey; // does not need ref or whatever - if it is present here, it is also still present in the TableVal. }; declare(PDict, InputHash); -struct InputMgr::Filter { +class InputMgr::Filter { +public: EnumVal* id; string name; + //int filter_type; // to distinguish between event and table filters + unsigned int num_idx_fields; unsigned int num_val_fields; bool want_record; @@ -68,6 +70,8 @@ struct InputMgr::ReaderInfo { //list events; // events we fire when "something" happens map filters; // filters that can prevent our actions + bool HasFilter(int id); + ~ReaderInfo(); }; @@ -80,6 +84,15 @@ InputMgr::ReaderInfo::~ReaderInfo() { delete(reader); } +bool InputMgr::ReaderInfo::HasFilter(int id) { + map::iterator it = filters.find(id); + if ( it == filters.end() ) { + return false; + } + return true; +} + + struct InputReaderDefinition { bro_int_t type; // the type const char *name; // descriptive name for error messages @@ -168,12 +181,12 @@ InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) int success = reader_obj->Init(source); if ( success == false ) { - assert( RemoveReader(id) ); + assert( RemoveStream(id) ); return 0; } success = reader_obj->Update(); if ( success == false ) { - assert ( RemoveReader(id) ); + assert ( RemoveStream(id) ); return 0; } @@ -224,6 +237,7 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { fields[i] = fieldsV[i]; } + // FIXME: remove those funky 0-tests again as the idea was changed. Filter filter; filter.name = name->AsString()->CheckString(); filter.id = id->Ref()->AsEnumVal(); @@ -231,8 +245,8 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { filter.num_idx_fields = idxfields; filter.num_val_fields = valfields; filter.tab = dst ? dst->Ref()->AsTableVal() : 0; - filter.rtype = rtype ? val->Ref()->AsRecordType() : 0; - filter.itype = itype ? idx->Ref()->AsRecordType() : 0; + filter.rtype = val ? val->Ref()->AsRecordType() : 0; + filter.itype = idx ? idx->Ref()->AsRecordType() : 0; // ya - well - we actually don't need them in every case... well, a few bytes of memory wasted filter.currDict = new PDict(InputHash); filter.lastDict = new PDict(InputHash); @@ -240,22 +254,11 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { Unref(want_record); // ref'd by lookupwithdefault if ( valfields > 1 ) { - assert(info->want_record); + assert(filter.want_record); } i->filters[id->InternalInt()] = filter; - - // ok, now we have to alert the reader of our new filter with our funky new fields - // the id is handled in a ... well, to be honest, a little bit sneaky manner. - // the "problem" is, that we can have several filters in the reader for one filter in the log manager. - // this is due to the fact, that a filter can either output it's result as a table, as an event... - // ...or as an table _and_ an event. And... if we have a table and an event, we actually need two different sets - // of filters in the reader, because the fields for the table and the event may differ and I absolutely do not want - // to build a union of these values and figure it out later. - // hence -> filter id is multiplicated with 2. - // filterId*2 -> results for table - // filterId*2+1 -> results for event - i->AddFilter( id->InternalInt() * 2, fieldsV.size(), idxfields, fields ); + i->reader->AddFilter( id->InternalInt(), fieldsV.size(), fields ); return true; } @@ -387,31 +390,15 @@ bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { return false; } -/* - std::list::iterator it = i->filters.begin(); - while ( it != i->filters.end() ) - { - if ( (*it).name == name ) { - it = i->filters.erase(it); - return true; - break; - } - else - ++it; - } - */ - map::iterator it = i->filters.find(id->InternalInt()); if ( it == i->filters.end() ) { return false; } - it->filters.erase(it); + i->filters.erase(it); return true; } - - Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const LogVal* const *vals) { Val* idxval; int position = 0; @@ -449,27 +436,28 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const bool updated = false; + assert(i->HasFilter(id)); //reporter->Error("Hashing %d index fields", i->num_idx_fields); - HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); + HashKey* idxhash = HashLogVals(i->filters[id].num_idx_fields, vals); //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); //reporter->Error("Hashing %d val fields", i->num_val_fields); - HashKey* valhash = HashLogVals(i->num_val_fields, vals+i->num_idx_fields); + HashKey* valhash = HashLogVals(i->filters[id].num_val_fields, vals+i->filters[id].num_idx_fields); //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); - InputHash *h = i->lastDict->Lookup(idxhash); + InputHash *h = i->filters[id].lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before if ( h->valhash->Hash() == valhash->Hash() ) { // ok, double. - i->lastDict->Remove(idxhash); - i->currDict->Insert(idxhash, h); + i->filters[id].lastDict->Remove(idxhash); + i->filters[id].currDict->Insert(idxhash, h); return; } else { // updated - i->lastDict->Remove(idxhash); + i->filters[id].lastDict->Remove(idxhash); delete(h); updated = true; @@ -477,27 +465,22 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const } - Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); Val* valval; - int position = i->num_idx_fields; - if ( i->num_val_fields == 1 && !i->want_record ) { - valval = LogValToVal(vals[i->num_idx_fields], i->rtype->FieldType(i->num_idx_fields)); + int position = i->filters[id].num_idx_fields; + if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { + valval = LogValToVal(vals[i->filters[id].num_idx_fields], i->filters[id].rtype->FieldType(i->filters[id].num_idx_fields)); } else { - RecordVal * r = new RecordVal(i->rtype); + RecordVal * r = new RecordVal(i->filters[id].rtype); - /* if ( i->rtype->NumFields() != (int) i->num_val_fields ) { - reporter->InternalError("Type mismatch"); - return; - } */ - - for ( int j = 0; j < i->rtype->NumFields(); j++) { + for ( int j = 0; j < i->filters[id].rtype->NumFields(); j++) { Val* val = 0; - if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); + if ( i->filters[id].rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, i->filters[id].rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->rtype->FieldType(j)); + val = LogValToVal(vals[position], i->filters[id].rtype->FieldType(j)); position++; } @@ -516,17 +499,12 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const Val* oldval = 0; if ( updated == true ) { // in that case, we need the old value to send the event (if we send an event). - oldval = i->tab->Lookup(idxval); + oldval = i->filters[id].tab->Lookup(idxval); } - // call filters first do determine if we really add / change the entry - std::list::iterator it = i->filters.begin(); - while ( it != i->filters.end() ) { - if (! (*it).pred ) { - continue; - } - + // call filter first to determine if we really add / change the entry + if ( i->filters[id].pred ) { EnumVal* ev; Ref(idxval); Ref(valval); @@ -541,44 +519,45 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const vl.append(ev); vl.append(idxval); vl.append(valval); - Val* v = (*it).pred->Call(&vl); + Val* v = i->filters[id].pred->Call(&vl); bool result = v->AsBool(); Unref(v); if ( result == false ) { if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... - delete(i->currDict->RemoveEntry(idxhash)); + delete(i->filters[id].currDict->RemoveEntry(idxhash)); return; } else { // keep old one - i->currDict->Insert(idxhash, h); + i->filters[id].currDict->Insert(idxhash, h); return; } } - ++it; } //i->tab->Assign(idxval, valval); - HashKey* k = i->tab->ComputeHash(idxval); + HashKey* k = i->filters[id].tab->ComputeHash(idxval); if ( !k ) { reporter->InternalError("could not hash"); return; } - i->tab->Assign(idxval, k, valval); + i->filters[id].tab->Assign(idxval, k, valval); InputHash* ih = new InputHash(); - k = i->tab->ComputeHash(idxval); + k = i->filters[id].tab->ComputeHash(idxval); ih->idxkey = k; ih->valhash = valhash; //i->tab->Delete(k); - i->currDict->Insert(idxhash, ih); + i->filters[id].currDict->Insert(idxhash, ih); // send events now that we are kind of finished. + + /* FIXME: fix me. std::list::iterator filter_iterator = i->events.begin(); while ( filter_iterator != i->events.end() ) { EnumVal* ev; @@ -597,7 +576,7 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const ++filter_iterator; - } + } */ } @@ -607,86 +586,74 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { reporter->InternalError("Unknown reader"); return; } + + assert(i->HasFilter(id)); + // lastdict contains all deleted entries and should be empty apart from that - IterCookie *c = i->lastDict->InitForIteration(); - i->lastDict->MakeRobustCookie(c); + IterCookie *c = i->filters[id].lastDict->InitForIteration(); + i->filters[id].lastDict->MakeRobustCookie(c); InputHash* ih; HashKey *lastDictIdxKey; //while ( ( ih = i->lastDict->NextEntry(c) ) ) { - while ( ( ih = i->lastDict->NextEntry(lastDictIdxKey, c) ) ) { - - if ( i->events.size() != 0 || i->filters.size() != 0 ) // we have a filter or an event - { + while ( ( ih = i->filters[id].lastDict->NextEntry(lastDictIdxKey, c) ) ) { - ListVal *idx = i->tab->RecoverIndex(ih->idxkey); + if ( i->filters[id].pred ) { + ListVal *idx = i->filters[id].tab->RecoverIndex(ih->idxkey); assert(idx != 0); - Val *val = i->tab->Lookup(idx); + Val *val = i->filters[id].tab->Lookup(idx); assert(val != 0); - { - bool doBreak = false; - // ask filter, if we want to expire this element... - std::list::iterator it = i->filters.begin(); - while ( it != i->filters.end() ) { - if (! (*it).pred ) { - continue; - } + bool doBreak = false; + // ask predicate, if we want to expire this element... - EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - Ref(idx); - Ref(val); + EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + Ref(idx); + Ref(val); - val_list vl(3); - vl.append(ev); - vl.append(idx); - vl.append(val); - Val* v = (*it).pred->Call(&vl); - bool result = v->AsBool(); - Unref(v); - - ++it; - - if ( result == false ) { - // Keep it. Hence - we quit and simply go to the next entry of lastDict - // ah well - and we have to add the entry to currDict... - i->currDict->Insert(lastDictIdxKey, i->lastDict->RemoveEntry(lastDictIdxKey)); - doBreak = true; - continue; - } - - } - - if ( doBreak ) { - continue; - } + val_list vl(3); + vl.append(ev); + vl.append(idx); + vl.append(val); + Val* v = i->filters[id].pred->Call(&vl); + bool result = v->AsBool(); + Unref(v); + + if ( result == false ) { + // Keep it. Hence - we quit and simply go to the next entry of lastDict + // ah well - and we have to add the entry to currDict... + i->filters[id].currDict->Insert(lastDictIdxKey, i->filters[id].lastDict->RemoveEntry(lastDictIdxKey)); + continue; } - + + // { - std::list::iterator it = i->events.begin(); - while ( it != i->events.end() ) { + /* FIXME: events + std::list::iterator it = i->filters[id].events.begin(); + while ( it != i->filters[id].events.end() ) { Ref(idx); Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); SendEvent(*it, ev, idx, val); ++it; } + */ } } - i->tab->Delete(ih->idxkey); - i->lastDict->Remove(lastDictIdxKey); // deletex in next line + i->filters[id].tab->Delete(ih->idxkey); + i->filters[id].lastDict->Remove(lastDictIdxKey); // deletex in next line delete(ih); } - i->lastDict->Clear(); // should be empty... but... well... who knows... - delete(i->lastDict); + i->filters[id].lastDict->Clear(); // should be empty... but... well... who knows... + delete(i->filters[id].lastDict); - i->lastDict = i->currDict; - i->currDict = new PDict(InputHash); + i->filters[id].lastDict = i->filters[id].currDict; + i->filters[id].currDict = new PDict(InputHash); } void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) { @@ -696,22 +663,24 @@ void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) return; } - Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + assert(i->HasFilter(id)); + + Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); Val* valval; - int position = i->num_idx_fields; - if ( i->num_val_fields == 1 && !i->want_record ) { - valval = LogValToVal(vals[i->num_idx_fields], i->rtype->FieldType(i->num_idx_fields)); + int position = i->filters[id].num_idx_fields; + if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { + valval = LogValToVal(vals[i->filters[id].num_idx_fields], i->filters[id].rtype->FieldType(i->filters[id].num_idx_fields)); } else { - RecordVal * r = new RecordVal(i->rtype); + RecordVal * r = new RecordVal(i->filters[id].rtype); - for ( int j = 0; j < i->rtype->NumFields(); j++) { + for ( int j = 0; j < i->filters[id].rtype->NumFields(); j++) { Val* val = 0; - if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); + if ( i->filters[id].rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, i->filters[id].rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->rtype->FieldType(j)); + val = LogValToVal(vals[position], i->filters[id].rtype->FieldType(j)); position++; } @@ -726,7 +695,7 @@ void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) valval = r; } - i->tab->Assign(idxval, valval); + i->filters[id].tab->Assign(idxval, valval); } void InputMgr::Clear(const InputReader* reader, int id) { @@ -735,20 +704,24 @@ void InputMgr::Clear(const InputReader* reader, int id) { reporter->InternalError("Unknown reader"); return; } - - i->tab->RemoveAll(); + + assert(i->HasFilter(id)); + + i->filters[id].tab->RemoveAll(); } -bool InputMgr::Delete(const InputReader* reader, const LogVal* const *vals) { +bool InputMgr::Delete(const InputReader* reader, int id, const LogVal* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); return false; } - - Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); - return ( i->tab->Delete(idxval) != 0 ); + assert(i->HasFilter(id)); + + Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); + + return ( i->filters[id].tab->Delete(idxval) != 0 ); } void InputMgr::Error(InputReader* reader, const char* msg) diff --git a/src/InputMgr.h b/src/InputMgr.h index 1cacf89143..5d531cd6fc 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -35,7 +35,7 @@ protected: void Error(InputReader* reader, const char* msg); // for readers to write to input stream in direct mode (reporting new/deleted values directly) - void Put(const InputReader* reader, int id. const LogVal* const *vals); + void Put(const InputReader* reader, int id, const LogVal* const *vals); void Clear(const InputReader* reader, int id); bool Delete(const InputReader* reader, int id, const LogVal* const *vals); diff --git a/src/InputReader.cc b/src/InputReader.cc index 1008cf1b67..1c65985fd6 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -26,17 +26,17 @@ void InputReader::Error(const string &msg) void InputReader::Put(int id, const LogVal* const *val) { - input_mgr->Put(this, int id, val); + input_mgr->Put(this, id, val); } void InputReader::Clear(int id) { - input_mgr->Clear(this, int id); + input_mgr->Clear(this, id); } void InputReader::Delete(int id, const LogVal* const *val) { - input_mgr->Delete(this, int id, val); + input_mgr->Delete(this, id, val); } @@ -52,12 +52,12 @@ bool InputReader::Init(string arg_source) bool InputReader::AddFilter(int id, int arg_num_fields, const LogField* const * arg_fields) { - return DoAddFilter(int id, arg_num_fields, arg_fields); + return DoAddFilter(id, arg_num_fields, arg_fields); } bool InputReader::RemoveFilter(int id) { - return DoRemoveFilter(int id); + return DoRemoveFilter(id); } void InputReader::Finish() @@ -105,10 +105,10 @@ const char* InputReader::Fmt(const char* format, ...) void InputReader::SendEntry(int id, const LogVal* const *vals) { - input_mgr->SendEntry(this, int id, vals); + input_mgr->SendEntry(this, id, vals); } void InputReader::EndCurrentSend(int id) { - input_mgr->EndCurrentSend(this, int id); + input_mgr->EndCurrentSend(this, id); } diff --git a/src/InputReader.h b/src/InputReader.h index 12f0bc9db4..6e3d689750 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -30,7 +30,8 @@ protected: virtual bool DoInit(string arg_sources) = 0; virtual bool DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ) = 0; - virtual bool DoRemoveFilter( int id ); + + virtual bool DoRemoveFilter( int id ) = 0; virtual void DoFinish() = 0; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 4a0d4157bc..84feb74e61 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -28,6 +28,7 @@ FieldMapping FieldMapping::subType() { return FieldMapping(name, subtype, position); } + InputReaderAscii::InputReaderAscii() { file = 0; @@ -58,7 +59,7 @@ InputReaderAscii::~InputReaderAscii() void InputReaderAscii::DoFinish() { - columnMap.empty(); + filters.empty(); if ( file != 0 ) { file->close(); delete(file); @@ -66,7 +67,7 @@ void InputReaderAscii::DoFinish() } } -bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const LogField* const * fields) +bool InputReaderAscii::DoInit(string path) { fname = path; @@ -76,11 +77,39 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const return false; } + return true; +} - this->num_fields = num_fields; - this->idx_fields = idx_fields; - this->fields = fields; +bool InputReaderAscii::DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ) { + if ( HasFilter(id) ) { + return false; // no, we don't want to add this a second time + } + Filter f; + f.num_fields = arg_num_fields; + f.fields = fields; + + filters[id] = f; + + return true; +} + +bool InputReaderAscii::DoRemoveFilter ( int id ) { + if (!HasFilter(id) ) { + return false; + } + + assert ( filters.erase(id) == 1 ); + + return true; +} + + +bool InputReaderAscii::HasFilter(int id) { + map::iterator it = filters.find(id); + if ( it == filters.end() ) { + return false; + } return true; } @@ -93,46 +122,47 @@ bool InputReaderAscii::ReadHeader() { return false; } - // split on tabs... - istringstream splitstream(line); - unsigned int currTab = 0; - int wantFields = 0; - while ( splitstream ) { - string s; - if ( !getline(splitstream, s, separator[0])) - break; - - // current found heading in s... compare if we want it - for ( unsigned int i = 0; i < num_fields; i++ ) { - const LogField* field = fields[i]; - if ( field->name == s ) { - // cool, found field. note position - FieldMapping f(field->name, field->type, field->subtype, i); - columnMap.push_back(f); - wantFields++; - break; // done with searching + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { + // split on tabs... + istringstream splitstream(line); + unsigned int currTab = 0; + int wantFields = 0; + while ( splitstream ) { + string s; + if ( !getline(splitstream, s, separator[0])) + break; + + // current found heading in s... compare if we want it + for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { + const LogField* field = (*it).second.fields[i]; + if ( field->name == s ) { + // cool, found field. note position + FieldMapping f(field->name, field->type, field->subtype, i); + (*it).second.columnMap.push_back(f); + wantFields++; + break; // done with searching + } } + + // look if we did push something... + if ( (*it).second.columnMap.size() == currTab ) { + // no, we didn't. note that... + FieldMapping empty; + (*it).second.columnMap.push_back(empty); + } + + // done + currTab++; + } + + if ( wantFields != (int) (*it).second.num_fields ) { + // we did not find all fields? + // :( + Error(Fmt("One of the requested fields could not be found in the input data file. Found %d fields, wanted %d. Filternum: %d", wantFields, (*it).second.num_fields, (*it).first)); + return false; } - - // look if we did push something... - if ( columnMap.size() == currTab ) { - // no, we didn't. note that... - FieldMapping empty; - columnMap.push_back(empty); - } - - // done - currTab++; - } - - if ( wantFields != (int) num_fields ) { - // we did not find all fields? - // :( - Error(Fmt("One of the requested fields could not be found in the input data file. Found %d fields, wanted %d", wantFields, num_fields)); - return false; } - // well, that seems to have worked... return true; } @@ -314,110 +344,77 @@ bool InputReaderAscii::DoUpdate() { return false; } - // TODO: all the stuff we need for a second reading. - // *cough* - // - // - - - // new keymap - //map *newKeyMap = new map(); - string line; while ( GetLine(line ) ) { - // split on tabs + + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - istringstream splitstream(line); - - LogVal** fields = new LogVal*[num_fields]; - //string string_fields[num_fields]; - - unsigned int currTab = 0; - unsigned int currField = 0; - while ( splitstream ) { - - string s; - if ( !getline(splitstream, s, separator[0]) ) - break; - + // split on tabs - if ( currTab >= columnMap.size() ) { - Error("Tabs in heading do not match tabs in data?"); - //disabled = true; - return false; - } - - FieldMapping currMapping = columnMap[currTab]; - currTab++; - - if ( currMapping.IsEmpty() ) { - // well, that was easy - continue; - } - - if ( currField >= num_fields ) { - Error("internal error - fieldnum greater as possible"); - return false; - } - - LogVal* val = EntryToVal(s, currMapping); - if ( val == 0 ) { - return false; - } - fields[currMapping.position] = val; - //string_fields[currMapping.position] = s; - - currField++; - } - - if ( currField != num_fields ) { - Error("curr_field != num_fields in DoUpdate. Columns in file do not match column definition."); - return false; - } - - - SendEntry(fields); - - /* - string indexstring = ""; - string valstring = ""; - for ( unsigned int i = 0; i < idx_fields; i++ ) { - indexstring.append(string_fields[i]); - } - - for ( unsigned int i = idx_fields; i < num_fields; i++ ) { - valstring.append(string_fields[i]); - } - - string valhash = Hash(valstring); - string indexhash = Hash(indexstring); - - if ( keyMap->find(indexhash) == keyMap->end() ) { - // new key - Put(fields); - } else if ( (*keyMap)[indexhash] != valhash ) { - // changed key - Put(fields); - keyMap->erase(indexhash); - } else { - // field not changed - keyMap->erase(indexhash); - } - - - (*newKeyMap)[indexhash] = valhash; - */ + istringstream splitstream(line); - for ( unsigned int i = 0; i < num_fields; i++ ) { - delete fields[i]; + LogVal** fields = new LogVal*[(*it).second.num_fields]; + //string string_fields[num_fields]; + + unsigned int currTab = 0; + unsigned int currField = 0; + while ( splitstream ) { + + string s; + if ( !getline(splitstream, s, separator[0]) ) + break; + + + if ( currTab >= (*it).second.columnMap.size() ) { + Error("Tabs in heading do not match tabs in data?"); + //disabled = true; + return false; + } + + FieldMapping currMapping = (*it).second.columnMap[currTab]; + currTab++; + + if ( currMapping.IsEmpty() ) { + // well, that was easy + continue; + } + + if ( currField >= (*it).second.num_fields ) { + Error("internal error - fieldnum greater as possible"); + return false; + } + + LogVal* val = EntryToVal(s, currMapping); + if ( val == 0 ) { + return false; + } + fields[currMapping.position] = val; + //string_fields[currMapping.position] = s; + + currField++; + } + + if ( currField != (*it).second.num_fields ) { + Error("curr_field != num_fields in DoUpdate. Columns in file do not match column definition."); + return false; + } + + + SendEntry((*it).first, fields); + + for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { + delete fields[i]; + } + delete [] fields; } - delete [] fields; } //file->clear(); // remove end of file evil bits //file->seekg(0, ios::beg); // and seek to start. - EndCurrentSend(); + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { + EndCurrentSend((*it).first); + } return true; } diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index c848c17110..01169a3cfc 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -34,13 +34,30 @@ public: protected: - virtual bool DoInit(string path, int arg_num_fields, int arg_idx_fields, - const LogField* const * fields); + virtual bool DoInit(string path); + + virtual bool DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ); + + virtual bool DoRemoveFilter ( int id ); + virtual void DoFinish(); virtual bool DoUpdate(); private: + + struct Filter { + unsigned int num_fields; + + const LogField* const * fields; // raw mapping + + // map columns in the file to columns to send back to the manager + vector columnMap; + + }; + + bool HasFilter(int id); + bool ReadHeader(); LogVal* EntryToVal(string s, FieldMapping type); @@ -49,15 +66,8 @@ private: ifstream* file; string fname; - unsigned int num_fields; - unsigned int idx_fields; + map filters; - // map columns in the file to columns to send back to the manager - vector columnMap; - const LogField* const * fields; // raw mapping - - //map *keyMap; - // // Options set from the script-level. string separator; From 7eb4d9934168fa0d37e01e3dbdd6feb60a88c485 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 20 Nov 2011 12:27:34 -0800 Subject: [PATCH 076/964] very basic functionality kind of works again --- src/InputMgr.cc | 52 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 403d656140..b51cd6fd27 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -47,10 +47,50 @@ public: EventHandlerPtr event; RecordType* event_type; - ~Filter(); + // ~Filter(); + // Filter(); + // Filter(const InputMgr::Filter& filter); + + void DoCleanup(); }; -InputMgr::Filter::~Filter() { +/* +InputMgr::Filter::Filter() { + tab = 0; + itype = 0; + rtype = 0; + event_type = 0; +} + +InputMgr::Filter::Filter(const InputMgr::Filter& f) { + id = f.id; + id->Ref(); + + tab = f.tab; + if ( tab ) + tab->Ref(); + + itype = f.itype; + if ( itype ) + itype->Ref(); + + rtype = f.rtype; + if ( rtype ) + Ref(rtype); + + event_type = f.event_type; + if ( event_type ) + Ref(event_type); + + name = f.name; + num_idx_fields = f.num_idx_fields; + num_val_fields = f.num_val_fields; + want_record = f.want_record; + + +} */ + +void InputMgr::Filter::DoCleanup() { Unref(id); if ( tab ) Unref(tab); @@ -60,7 +100,10 @@ InputMgr::Filter::~Filter() { Unref(rtype); if ( event_type) Unref(event_type); -} + + delete currDict; + delete lastDict; +} struct InputMgr::ReaderInfo { EnumVal* id; @@ -395,6 +438,8 @@ bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { return false; } + i->filters[id->InternalInt()].DoCleanup(); + i->filters.erase(it); return true; } @@ -545,6 +590,7 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const return; } + reporter->Error("assigning"); i->filters[id].tab->Assign(idxval, k, valval); InputHash* ih = new InputHash(); From 029871e48c4c63add07d63a52127bb7b50f47189 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 20 Nov 2011 13:42:02 -0800 Subject: [PATCH 077/964] first test. --- .../scripts.base.frameworks.input.basic/out | 14 +++++ .../scripts/base/frameworks/input/basic.bro | 52 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.basic/out create mode 100644 testing/btest/scripts/base/frameworks/input/basic.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.basic/out b/testing/btest/Baseline/scripts.base.frameworks.input.basic/out new file mode 100644 index 0000000000..ebac1866b6 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.basic/out @@ -0,0 +1,14 @@ +{ +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro new file mode 100644 index 0000000000..5e0c7be12e --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -0,0 +1,52 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type idx: record { + i: int; +}; + +type val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_filter(A::LOG, [$name="ssh", $idx=idx, $val=val, $destination=servers]); + Input::force_update(A::LOG); + print servers; +} From 0325b5ea32f9b86c89ca020097de4586d782f636 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Sun, 20 Nov 2011 21:41:41 -0800 Subject: [PATCH 078/964] to_port() now parses a string instead of a count. Addresses #684. --- scripts/base/protocols/irc/dcc-send.bro | 2 +- src/bro.bif | 22 ++++++++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/scripts/base/protocols/irc/dcc-send.bro b/scripts/base/protocols/irc/dcc-send.bro index b2a48a472a..669cc03e55 100644 --- a/scripts/base/protocols/irc/dcc-send.bro +++ b/scripts/base/protocols/irc/dcc-send.bro @@ -99,7 +99,7 @@ event irc_dcc_message(c: connection, is_orig: bool, return; c$irc$dcc_file_name = argument; c$irc$dcc_file_size = size; - local p = to_port(dest_port, tcp); + local p = count_to_port(dest_port, tcp); expect_connection(to_addr("0.0.0.0"), address, p, ANALYZER_FILE, 5 min); dcc_expected_transfers[address, p] = c$irc; } diff --git a/src/bro.bif b/src/bro.bif index a2f97356a7..6d4d7ce1dd 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -586,9 +586,27 @@ function raw_bytes_to_v4_addr%(b: string%): addr return new AddrVal(htonl(a)); %} -function to_port%(num: count, proto: transport_proto%): port +function to_port%(s: string%): port %{ - return new PortVal(num, (TransportProto)proto->AsEnum()); + int port = 0; + if ( s->Len() < 10 ) + { + char* slash; + port = strtol(s->CheckString(), &slash, 10); + if ( port ) + { + ++slash; + if ( streq(slash, "tcp") ) + return new PortVal(port, TRANSPORT_TCP); + else if ( streq(slash, "udp") ) + return new PortVal(port, TRANSPORT_UDP); + else if ( streq(slash, "icmp") ) + return new PortVal(port, TRANSPORT_ICMP); + } + } + + builtin_error("wrong port format, must be /[0-9]{1,5}\\/(tcp|udp|icmp)/"); + return new PortVal(port, TRANSPORT_UNKNOWN); %} function mask_addr%(a: addr, top_bits_to_keep: count%): subnet From f0e5303330d9842a7454001276ef5857c252bba9 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 15:09:00 -0800 Subject: [PATCH 079/964] make want_record field for tablefilter work... --- src/InputMgr.cc | 2 +- .../out | 3 ++ .../out | 3 ++ .../frameworks/input/onecolumn-norecord.bro | 38 +++++++++++++++++++ .../frameworks/input/onecolumn-record.bro | 38 +++++++++++++++++++ 5 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out create mode 100644 testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro create mode 100644 testing/btest/scripts/base/frameworks/input/onecolumn-record.bro diff --git a/src/InputMgr.cc b/src/InputMgr.cc index b51cd6fd27..c4180f4f8d 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -515,7 +515,7 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const int position = i->filters[id].num_idx_fields; if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { - valval = LogValToVal(vals[i->filters[id].num_idx_fields], i->filters[id].rtype->FieldType(i->filters[id].num_idx_fields)); + valval = LogValToVal(vals[position], i->filters[id].rtype->FieldType(0)); } else { RecordVal * r = new RecordVal(i->filters[id].rtype); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out new file mode 100644 index 0000000000..bbce48f4f6 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out @@ -0,0 +1,3 @@ +{ +[-42] = T +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out new file mode 100644 index 0000000000..3f9af35c59 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out @@ -0,0 +1,3 @@ +{ +[-42] = [b=T] +} diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro new file mode 100644 index 0000000000..74fd477e28 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -0,0 +1,38 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i +#types bool int +T -42 +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type idx: record { + i: int; +}; + +type val: record { + b: bool; +}; + +global servers: table[int] of val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_filter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F]); + Input::force_update(A::LOG); + print servers; +} diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro new file mode 100644 index 0000000000..3cc7090462 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -0,0 +1,38 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i +#types bool int +T -42 +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type idx: record { + i: int; +}; + +type val: record { + b: bool; +}; + +global servers: table[int] of val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_filter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers]); + Input::force_update(A::LOG); + print servers; +} From 18591b53d422e5b97dc8595a5171e50a64feec78 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 15:20:52 -0800 Subject: [PATCH 080/964] rename filter to tablefilter in preparation of event filters... --- scripts/base/frameworks/input/main.bro | 38 +++++++++---------- src/InputMgr.cc | 6 +-- src/InputMgr.h | 4 +- src/input.bif | 10 ++--- testing/btest/btest.cfg | 2 +- .../scripts/base/frameworks/input/basic.bro | 2 +- .../frameworks/input/onecolumn-norecord.bro | 2 +- .../frameworks/input/onecolumn-record.bro | 2 +- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 9d83d73ec6..4560421ecc 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -9,7 +9,7 @@ export { reader: Reader &default=default_reader; }; - type Filter: record { + type TableFilter: record { ## descriptive name. for later removal name: string; @@ -29,14 +29,14 @@ export { # ev_description: any &optional; }; - const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. + #const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. global create_stream: function(id: Log::ID, description: Input::StreamDescription) : bool; global remove_stream: function(id: Log::ID) : bool; global force_update: function(id: Log::ID) : bool; - global add_filter: function(id: Log::ID, filter: Input::Filter) : bool; - global remove_filter: function(id: Log::ID, name: string) : bool; - global get_filter: function(id: ID, name: string) : Filter; + global add_tablefilter: function(id: Log::ID, filter: Input::TableFilter) : bool; + global remove_tablefilter: function(id: Log::ID, name: string) : bool; + #global get_filter: function(id: ID, name: string) : Filter; } @@ -45,7 +45,7 @@ export { module Input; -global filters: table[ID, string] of Filter; +#global filters: table[ID, string] of Filter; function create_stream(id: Log::ID, description: Input::StreamDescription) : bool { @@ -62,22 +62,22 @@ function force_update(id: Log::ID) : bool return __force_update(id); } -function add_filter(id: Log::ID, filter: Input::Filter) : bool +function add_tablefilter(id: Log::ID, filter: Input::TableFilter) : bool { - filters[id, filter$name] = filter; - return __add_filter(id, filter); +# filters[id, filter$name] = filter; + return __add_tablefilter(id, filter); } -function remove_filter(id: Log::ID, name: string) : bool +function remove_tablefilter(id: Log::ID, name: string) : bool { - delete filters[id, name]; - return __remove_filter(id, name); +# delete filters[id, name]; + return __remove_tablefilter(id, name); } -function get_filter(id: ID, name: string) : Filter - { - if ( [id, name] in filters ) - return filters[id, name]; - - return no_filter; - } +#function get_filter(id: ID, name: string) : Filter +# { +# if ( [id, name] in filters ) +# return filters[id, name]; +# +# return no_filter; +# } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index c4180f4f8d..8fda5d506f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -237,7 +237,7 @@ InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) } -bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { +bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { reporter->Error("Stream not found"); @@ -245,7 +245,7 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { } RecordType* rtype = fval->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::Filter, 0) ) + if ( ! same_type(rtype, BifType::Record::Input::TableFilter, 0) ) { reporter->Error("filter argument not of right type"); return false; @@ -426,7 +426,7 @@ bool InputMgr::ForceUpdate(const EnumVal* id) return i->reader->Update(); } -bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { +bool InputMgr::RemoveTableFilter(EnumVal* id, const string &name) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { reporter->Error("Reader not found"); diff --git a/src/InputMgr.h b/src/InputMgr.h index 5d531cd6fc..4280ba1d81 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -24,8 +24,8 @@ public: bool ForceUpdate(const EnumVal* id); bool RemoveStream(const EnumVal* id); - bool AddFilter(EnumVal *id, RecordVal* filter); - bool RemoveFilter(EnumVal* id, const string &name); + bool AddTableFilter(EnumVal *id, RecordVal* filter); + bool RemoveTableFilter(EnumVal* id, const string &name); protected: diff --git a/src/input.bif b/src/input.bif index ef069316ab..1300f91bea 100644 --- a/src/input.bif +++ b/src/input.bif @@ -8,7 +8,7 @@ module Input; %%} type StreamDescription: record; -type Filter: record; +type TableFilter: record; function Input::__create_stream%(id: Log::ID, description: Input::StreamDescription%) : bool %{ @@ -28,15 +28,15 @@ function Input::__force_update%(id: Log::ID%) : bool return new Val( res, TYPE_BOOL ); %} -function Input::__add_filter%(id: Log::ID, filter: Input::Filter%) : bool +function Input::__add_tablefilter%(id: Log::ID, filter: Input::TableFilter%) : bool %{ - bool res = input_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); + bool res = input_mgr->AddTableFilter(id->AsEnumVal(), filter->AsRecordVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_filter%(id: Log::ID, name: string%) : bool +function Input::__remove_tablefilter%(id: Log::ID, name: string%) : bool %{ - bool res = input_mgr->RemoveFilter(id->AsEnumVal(), name->AsString()->CheckString()); + bool res = input_mgr->RemoveTableFilter(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL); %} diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 7d8283587c..739d0b2ad4 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -3,7 +3,7 @@ TestDirs = doc bifs language core scripts istate coverage TmpDir = %(testbase)s/.tmp BaselineDir = %(testbase)s/Baseline IgnoreDirs = .svn CVS .tmp -IgnoreFiles = *.tmp *.swp #* *.trace +IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store [environment] BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev` diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 5e0c7be12e..139888fa7c 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -46,7 +46,7 @@ event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_filter(A::LOG, [$name="ssh", $idx=idx, $val=val, $destination=servers]); + Input::add_tablefilter(A::LOG, [$name="ssh", $idx=idx, $val=val, $destination=servers]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 74fd477e28..12dbdd42aa 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -32,7 +32,7 @@ event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_filter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index 3cc7090462..4eef12d752 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -32,7 +32,7 @@ event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_filter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers]); Input::force_update(A::LOG); print servers; } From 92b3723b0947739fb4aff9a3d7a87662f65011c7 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 15:36:03 -0800 Subject: [PATCH 081/964] add very basic predicate test. --- .../out | 7 ++ .../base/frameworks/input/predicate.bro | 66 +++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.predicate/out create mode 100644 testing/btest/scripts/base/frameworks/input/predicate.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.predicate/out b/testing/btest/Baseline/scripts.base.frameworks.input.predicate/out new file mode 100644 index 0000000000..d805f804d8 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.predicate/out @@ -0,0 +1,7 @@ +VALID +VALID +VALID +VALID +VALID +VALID +VALID diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro new file mode 100644 index 0000000000..e82ded6fd0 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -0,0 +1,66 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type idx: record { + i: int; +}; + +type val: record { + b: bool; +}; + +global servers: table[int] of val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F, + $pred(typ: Input::Event, left: idx, right: bool) = { return right; } + ]); + Input::force_update(A::LOG); + if ( 1 in servers ) { + print "VALID"; + } + if ( 2 in servers ) { + print "VALID"; + } + if ( !(3 in servers) ) { + print "VALID"; + } + if ( !(4 in servers) ) { + print "VALID"; + } + if ( !(5 in servers) ) { + print "VALID"; + } + if ( !(6 in servers) ) { + print "VALID"; + } + if ( 7 in servers ) { + print "VALID"; + } +} From 77a517f2b5e90d94c9de144605c24247c620e4af Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 15:45:27 -0800 Subject: [PATCH 082/964] camel-casing for types --- testing/btest/scripts/base/frameworks/input/basic.bro | 8 ++++---- .../base/frameworks/input/onecolumn-norecord.bro | 8 ++++---- .../scripts/base/frameworks/input/onecolumn-record.bro | 8 ++++---- .../btest/scripts/base/frameworks/input/predicate.bro | 10 +++++----- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 139888fa7c..3ad45eac69 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -18,11 +18,11 @@ export { redef enum Log::ID += { LOG }; } -type idx: record { +type Idx: record { i: int; }; -type val: record { +type Val: record { b: bool; e: Log::ID; c: count; @@ -40,13 +40,13 @@ type val: record { ve: vector of int; }; -global servers: table[int] of val = table(); +global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="ssh", $idx=idx, $val=val, $destination=servers]); + Input::add_tablefilter(A::LOG, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 12dbdd42aa..134ceb49e6 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -18,21 +18,21 @@ export { redef enum Log::ID += { LOG }; } -type idx: record { +type Idx: record { i: int; }; -type val: record { +type Val: record { b: bool; }; -global servers: table[int] of val = table(); +global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index 4eef12d752..c07c9c826c 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -18,21 +18,21 @@ export { redef enum Log::ID += { LOG }; } -type idx: record { +type Idx: record { i: int; }; -type val: record { +type Val: record { b: bool; }; -global servers: table[int] of val = table(); +global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index e82ded6fd0..769536d2a6 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -24,22 +24,22 @@ export { redef enum Log::ID += { LOG }; } -type idx: record { +type Idx: record { i: int; }; -type val: record { +type Val: record { b: bool; }; -global servers: table[int] of val = table(); +global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F, - $pred(typ: Input::Event, left: idx, right: bool) = { return right; } + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, + $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); Input::force_update(A::LOG); if ( 1 in servers ) { From a16454b151fbbd530b2944c6b48393810fb23129 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 15:55:45 -0800 Subject: [PATCH 083/964] vector entries also have to be atomic. --- src/LogMgr.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index ab1706ca72..0b706f6417 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -157,7 +157,7 @@ bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) if ( atomic_only ) return false; - return IsCompatibleType(t->AsVectorType()->YieldType()); + return IsCompatibleType(t->AsVectorType()->YieldType(), true); } default: From 53af0544ccc9a60254228ab702ec39639b5b577d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 19:01:07 -0800 Subject: [PATCH 084/964] re-enable table events --- scripts/base/frameworks/input/main.bro | 2 +- src/InputMgr.cc | 120 +++++++++++------- src/InputMgr.h | 2 +- .../scripts.base.frameworks.input.event/out | 21 +++ .../scripts/base/frameworks/input/event.bro | 48 +++++++ 5 files changed, 147 insertions(+), 46 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.event/out create mode 100644 testing/btest/scripts/base/frameworks/input/event.bro diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 4560421ecc..69b4d41ebb 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -18,7 +18,7 @@ export { val: any; destination: any; want_record: bool &default=T; - table_ev: any &optional; # event containing idx, val as values. + ev: any &optional; # event containing idx, val as values. ## decision function, that decides if an insertion, update or removal should really be executed. ## or events should be thought diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 8fda5d506f..de9ef158b7 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -214,7 +214,6 @@ InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); string source((const char*) bsource->Bytes(), bsource->Len()); - ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault @@ -259,6 +258,50 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { RecordType *val = fval->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); TableVal *dst = fval->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); + Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); + + Val* event_val = fval->Lookup(rtype->FieldOffset("ev")); + Func* event = event_val ? event_val->AsFunc() : 0; + + if ( event ) { + FuncType* etype = event->FType()->AsFuncType(); + + if ( ! etype->IsEvent() ) { + reporter->Error("stream event is a function, not an event"); + return false; + } + + const type_list* args = etype->ArgTypes()->Types(); + + if ( args->length() != 3 ) + { + reporter->Error("Table event must take 3 arguments"); + return false; + } + + if ( ! same_type((*args)[0], BifType::Enum::Input::Event, 0) ) + { + reporter->Error("table events first attribute must be of type Input::Event"); + return false; + } + + if ( ! same_type((*args)[1], idx) ) + { + reporter->Error("table events index attributes do not match"); + return false; + } + + if ( want_record->InternalInt() == 1 && ! same_type((*args)[2], val) ) + { + reporter->Error("table events value attributes do not match"); + return false; + } else if ( want_record->InternalInt() == 0 && !same_type((*args)[2], val->FieldType(0) ) ) { + reporter->Error("table events value attribute does not match"); + return false; + } + + } + vector fieldsV; // vector, because we don't know the length beforehands bool status = !UnrollRecordType(&fieldsV, idx, ""); @@ -273,24 +316,22 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { return false; } - Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); LogField** fields = new LogField*[fieldsV.size()]; for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { fields[i] = fieldsV[i]; } - // FIXME: remove those funky 0-tests again as the idea was changed. Filter filter; filter.name = name->AsString()->CheckString(); filter.id = id->Ref()->AsEnumVal(); filter.pred = pred ? pred->AsFunc() : 0; filter.num_idx_fields = idxfields; filter.num_val_fields = valfields; - filter.tab = dst ? dst->Ref()->AsTableVal() : 0; - filter.rtype = val ? val->Ref()->AsRecordType() : 0; - filter.itype = idx ? idx->Ref()->AsRecordType() : 0; - // ya - well - we actually don't need them in every case... well, a few bytes of memory wasted + filter.tab = dst->Ref()->AsTableVal(); + filter.rtype = val->Ref()->AsRecordType(); + filter.itype = idx->Ref()->AsRecordType(); + filter.event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; filter.currDict = new PDict(InputHash); filter.lastDict = new PDict(InputHash); filter.want_record = ( want_record->InternalInt() == 1 ); @@ -601,11 +642,7 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const i->filters[id].currDict->Insert(idxhash, ih); - // send events now that we are kind of finished. - - /* FIXME: fix me. - std::list::iterator filter_iterator = i->events.begin(); - while ( filter_iterator != i->events.end() ) { + if ( i->filters[id].event ) { EnumVal* ev; Ref(idxval); @@ -613,16 +650,13 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); Ref(oldval); - SendEvent(*filter_iterator, ev, idxval, oldval); + SendEvent(i->filters[id].event, ev, idxval, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); Ref(valval); - SendEvent(*filter_iterator, ev, idxval, valval); + SendEvent(i->filters[id].event, ev, idxval, valval); } - - - ++filter_iterator; - } */ + } } @@ -643,12 +677,17 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { //while ( ( ih = i->lastDict->NextEntry(c) ) ) { while ( ( ih = i->filters[id].lastDict->NextEntry(lastDictIdxKey, c) ) ) { - if ( i->filters[id].pred ) { - ListVal *idx = i->filters[id].tab->RecoverIndex(ih->idxkey); - assert(idx != 0); - Val *val = i->filters[id].tab->Lookup(idx); - assert(val != 0); + ListVal * idx; + Val *val; + if ( i->filters[id].pred || i->filters[id].event ) { + idx = i->filters[id].tab->RecoverIndex(ih->idxkey); + assert(idx != 0); + val = i->filters[id].tab->Lookup(idx); + assert(val != 0); + } + + if ( i->filters[id].pred ) { bool doBreak = false; // ask predicate, if we want to expire this element... @@ -673,21 +712,13 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { } - // - - { - /* FIXME: events - std::list::iterator it = i->filters[id].events.begin(); - while ( it != i->filters[id].events.end() ) { - Ref(idx); - Ref(val); - EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEvent(*it, ev, idx, val); - ++it; - } - */ - } + } + if ( i->filters[id].event ) { + Ref(idx); + Ref(val); + EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEvent(i->filters[id].event, ev, idx, val); } i->filters[id].tab->Delete(ih->idxkey); @@ -792,20 +823,21 @@ void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c mgr.Dispatch(new Event(handler, vl)); } */ -void InputMgr::SendEvent(const string& name, EnumVal* event, Val* left, Val* right) +void InputMgr::SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right) { - EventHandler* handler = event_registry->Lookup(name.c_str()); - if ( handler == 0 ) { - reporter->Error("Event %s not found", name.c_str()); - return; - } + //EventHandler* handler = event_registry->Lookup(name.c_str()); + //if ( handler == 0 ) { + // reporter->Error("Event %s not found", name.c_str()); + // return; + //} val_list* vl = new val_list; vl->append(event); vl->append(left); vl->append(right); - mgr.Dispatch(new Event(handler, vl)); + //mgr.Dispatch(new Event(handler, vl)); + mgr.QueueEvent(ev, vl, SOURCE_LOCAL); } diff --git a/src/InputMgr.h b/src/InputMgr.h index 4280ba1d81..d04b7c9a2c 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -49,7 +49,7 @@ private: bool IsCompatibleType(BroType* t, bool atomic_only=false); bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); - void SendEvent(const string& name, EnumVal* event, Val* left, Val* right); + void SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right); HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); int GetLogValLength(const LogVal* val); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.event/out b/testing/btest/Baseline/scripts.base.frameworks.input.event/out new file mode 100644 index 0000000000..e32a2aea00 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.event/out @@ -0,0 +1,21 @@ +Input::EVENT_NEW +1 +T +Input::EVENT_NEW +2 +T +Input::EVENT_NEW +3 +F +Input::EVENT_NEW +4 +F +Input::EVENT_NEW +5 +F +Input::EVENT_NEW +6 +F +Input::EVENT_NEW +7 +T diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro new file mode 100644 index 0000000000..36e8171689 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -0,0 +1,48 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global destination: table[int] of Val = table(); + +event line(tpe: Input::Event, left: Idx, right: bool) { + print tpe; + print left; + print right; +} + +event bro_init() +{ + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::force_update(A::LOG); +} From 3035eb2b219946144fe67097781226d77f34176e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 19:30:16 -0800 Subject: [PATCH 085/964] fix a little bug that prevented several simultaneous filters from working. --- src/InputMgr.cc | 8 +- .../out | 15 +++ .../base/frameworks/input/twofilters.bro | 95 +++++++++++++++++++ 3 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out create mode 100644 testing/btest/scripts/base/frameworks/input/twofilters.bro diff --git a/src/InputMgr.cc b/src/InputMgr.cc index de9ef158b7..2ec3c649be 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -341,8 +341,12 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { assert(filter.want_record); } - i->filters[id->InternalInt()] = filter; - i->reader->AddFilter( id->InternalInt(), fieldsV.size(), fields ); + int filterid = 0; + if ( i->filters.size() > 0 ) { + filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map. new id = old id + 1. + } + i->filters[filterid] = filter; + i->reader->AddFilter( filterid, fieldsV.size(), fields ); return true; } diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out b/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out new file mode 100644 index 0000000000..5b1ee5e983 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out @@ -0,0 +1,15 @@ +VALID +VALID +VALID +VALID +VALID +VALID +VALID +MARK +VALID +VALID +VALID +VALID +VALID +VALID +VALID diff --git a/testing/btest/scripts/base/frameworks/input/twofilters.bro b/testing/btest/scripts/base/frameworks/input/twofilters.bro new file mode 100644 index 0000000000..575665e6e5 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/twofilters.bro @@ -0,0 +1,95 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global destination1: table[int] of Val = table(); +global destination2: table[int] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination1, $want_record=F, + $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } + ]); + Input::add_tablefilter(A::LOG, [$name="input2",$idx=Idx, $val=Val, $destination=destination2]); + + Input::force_update(A::LOG); + if ( 1 in destination1 ) { + print "VALID"; + } + if ( 2 in destination1 ) { + print "VALID"; + } + if ( !(3 in destination1) ) { + print "VALID"; + } + if ( !(4 in destination1) ) { + print "VALID"; + } + if ( !(5 in destination1) ) { + print "VALID"; + } + if ( !(6 in destination1) ) { + print "VALID"; + } + if ( 7 in destination1 ) { + print "VALID"; + } + + print "MARK"; + + if ( 2 in destination2 ) { + print "VALID"; + } + if ( 2 in destination2 ) { + print "VALID"; + } + if ( 3 in destination2 ) { + print "VALID"; + } + if ( 4 in destination2 ) { + print "VALID"; + } + if ( 5 in destination2 ) { + print "VALID"; + } + if ( 6 in destination2 ) { + print "VALID"; + } + if ( 7 in destination2 ) { + print "VALID"; + } + + +} From 6a563c88291eb5967e9bb43597d06c6ae52e206b Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Mon, 21 Nov 2011 22:30:53 -0800 Subject: [PATCH 086/964] Make exit() parameterizable. The exit() BiF used to have no arguments and always invoked exit(0) from libc. This small fix allows for non-zero exit values of the Bro process. --- src/bro.bif | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index 6d4d7ce1dd..8b4c8d3038 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -845,9 +845,9 @@ function log10%(d: double%): double return new Val(log10(d), TYPE_DOUBLE); %} -function exit%(%): int +function exit%(code: int%): any %{ - exit(0); + exit(code); return 0; %} From 1179c1a598602ffbdb08086c8edc3c709d7a4669 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Mon, 21 Nov 2011 22:55:14 -0800 Subject: [PATCH 087/964] Remove redundant active_connection() BiF. The BiF connection_exists has a more intuitive name and provides the same functionality, thus we can remove active_connection(). --- src/bro.bif | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index 8b4c8d3038..fd3a738126 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -720,12 +720,6 @@ function active_file%(f: file%): bool return new Val(f->IsOpen(), TYPE_BOOL); %} -function active_connection%(id: conn_id%): bool - %{ - Connection* c = sessions->FindConnection(id); - return new Val(c ? 1 : 0, TYPE_BOOL); - %} - # Note, you *must* first make sure that the connection is active # (e.g., by calling active_connection()) before invoking this. function connection_record%(cid: conn_id%): connection @@ -1027,12 +1021,10 @@ static Val* parse_port(const char* line) %%} # Returns true if the given connection exists, false otherwise. -function connection_exists%(c: conn_id%): bool +function connection_exists%(id: conn_id%): bool %{ - if ( sessions->FindConnection(c) ) - return new Val(1, TYPE_BOOL); - else - return new Val(0, TYPE_BOOL); + Connection* c = sessions->FindConnection(id); + return new Val(c ? 1 : 0, TYPE_BOOL); %} # For a given connection ID, returns the corresponding "connection" record. From c04b261376b9f00d3be9e2a93a132e7d5ebec923 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Mon, 21 Nov 2011 23:03:46 -0800 Subject: [PATCH 088/964] Remove redundant connection_record() BiF. The function lookup_connection() provides the same functionality and has more graceful failure semantics. --- src/bro.bif | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index fd3a738126..842535b657 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -720,22 +720,6 @@ function active_file%(f: file%): bool return new Val(f->IsOpen(), TYPE_BOOL); %} -# Note, you *must* first make sure that the connection is active -# (e.g., by calling active_connection()) before invoking this. -function connection_record%(cid: conn_id%): connection - %{ - Connection* c = sessions->FindConnection(cid); - if ( c ) - return c->BuildConnVal(); - else - { - // Hard to recover from this until we have union types ... - builtin_error("connection ID not a known connection (fatal)", cid); - exit(0); - return 0; - } - %} - %%{ EnumVal* map_conn_type(TransportProto tp) { From f82bf3f35fb5f1e298366f9af6d9ec0372ac3d58 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 22 Nov 2011 11:09:06 -0800 Subject: [PATCH 089/964] re-enable direct event sending from input readers --- src/InputMgr.cc | 18 +++++++++++++----- src/InputMgr.h | 3 ++- src/InputReader.cc | 7 +++---- src/InputReader.h | 2 +- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 2ec3c649be..e6d739d26f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -810,22 +810,30 @@ void InputMgr::Error(InputReader* reader, const char* msg) reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); } -/* Does not work atm, because LogValToVal needs BroType -void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) +bool InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { EventHandler* handler = event_registry->Lookup(name.c_str()); if ( handler == 0 ) { reporter->Error("Event %s not found", name.c_str()); - return; + return false; + } + + RecordType *type = handler->FType()->Args(); + int num_event_vals = type->NumFields(); + if ( num_vals != num_event_vals ) { + reporter->Error("Wrong number of values for event %s", name.c_str()); + return false; } val_list* vl = new val_list; for ( int i = 0; i < num_vals; i++) { - vl->append(LogValToVal(vals[i])); + vl->append(LogValToVal(vals[i], type->FieldType(i))); } mgr.Dispatch(new Event(handler, vl)); -} */ + + return true; +} void InputMgr::SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right) { diff --git a/src/InputMgr.h b/src/InputMgr.h index d04b7c9a2c..ba1ddafc92 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -59,7 +59,8 @@ private: Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); - //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + bool SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); diff --git a/src/InputReader.cc b/src/InputReader.cc index 1c65985fd6..3f296dc0aa 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -71,11 +71,10 @@ bool InputReader::Update() return DoUpdate(); } -/* -void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) +bool InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { - input_mgr->SendEvent(name, num_vals, vals); -} */ + return input_mgr->SendEvent(name, num_vals, vals); +} // stolen from logwriter const char* InputReader::Fmt(const char* format, ...) diff --git a/src/InputReader.h b/src/InputReader.h index 6e3d689750..34d549308e 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -48,7 +48,7 @@ protected: // A thread-safe version of fmt(). (stolen from logwriter) const char* Fmt(const char* format, ...); - //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + bool SendEvent(const string& name, const int num_vals, const LogVal* const *vals); // Content-sendinf-functions (simple mode). Including table-specific stuff that simply is not used if we have no table void Put(int id, const LogVal* const *val); From 3c40f00a539e5d3d2945ffd1df71e19e93c7133d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 22 Nov 2011 11:39:27 -0800 Subject: [PATCH 090/964] make filters pointers (for inheritance) --- scripts/base/frameworks/input/main.bro | 29 +- src/InputMgr.cc | 519 +++++++++++++----- src/InputMgr.h | 21 +- src/input.bif | 13 + .../out | 21 + .../scripts/base/frameworks/input/event.bro | 16 +- .../base/frameworks/input/tableevent.bro | 48 ++ 7 files changed, 507 insertions(+), 160 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out create mode 100644 testing/btest/scripts/base/frameworks/input/tableevent.bro diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 69b4d41ebb..a036eeeebd 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -23,10 +23,19 @@ export { ## decision function, that decides if an insertion, update or removal should really be executed. ## or events should be thought pred: function(typ: Input::Event, left: any, right: any): bool &optional; + }; - ## for "normalized" events - # ev: any &optional; - # ev_description: any &optional; + type EventFilter: record { + ## descriptive name. for later removal + name: string; + + # the event + ev: any; + # record describing the fields + fields: any; + + # does the event want the field unrolled (default) or as a simple record value? + want_record: bool &default=F; }; #const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. @@ -36,6 +45,8 @@ export { global force_update: function(id: Log::ID) : bool; global add_tablefilter: function(id: Log::ID, filter: Input::TableFilter) : bool; global remove_tablefilter: function(id: Log::ID, name: string) : bool; + global add_eventfilter: function(id: Log::ID, filter: Input::EventFilter) : bool; + global remove_eventfilter: function(id: Log::ID, name: string) : bool; #global get_filter: function(id: ID, name: string) : Filter; } @@ -74,6 +85,18 @@ function remove_tablefilter(id: Log::ID, name: string) : bool return __remove_tablefilter(id, name); } +function add_eventfilter(id: Log::ID, filter: Input::EventFilter) : bool + { +# filters[id, filter$name] = filter; + return __add_eventfilter(id, filter); + } + +function remove_eventfilter(id: Log::ID, name: string) : bool + { +# delete filters[id, name]; + return __remove_eventfilter(id, name); + } + #function get_filter(id: ID, name: string) : Filter # { # if ( [id, name] in filters ) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index e6d739d26f..0df11ea359 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -28,7 +28,13 @@ public: EnumVal* id; string name; - //int filter_type; // to distinguish between event and table filters + FilterType filter_type; // to distinguish between event and table filters + + virtual ~Filter(); +}; + +class InputMgr::TableFilter: public InputMgr::Filter { +public: unsigned int num_idx_fields; unsigned int num_val_fields; @@ -45,61 +51,42 @@ public: Func* pred; EventHandlerPtr event; - RecordType* event_type; - // ~Filter(); - // Filter(); - // Filter(const InputMgr::Filter& filter); - - void DoCleanup(); + TableFilter(); + ~TableFilter(); }; -/* -InputMgr::Filter::Filter() { +class InputMgr::EventFilter: public InputMgr::Filter { +public: + EventHandlerPtr event; + + RecordType* fields; + unsigned int num_fields; + + bool want_record; + EventFilter(); +}; + +InputMgr::TableFilter::TableFilter() { + filter_type = TABLE_FILTER; + tab = 0; itype = 0; rtype = 0; - event_type = 0; } -InputMgr::Filter::Filter(const InputMgr::Filter& f) { - id = f.id; - id->Ref(); +InputMgr::EventFilter::EventFilter() { + filter_type = EVENT_FILTER; +} - tab = f.tab; - if ( tab ) - tab->Ref(); - - itype = f.itype; - if ( itype ) - itype->Ref(); - - rtype = f.rtype; - if ( rtype ) - Ref(rtype); - - event_type = f.event_type; - if ( event_type ) - Ref(event_type); - - name = f.name; - num_idx_fields = f.num_idx_fields; - num_val_fields = f.num_val_fields; - want_record = f.want_record; - - -} */ - -void InputMgr::Filter::DoCleanup() { +InputMgr::Filter::~Filter() { Unref(id); - if ( tab ) - Unref(tab); - if ( itype ) - Unref(itype); - if ( rtype ) - Unref(rtype); - if ( event_type) - Unref(event_type); +} + +InputMgr::TableFilter::~TableFilter() { + Unref(tab); + Unref(itype); + Unref(rtype); delete currDict; delete lastDict; @@ -111,7 +98,7 @@ struct InputMgr::ReaderInfo { InputReader* reader; //list events; // events we fire when "something" happens - map filters; // filters that can prevent our actions + map filters; // filters that can prevent our actions bool HasFilter(int id); @@ -119,7 +106,11 @@ struct InputMgr::ReaderInfo { }; InputMgr::ReaderInfo::~ReaderInfo() { - // all the contents of filters should delete themselves automatically... + map::iterator it = filters.begin(); + + while ( it != filters.end() ) { + delete (*it).second; + } Unref(type); Unref(id); @@ -128,7 +119,7 @@ InputMgr::ReaderInfo::~ReaderInfo() { } bool InputMgr::ReaderInfo::HasFilter(int id) { - map::iterator it = filters.find(id); + map::iterator it = filters.find(id); if ( it == filters.end() ) { return false; } @@ -236,6 +227,114 @@ InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) } +bool InputMgr::AddEventFilter(EnumVal *id, RecordVal* fval) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->Error("Stream not found"); + return false; + } + + RecordType* rtype = fval->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::EventFilter, 0) ) + { + reporter->Error("filter argument not of right type"); + return false; + } + + Val* name = fval->Lookup(rtype->FieldOffset("name")); + RecordType *fields = fval->Lookup(rtype->FieldOffset("fields"))->AsType()->AsTypeType()->Type()->AsRecordType(); + + Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); + + Val* event_val = fval->Lookup(rtype->FieldOffset("ev")); + Func* event = event_val->AsFunc(); + + { + FuncType* etype = event->FType()->AsFuncType(); + + if ( ! etype->IsEvent() ) { + reporter->Error("stream event is a function, not an event"); + return false; + } + + const type_list* args = etype->ArgTypes()->Types(); + + if ( args->length() < 2 ) { + reporter->Error("event takes not enough arguments"); + return false; + } + + if ( ! same_type((*args)[0], BifType::Enum::Input::Event, 0) ) + { + reporter->Error("events first attribute must be of type Input::Event"); + return false; + } + + if ( want_record->InternalInt() == 0 ) { + if ( args->length() != fields->NumFields() + 1 ) { + reporter->Error("events has wrong number of arguments"); + return false; + } + + for ( int i = 0; i < fields->NumFields(); i++ ) { + if ( !same_type((*args)[i+1], fields->FieldType(i) ) ) { + reporter->Error("Incompatible type for event"); + return false; + } + } + + } else if ( want_record->InternalInt() == 1 ) { + if ( args->length() != 2 ) { + reporter->Error("events has wrong number of arguments"); + return false; + } + + if ( !same_type((*args)[1], fields ) ) { + reporter->Error("Incompatible type for event"); + return false; + } + + } else { + assert(false); + } + + } + + + vector fieldsV; // vector, because UnrollRecordType needs it + + bool status = !UnrollRecordType(&fieldsV, fields, ""); + + if ( status ) { + reporter->Error("Problem unrolling"); + return false; + } + + + LogField** logf = new LogField*[fieldsV.size()]; + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { + logf[i] = fieldsV[i]; + } + + EventFilter* filter = new EventFilter(); + filter->name = name->AsString()->CheckString(); + filter->id = id->Ref()->AsEnumVal(); + filter->num_fields = fieldsV.size(); + filter->fields = fields->Ref()->AsRecordType(); + filter->event = event_registry->Lookup(event->GetID()->Name()); + filter->want_record = ( want_record->InternalInt() == 1 ); + Unref(want_record); // ref'd by lookupwithdefault + + int filterid = 0; + if ( i->filters.size() > 0 ) { + filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map-> new id = old id + 1-> + } + i->filters[filterid] = filter; + i->reader->AddFilter( filterid, fieldsV.size(), logf ); + + return true; +} + bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { @@ -299,6 +398,7 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { reporter->Error("table events value attribute does not match"); return false; } + assert(want_record->InternalInt() == 1 || want_record->InternalInt() == 0); } @@ -322,28 +422,28 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { fields[i] = fieldsV[i]; } - Filter filter; - filter.name = name->AsString()->CheckString(); - filter.id = id->Ref()->AsEnumVal(); - filter.pred = pred ? pred->AsFunc() : 0; - filter.num_idx_fields = idxfields; - filter.num_val_fields = valfields; - filter.tab = dst->Ref()->AsTableVal(); - filter.rtype = val->Ref()->AsRecordType(); - filter.itype = idx->Ref()->AsRecordType(); - filter.event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; - filter.currDict = new PDict(InputHash); - filter.lastDict = new PDict(InputHash); - filter.want_record = ( want_record->InternalInt() == 1 ); + TableFilter* filter = new TableFilter(); + filter->name = name->AsString()->CheckString(); + filter->id = id->Ref()->AsEnumVal(); + filter->pred = pred ? pred->AsFunc() : 0; + filter->num_idx_fields = idxfields; + filter->num_val_fields = valfields; + filter->tab = dst->Ref()->AsTableVal(); + filter->rtype = val->Ref()->AsRecordType(); + filter->itype = idx->Ref()->AsRecordType(); + filter->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; + filter->currDict = new PDict(InputHash); + filter->lastDict = new PDict(InputHash); + filter->want_record = ( want_record->InternalInt() == 1 ); Unref(want_record); // ref'd by lookupwithdefault if ( valfields > 1 ) { - assert(filter.want_record); + assert(filter->want_record); } int filterid = 0; if ( i->filters.size() > 0 ) { - filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map. new id = old id + 1. + filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map-> new id = old id + 1-> } i->filters[filterid] = filter; i->reader->AddFilter( filterid, fieldsV.size(), fields ); @@ -478,13 +578,39 @@ bool InputMgr::RemoveTableFilter(EnumVal* id, const string &name) { return false; } - map::iterator it = i->filters.find(id->InternalInt()); + map::iterator it = i->filters.find(id->InternalInt()); if ( it == i->filters.end() ) { return false; } - i->filters[id->InternalInt()].DoCleanup(); + if ( i->filters[id->InternalInt()]->filter_type != TABLE_FILTER ) { + // wrong type; + return false; + } + delete (*it).second; + i->filters.erase(it); + return true; +} + +bool InputMgr::RemoveEventFilter(EnumVal* id, const string &name) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->Error("Reader not found"); + return false; + } + + map::iterator it = i->filters.find(id->InternalInt()); + if ( it == i->filters.end() ) { + return false; + } + + if ( i->filters[id->InternalInt()]->filter_type != EVENT_FILTER ) { + // wrong type; + return false; + } + + delete (*it).second; i->filters.erase(it); return true; } @@ -524,30 +650,53 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const return; } + if ( !i->HasFilter(id) ) { + reporter->InternalError("Unknown filter"); + return; + } + + if ( i->filters[id]->filter_type == TABLE_FILTER ) { + SendEntryTable(reader, id, vals); + } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + SendEventFilterEvent(reader, type, id, vals); + } else { + assert(false); + } + +} + +void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + bool updated = false; + assert(i); assert(i->HasFilter(id)); + assert(i->filters[id]->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i->filters[id]; + //reporter->Error("Hashing %d index fields", i->num_idx_fields); - HashKey* idxhash = HashLogVals(i->filters[id].num_idx_fields, vals); + HashKey* idxhash = HashLogVals(filter->num_idx_fields, vals); //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); //reporter->Error("Hashing %d val fields", i->num_val_fields); - HashKey* valhash = HashLogVals(i->filters[id].num_val_fields, vals+i->filters[id].num_idx_fields); + HashKey* valhash = HashLogVals(filter->num_val_fields, vals+filter->num_idx_fields); //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); - InputHash *h = i->filters[id].lastDict->Lookup(idxhash); + InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before if ( h->valhash->Hash() == valhash->Hash() ) { // ok, double. - i->filters[id].lastDict->Remove(idxhash); - i->filters[id].currDict->Insert(idxhash, h); + filter->lastDict->Remove(idxhash); + filter->currDict->Insert(idxhash, h); return; } else { // updated - i->filters[id].lastDict->Remove(idxhash); + filter->lastDict->Remove(idxhash); delete(h); updated = true; @@ -555,30 +704,25 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const } - Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); + Val* idxval = LogValToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; - int position = i->filters[id].num_idx_fields; - if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { - valval = LogValToVal(vals[position], i->filters[id].rtype->FieldType(0)); + int position = filter->num_idx_fields; + if ( filter->num_val_fields == 1 && !filter->want_record ) { + valval = LogValToVal(vals[position], filter->rtype->FieldType(0)); } else { - RecordVal * r = new RecordVal(i->filters[id].rtype); + RecordVal * r = new RecordVal(filter->rtype); - for ( int j = 0; j < i->filters[id].rtype->NumFields(); j++) { + for ( int j = 0; j < filter->rtype->NumFields(); j++) { Val* val = 0; - if ( i->filters[id].rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, i->filters[id].rtype->FieldType(j)->AsRecordType(), &position); + if ( filter->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, filter->rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->filters[id].rtype->FieldType(j)); + val = LogValToVal(vals[position], filter->rtype->FieldType(j)); position++; } - /* if ( val == 0 ) { - reporter->InternalError("conversion error"); - return; - } */ - r->Assign(j,val); } @@ -589,12 +733,12 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const Val* oldval = 0; if ( updated == true ) { // in that case, we need the old value to send the event (if we send an event). - oldval = i->filters[id].tab->Lookup(idxval); + oldval = filter->tab->Lookup(idxval); } // call filter first to determine if we really add / change the entry - if ( i->filters[id].pred ) { + if ( filter->pred ) { EnumVal* ev; Ref(idxval); Ref(valval); @@ -609,18 +753,18 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const vl.append(ev); vl.append(idxval); vl.append(valval); - Val* v = i->filters[id].pred->Call(&vl); + Val* v = filter->pred->Call(&vl); bool result = v->AsBool(); Unref(v); if ( result == false ) { if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... - delete(i->filters[id].currDict->RemoveEntry(idxhash)); + delete(filter->currDict->RemoveEntry(idxhash)); return; } else { // keep old one - i->filters[id].currDict->Insert(idxhash, h); + filter->currDict->Insert(idxhash, h); return; } } @@ -629,24 +773,23 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const //i->tab->Assign(idxval, valval); - HashKey* k = i->filters[id].tab->ComputeHash(idxval); + HashKey* k = filter->tab->ComputeHash(idxval); if ( !k ) { reporter->InternalError("could not hash"); return; } - reporter->Error("assigning"); - i->filters[id].tab->Assign(idxval, k, valval); + filter->tab->Assign(idxval, k, valval); InputHash* ih = new InputHash(); - k = i->filters[id].tab->ComputeHash(idxval); + k = filter->tab->ComputeHash(idxval); ih->idxkey = k; ih->valhash = valhash; //i->tab->Delete(k); - i->filters[id].currDict->Insert(idxhash, ih); + filter->currDict->Insert(idxhash, ih); - if ( i->filters[id].event ) { + if ( filter->event ) { EnumVal* ev; Ref(idxval); @@ -654,11 +797,11 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); Ref(oldval); - SendEvent(i->filters[id].event, ev, idxval, oldval); + SendEvent(filter->event, 3, ev, idxval, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); Ref(valval); - SendEvent(i->filters[id].event, ev, idxval, valval); + SendEvent(filter->event, 3, ev, idxval, valval); } } } @@ -673,25 +816,33 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { assert(i->HasFilter(id)); + if ( i->filters[id]->filter_type == EVENT_FILTER ) { + // nothing to do.. + return; + } + + assert(i->filters[id]->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i->filters[id]; + // lastdict contains all deleted entries and should be empty apart from that - IterCookie *c = i->filters[id].lastDict->InitForIteration(); - i->filters[id].lastDict->MakeRobustCookie(c); + IterCookie *c = filter->lastDict->InitForIteration(); + filter->lastDict->MakeRobustCookie(c); InputHash* ih; HashKey *lastDictIdxKey; //while ( ( ih = i->lastDict->NextEntry(c) ) ) { - while ( ( ih = i->filters[id].lastDict->NextEntry(lastDictIdxKey, c) ) ) { + while ( ( ih = filter->lastDict->NextEntry(lastDictIdxKey, c) ) ) { ListVal * idx; Val *val; - if ( i->filters[id].pred || i->filters[id].event ) { - idx = i->filters[id].tab->RecoverIndex(ih->idxkey); + if ( filter->pred || filter->event ) { + idx = filter->tab->RecoverIndex(ih->idxkey); assert(idx != 0); - val = i->filters[id].tab->Lookup(idx); + val = filter->tab->Lookup(idx); assert(val != 0); } - if ( i->filters[id].pred ) { + if ( filter->pred ) { bool doBreak = false; // ask predicate, if we want to expire this element... @@ -704,37 +855,37 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { vl.append(ev); vl.append(idx); vl.append(val); - Val* v = i->filters[id].pred->Call(&vl); + Val* v = filter->pred->Call(&vl); bool result = v->AsBool(); Unref(v); if ( result == false ) { // Keep it. Hence - we quit and simply go to the next entry of lastDict // ah well - and we have to add the entry to currDict... - i->filters[id].currDict->Insert(lastDictIdxKey, i->filters[id].lastDict->RemoveEntry(lastDictIdxKey)); + filter->currDict->Insert(lastDictIdxKey, filter->lastDict->RemoveEntry(lastDictIdxKey)); continue; } } - if ( i->filters[id].event ) { + if ( filter->event ) { Ref(idx); Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEvent(i->filters[id].event, ev, idx, val); + SendEvent(filter->event, 3, ev, idx, val); } - i->filters[id].tab->Delete(ih->idxkey); - i->filters[id].lastDict->Remove(lastDictIdxKey); // deletex in next line + filter->tab->Delete(ih->idxkey); + filter->lastDict->Remove(lastDictIdxKey); // deletex in next line delete(ih); } - i->filters[id].lastDict->Clear(); // should be empty... but... well... who knows... - delete(i->filters[id].lastDict); + filter->lastDict->Clear(); // should be empty->->-> but->->-> well->->-> who knows->->-> + delete(filter->lastDict); - i->filters[id].lastDict = i->filters[id].currDict; - i->filters[id].currDict = new PDict(InputHash); + filter->lastDict = filter->currDict; + filter->currDict = new PDict(InputHash); } void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) { @@ -744,24 +895,86 @@ void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) return; } + if ( !i->HasFilter(id) ) { + reporter->InternalError("Unknown filter"); + return; + } + + if ( i->filters[id]->filter_type == TABLE_FILTER ) { + PutTable(reader, id, vals); + } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + SendEventFilterEvent(reader, type, id, vals); + } else { + assert(false); + } + +} + +void InputMgr::SendEventFilterEvent(const InputReader* reader, EnumVal* type, int id, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + + bool updated = false; + + assert(i); assert(i->HasFilter(id)); - Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); + assert(i->filters[id]->filter_type == EVENT_FILTER); + EventFilter* filter = (EventFilter*) i->filters[id]; + + Val *val; + list out_vals; + // no tracking, send everything with a new event... + //out_vals.push_back(new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event)); + out_vals.push_back(type); + + int position = 0; + if ( filter->want_record ) { + RecordVal * r = LogValToRecordVal(vals, filter->fields, &position); + out_vals.push_back(r); + + } else { + for ( int j = 0; j < filter->fields->NumFields(); j++) { + Val* val = 0; + if ( filter->fields->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, filter->fields->FieldType(j)->AsRecordType(), &position); + } else { + val = LogValToVal(vals[position], filter->fields->FieldType(j)); + position++; + } + out_vals.push_back(val); + } + } + + SendEvent(filter->event, out_vals); + +} + +void InputMgr::PutTable(const InputReader* reader, int id, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + + assert(i); + assert(i->HasFilter(id)); + + assert(i->filters[id]->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i->filters[id]; + + Val* idxval = LogValToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; - int position = i->filters[id].num_idx_fields; - if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { - valval = LogValToVal(vals[i->filters[id].num_idx_fields], i->filters[id].rtype->FieldType(i->filters[id].num_idx_fields)); + int position = filter->num_idx_fields; + if ( filter->num_val_fields == 1 && !filter->want_record ) { + valval = LogValToVal(vals[filter->num_idx_fields], filter->rtype->FieldType(filter->num_idx_fields)); } else { - RecordVal * r = new RecordVal(i->filters[id].rtype); + RecordVal * r = new RecordVal(filter->rtype); - for ( int j = 0; j < i->filters[id].rtype->NumFields(); j++) { + for ( int j = 0; j < filter->rtype->NumFields(); j++) { Val* val = 0; - if ( i->filters[id].rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, i->filters[id].rtype->FieldType(j)->AsRecordType(), &position); + if ( filter->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, filter->rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->filters[id].rtype->FieldType(j)); + val = LogValToVal(vals[position], filter->rtype->FieldType(j)); position++; } @@ -776,7 +989,7 @@ void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) valval = r; } - i->filters[id].tab->Assign(idxval, valval); + filter->tab->Assign(idxval, valval); } void InputMgr::Clear(const InputReader* reader, int id) { @@ -788,7 +1001,10 @@ void InputMgr::Clear(const InputReader* reader, int id) { assert(i->HasFilter(id)); - i->filters[id].tab->RemoveAll(); + assert(i->filters[id]->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i->filters[id]; + + filter->tab->RemoveAll(); } bool InputMgr::Delete(const InputReader* reader, int id, const LogVal* const *vals) { @@ -800,9 +1016,18 @@ bool InputMgr::Delete(const InputReader* reader, int id, const LogVal* const *va assert(i->HasFilter(id)); - Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); - - return ( i->filters[id].tab->Delete(idxval) != 0 ); + if ( i->filters[id]->filter_type == TABLE_FILTER ) { + TableFilter* filter = (TableFilter*) i->filters[id]; + Val* idxval = LogValToIndexVal(filter->num_idx_fields, filter->itype, vals); + return( filter->tab->Delete(idxval) != 0 ); + } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEventFilterEvent(reader, type, id, vals); + return true; + } else { + assert(false); + return false; + } } void InputMgr::Error(InputReader* reader, const char* msg) @@ -835,25 +1060,35 @@ bool InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c return true; } -void InputMgr::SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right) +void InputMgr::SendEvent(EventHandlerPtr ev, const int numvals, ...) { - //EventHandler* handler = event_registry->Lookup(name.c_str()); - //if ( handler == 0 ) { - // reporter->Error("Event %s not found", name.c_str()); - // return; - //} - val_list* vl = new val_list; - vl->append(event); - vl->append(left); - vl->append(right); + + va_list lP; + va_start(lP, numvals); + for ( int i = 0; i < numvals; i++ ) + { + vl->append( va_arg(lP, Val*) ); + } + va_end(lP); + + mgr.QueueEvent(ev, vl, SOURCE_LOCAL); +} + +void InputMgr::SendEvent(EventHandlerPtr ev, list events) +{ + val_list* vl = new val_list; + + for ( list::iterator i = events.begin(); i != events.end(); i++ ) { + vl->append( *i ); + } - //mgr.Dispatch(new Event(handler, vl)); mgr.QueueEvent(ev, vl, SOURCE_LOCAL); } -Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { + +RecordVal* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { if ( position == 0 ) { reporter->InternalError("Need position"); return 0; diff --git a/src/InputMgr.h b/src/InputMgr.h index ba1ddafc92..cebed231e4 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -27,6 +27,8 @@ public: bool AddTableFilter(EnumVal *id, RecordVal* filter); bool RemoveTableFilter(EnumVal* id, const string &name); + bool AddEventFilter(EnumVal *id, RecordVal* filter); + bool RemoveEventFilter(EnumVal* id, const string &name); protected: friend class InputReader; @@ -46,10 +48,17 @@ protected: private: struct ReaderInfo; + void SendEntryTable(const InputReader* reader, int id, const LogVal* const *vals); + void PutTable(const InputReader* reader, int id, const LogVal* const *vals); + void SendEventFilterEvent(const InputReader* reader, EnumVal* type, int id, const LogVal* const *vals); + bool IsCompatibleType(BroType* t, bool atomic_only=false); bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); - void SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right); + + void SendEvent(EventHandlerPtr ev, const int numvals, ...); + void SendEvent(EventHandlerPtr ev, list events); + bool SendEvent(const string& name, const int num_vals, const LogVal* const *vals); HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); int GetLogValLength(const LogVal* val); @@ -57,9 +66,8 @@ private: Val* LogValToVal(const LogVal* val, BroType* request_type); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); - Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); + RecordVal* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); - bool SendEvent(const string& name, const int num_vals, const LogVal* const *vals); ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); @@ -68,7 +76,12 @@ private: string Hash(const string &input); - struct Filter; + class Filter; + class TableFilter; + class EventFilter; + + enum FilterType { TABLE_FILTER, EVENT_FILTER }; + }; diff --git a/src/input.bif b/src/input.bif index 1300f91bea..b1d57b1df6 100644 --- a/src/input.bif +++ b/src/input.bif @@ -9,6 +9,7 @@ module Input; type StreamDescription: record; type TableFilter: record; +type EventFilter: record; function Input::__create_stream%(id: Log::ID, description: Input::StreamDescription%) : bool %{ @@ -40,6 +41,18 @@ function Input::__remove_tablefilter%(id: Log::ID, name: string%) : bool return new Val( res, TYPE_BOOL); %} +function Input::__add_eventfilter%(id: Log::ID, filter: Input::EventFilter%) : bool + %{ + bool res = input_mgr->AddEventFilter(id->AsEnumVal(), filter->AsRecordVal()); + return new Val( res, TYPE_BOOL ); + %} + +function Input::__remove_eventfilter%(id: Log::ID, name: string%) : bool + %{ + bool res = input_mgr->RemoveEventFilter(id->AsEnumVal(), name->AsString()->CheckString()); + return new Val( res, TYPE_BOOL); + %} + # Options for Ascii Reader module InputAscii; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out new file mode 100644 index 0000000000..e32a2aea00 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out @@ -0,0 +1,21 @@ +Input::EVENT_NEW +1 +T +Input::EVENT_NEW +2 +T +Input::EVENT_NEW +3 +F +Input::EVENT_NEW +4 +F +Input::EVENT_NEW +5 +F +Input::EVENT_NEW +6 +F +Input::EVENT_NEW +7 +T diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro index 36e8171689..d9be733a1b 100644 --- a/testing/btest/scripts/base/frameworks/input/event.bro +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -16,7 +16,6 @@ 7 T @TEST-END-FILE -redef InputAscii::empty_field = "EMPTY"; module A; @@ -24,25 +23,20 @@ export { redef enum Log::ID += { LOG }; } -type Idx: record { - i: int; -}; - type Val: record { + i: int; b: bool; }; -global destination: table[int] of Val = table(); - -event line(tpe: Input::Event, left: Idx, right: bool) { +event line(tpe: Input::Event, i: int, b: bool) { print tpe; - print left; - print right; + print i; + print b; } event bro_init() { Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::add_eventfilter(A::LOG, [$name="input", $fields=Val, $ev=line]); Input::force_update(A::LOG); } diff --git a/testing/btest/scripts/base/frameworks/input/tableevent.bro b/testing/btest/scripts/base/frameworks/input/tableevent.bro new file mode 100644 index 0000000000..36e8171689 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/tableevent.bro @@ -0,0 +1,48 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global destination: table[int] of Val = table(); + +event line(tpe: Input::Event, left: Idx, right: bool) { + print tpe; + print left; + print right; +} + +event bro_init() +{ + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::force_update(A::LOG); +} From 03646a8d55e881a2e0cddf6486e87bae14eeaa39 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 23 Nov 2011 16:10:58 -0500 Subject: [PATCH 091/964] Added the ssl_session_ticket_handshake event and fixed a few SSL bugs. --- src/event.bif | 1 + src/ssl-analyzer.pac | 23 +++++++++++-- src/ssl-protocol.pac | 81 ++++++++++++++++++++++---------------------- 3 files changed, 61 insertions(+), 44 deletions(-) diff --git a/src/event.bif b/src/event.bif index d953ac78fe..289d781819 100644 --- a/src/event.bif +++ b/src/event.bif @@ -280,6 +280,7 @@ event ssh_server_version%(c: connection, version: string%); event ssl_client_hello%(c: connection, version: count, possible_ts: time, session_id: string, ciphers: count_set%); event ssl_server_hello%(c: connection, version: count, possible_ts: time, session_id: string, cipher: count, comp_method: count%); event ssl_extension%(c: connection, code: count, val: string%); +event ssl_session_ticket_handshake%(c: connection, ticket_lifetime_hint: count, ticket: string%); event ssl_established%(c: connection%); event ssl_alert%(c: connection, level: count, desc: count%); diff --git a/src/ssl-analyzer.pac b/src/ssl-analyzer.pac index 6471d9c4a4..5440e0f4d5 100644 --- a/src/ssl-analyzer.pac +++ b/src/ssl-analyzer.pac @@ -138,7 +138,7 @@ refine connection SSL_Conn += { if ( ssl_client_hello ) { vector* cipher_suites = new vector(); - if ( cipher_suites16 ) + if ( cipher_suites16 ) std::copy(cipher_suites16->begin(), cipher_suites16->end(), std::back_inserter(*cipher_suites)); else std::transform(cipher_suites24->begin(), cipher_suites24->end(), std::back_inserter(*cipher_suites), to_int()); @@ -199,6 +199,18 @@ refine connection SSL_Conn += { return true; %} + + function proc_session_ticket_handshake(rec: SessionTicketHandshake, is_orig: bool): bool + %{ + if ( ssl_session_ticket_handshake ) + { + BifEvent::generate_ssl_session_ticket_handshake(bro_analyzer(), + bro_analyzer()->Conn(), + ${rec.ticket_lifetime_hint}, + new StringVal(${rec.data}.length(), (const char*) ${rec.data}.data())); + } + return true; + %} function proc_ssl_extension(type: int, data: bytestring) : bool %{ @@ -263,13 +275,14 @@ refine connection SSL_Conn += { der_cert); // Are there any X509 extensions? + //printf("Number of x509 extensions: %d\n", X509_get_ext_count(pTemp)); if ( x509_extension && X509_get_ext_count(pTemp) > 0 ) { int num_ext = X509_get_ext_count(pTemp); for ( int k = 0; k < num_ext; ++k ) { unsigned char *pBuffer = 0; - int length = 0; + uint length = 0; X509_EXTENSION* ex = X509_get_ext(pTemp, k); if (ex) @@ -277,7 +290,7 @@ refine connection SSL_Conn += { ASN1_STRING *pString = X509_EXTENSION_get_data(ex); length = ASN1_STRING_to_UTF8(&pBuffer, pString); //i2t_ASN1_OBJECT(&pBuffer, length, obj) - + printf("extension length: %u\n", length); // -1 indicates an error. if ( length < 0 ) continue; @@ -436,6 +449,10 @@ refine typeattr Handshake += &let { proc : bool = $context.connection.proc_handshake(this, rec.is_orig); }; +refine typeattr SessionTicketHandshake += &let { + proc : bool = $context.connection.proc_session_ticket_handshake(this, rec.is_orig); +} + refine typeattr UnknownRecord += &let { proc : bool = $context.connection.proc_unknown_record(rec); }; diff --git a/src/ssl-protocol.pac b/src/ssl-protocol.pac index f60d73b27e..bd5b692b80 100644 --- a/src/ssl-protocol.pac +++ b/src/ssl-protocol.pac @@ -57,22 +57,15 @@ type SSLRecord(is_orig: bool) = record { type RecordText(rec: SSLRecord, is_orig: bool) = case $context.connection.state() of { STATE_ABBREV_SERVER_ENCRYPTED, STATE_CLIENT_ENCRYPTED, STATE_COMM_ENCRYPTED, STATE_CONN_ESTABLISHED - -> ciphertext : CiphertextRecord(rec, is_orig); + -> ciphertext : CiphertextRecord(rec); default - -> plaintext : PlaintextRecord(rec, is_orig); + -> plaintext : PlaintextRecord(rec); }; -type PossibleEncryptedHandshake(rec: SSLRecord, is_orig: bool) = case $context.connection.state() of { - # Deal with encrypted handshakes before the server cipher spec change. - STATE_CLIENT_FINISHED, STATE_CLIENT_ENCRYPTED - -> ct : CiphertextRecord(rec, is_orig); - default -> hs : Handshake(rec); -}; - -type PlaintextRecord(rec: SSLRecord, is_orig: bool) = case rec.content_type of { +type PlaintextRecord(rec: SSLRecord) = case rec.content_type of { CHANGE_CIPHER_SPEC -> ch_cipher : ChangeCipherSpec(rec); ALERT -> alert : Alert(rec); - HANDSHAKE -> handshake : PossibleEncryptedHandshake(rec, is_orig); + HANDSHAKE -> handshake : Handshake(rec); APPLICATION_DATA -> app_data : ApplicationData(rec); V2_ERROR -> v2_error : V2Error(rec); V2_CLIENT_HELLO -> v2_client_hello : V2ClientHello(rec); @@ -265,18 +258,19 @@ enum AnalyzerState { ###################################################################### enum HandshakeType { - HELLO_REQUEST = 0, - CLIENT_HELLO = 1, - SERVER_HELLO = 2, - CERTIFICATE = 11, - SERVER_KEY_EXCHANGE = 12, - CERTIFICATE_REQUEST = 13, - SERVER_HELLO_DONE = 14, - CERTIFICATE_VERIFY = 15, - CLIENT_KEY_EXCHANGE = 16, - FINISHED = 20, - CERTIFICATE_URL = 21, # RFC 3546 - CERTIFICATE_STATUS = 22, # RFC 3546 + HELLO_REQUEST = 0, + CLIENT_HELLO = 1, + SERVER_HELLO = 2, + SESSION_TICKET = 4, # RFC 5077 + CERTIFICATE = 11, + SERVER_KEY_EXCHANGE = 12, + CERTIFICATE_REQUEST = 13, + SERVER_HELLO_DONE = 14, + CERTIFICATE_VERIFY = 15, + CLIENT_KEY_EXCHANGE = 16, + FINISHED = 20, + CERTIFICATE_URL = 21, # RFC 3546 + CERTIFICATE_STATUS = 22, # RFC 3546 }; %code{ @@ -286,6 +280,7 @@ enum HandshakeType { case HELLO_REQUEST: return string("HELLO_REQUEST"); case CLIENT_HELLO: return string("CLIENT_HELLO"); case SERVER_HELLO: return string("SERVER_HELLO"); + case SESSION_TICKET: return string("SESSION_TICKET"); case CERTIFICATE: return string("CERTIFICATE"); case SERVER_KEY_EXCHANGE: return string("SERVER_KEY_EXCHANGE"); case CERTIFICATE_REQUEST: return string("CERTIFICATE_REQUEST"); @@ -457,8 +452,7 @@ type V2ServerHello(rec: SSLRecord) = record { cert_data : bytestring &length = cert_len; ciphers : uint24[ciph_len/3]; conn_id_data : bytestring &length = conn_id_len; -} #&length = 8 + cert_len + ciph_len + conn_id_len, -&let { +} &let { state_changed : bool = (session_id_hit > 0 ? $context.connection.transition(STATE_CLIENT_HELLO_RCVD, @@ -608,7 +602,7 @@ type CertificateVerify(rec: SSLRecord) = record { ###################################################################### # The finished messages are always sent after encryption is in effect, -# so we will not be able to read those message. +# so we will not be able to read those messages. type Finished(rec: SSLRecord) = record { cont : bytestring &restofdata &transient; } &let { @@ -620,13 +614,17 @@ type Finished(rec: SSLRecord) = record { $context.connection.lost_track(); }; +type SessionTicketHandshake(rec: SSLRecord) = record { + ticket_lifetime_hint: uint32; + data: bytestring &restofdata; +}; ###################################################################### # V3 Handshake Protocol (7.) ###################################################################### type UnknownHandshake(hs: Handshake, is_orig: bool) = record { - cont : bytestring &restofdata &transient; + data : bytestring &restofdata &transient; } &let { state_changed : bool = $context.connection.lost_track(); }; @@ -636,19 +634,20 @@ type Handshake(rec: SSLRecord) = record { length : uint24; body : case msg_type of { - HELLO_REQUEST -> hello_request : HelloRequest(rec); - CLIENT_HELLO -> client_hello : ClientHello(rec); - SERVER_HELLO -> server_hello : ServerHello(rec); - CERTIFICATE -> certificate : Certificate(rec); - SERVER_KEY_EXCHANGE -> server_key_exchange : ServerKeyExchange(rec); - CERTIFICATE_REQUEST -> certificate_request : CertificateRequest(rec); - SERVER_HELLO_DONE -> server_hello_done : ServerHelloDone(rec); - CERTIFICATE_VERIFY -> certificate_verify : CertificateVerify(rec); - CLIENT_KEY_EXCHANGE -> client_key_exchange : ClientKeyExchange(rec); - FINISHED -> finished : Finished(rec); - CERTIFICATE_URL -> certificate_url : bytestring &restofdata &transient; - CERTIFICATE_STATUS -> certificate_status : bytestring &restofdata &transient; - default -> unknown_handshake : UnknownHandshake(this, rec.is_orig); + HELLO_REQUEST -> hello_request : HelloRequest(rec); + CLIENT_HELLO -> client_hello : ClientHello(rec); + SERVER_HELLO -> server_hello : ServerHello(rec); + SESSION_TICKET -> session_ticket : SessionTicketHandshake(rec); + CERTIFICATE -> certificate : Certificate(rec); + SERVER_KEY_EXCHANGE -> server_key_exchange : ServerKeyExchange(rec); + CERTIFICATE_REQUEST -> certificate_request : CertificateRequest(rec); + SERVER_HELLO_DONE -> server_hello_done : ServerHelloDone(rec); + CERTIFICATE_VERIFY -> certificate_verify : CertificateVerify(rec); + CLIENT_KEY_EXCHANGE -> client_key_exchange : ClientKeyExchange(rec); + FINISHED -> finished : Finished(rec); + CERTIFICATE_URL -> certificate_url : bytestring &restofdata &transient; + CERTIFICATE_STATUS -> certificate_status : bytestring &restofdata &transient; + default -> unknown_handshake : UnknownHandshake(this, rec.is_orig); } &length = to_int()(length); }; @@ -663,7 +662,7 @@ type UnknownRecord(rec: SSLRecord) = record { state_changed : bool = $context.connection.lost_track(); }; -type CiphertextRecord(rec: SSLRecord, is_orig: bool) = record { +type CiphertextRecord(rec: SSLRecord) = record { cont : bytestring &restofdata &transient; } &let { state_changed : bool = From 71f2f81bc6619ef8786c3de5ea94fd5a4fc9ad56 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 24 Nov 2011 16:42:18 -0600 Subject: [PATCH 092/964] Fix order of include directories. This change prevents locally installed header files from overshadowing the header files of the same name that Bro needs during compilation. --- src/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 897acc9d37..47314514f6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,5 @@ -include_directories(${CMAKE_CURRENT_SOURCE_DIR} +include_directories(BEFORE + ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ) From e9f05348b003492c5a269501f4eac636663bcbe8 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Tue, 22 Nov 2011 09:04:22 -0800 Subject: [PATCH 093/964] Perform type checking on count-to-port conversion. Related to #684. --- src/bro.bif | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index 842535b657..65b9d9791a 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -540,9 +540,9 @@ function port_to_count%(p: port%): count return new Val(p->Port(), TYPE_COUNT); %} -function count_to_port%(c: count, t: transport_proto%): port +function count_to_port%(num: count, proto: transport_proto%): port %{ - return new PortVal(c, (TransportProto)(t->InternalInt())); + return new PortVal(num, (TransportProto)proto->AsEnum()); %} function floor%(d: double%): double From 8f8290c8521ebccde2023b3d41c939649fffbf6d Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Sun, 27 Nov 2011 15:57:18 -0600 Subject: [PATCH 094/964] Add simple profiling class to accumulate Stmt usage stats across runs. Use the BROFILER_FILE environment variable to point to a file in which Stmt usage statistics from Bro script-layer can be output. This should be able to be used to check Bro script coverage that that e.g. the entire test suite covers. --- src/Brofiler.cc | 82 ++++++++++++++++++++++++++++++++++++++++++++++ src/Brofiler.h | 65 ++++++++++++++++++++++++++++++++++++ src/CMakeLists.txt | 1 + src/Stmt.cc | 1 + src/Stmt.h | 1 + src/main.cc | 6 ++++ src/parse.y | 25 +++++++++++++- src/util.cc | 1 + 8 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 src/Brofiler.cc create mode 100644 src/Brofiler.h diff --git a/src/Brofiler.cc b/src/Brofiler.cc new file mode 100644 index 0000000000..fc23f10a7d --- /dev/null +++ b/src/Brofiler.cc @@ -0,0 +1,82 @@ +#include +#include +#include +#include "Brofiler.h" +#include "util.h" + +Brofiler::Brofiler() + : delim('\t') + { + } + +Brofiler::~Brofiler() + { + } + +void Brofiler::ReadStats() + { + char* bf = getenv("BROFILER_FILE"); + if ( ! bf ) return; + FILE* f = fopen(bf, "r"); + if ( ! f ) + { + fprintf(stderr, "Failed to open Brofiler file '%s' for reading\n", bf); + return; + } + + char line[16384]; + string delimiter; + delimiter = delim; + while( fgets(line, sizeof(line), f) ) + { + line[strlen(line) - 1] = 0; //remove newline + string cnt(strtok(line, delimiter.c_str())); + string location(strtok(0, delimiter.c_str())); + string desc(strtok(0, delimiter.c_str())); + pair location_desc(location, desc); + uint64 count; + atoi_n(cnt.size(), cnt.c_str(), 0, 10, count); + usage_map[location_desc] = count; + } + + fclose(f); + } + +void Brofiler::WriteStats() + { + char* bf = getenv("BROFILER_FILE"); + if ( ! bf ) return; + + FILE* f = fopen(bf, "w"); + if ( ! f ) + { + fprintf(stderr, "Failed to open Brofiler file '%s' for writing\n", bf); + return; + } + + for ( list::const_iterator it = stmts.begin(); + it != stmts.end(); ++it ) + { + ODesc location_info; + (*it)->GetLocationInfo()->Describe(&location_info); + ODesc desc_info; + (*it)->Describe(&desc_info); + string desc(desc_info.Description()); + for_each(desc.begin(), desc.end(), canonicalize_desc()); + pair location_desc(location_info.Description(), desc); + if ( usage_map.find(location_desc) != usage_map.end() ) + usage_map[location_desc] += (*it)->GetAccessCount(); + else + usage_map[location_desc] = (*it)->GetAccessCount(); + } + + map, uint64 >::const_iterator it; + for ( it = usage_map.begin(); it != usage_map.end(); ++it ) + { + fprintf(f, "%"PRIu64"%c%s%c%s\n", it->second, delim, + it->first.first.c_str(), delim, it->first.second.c_str()); + } + + fclose(f); + } + diff --git a/src/Brofiler.h b/src/Brofiler.h new file mode 100644 index 0000000000..6ded906698 --- /dev/null +++ b/src/Brofiler.h @@ -0,0 +1,65 @@ +#ifndef BROFILER_H_ +#define BROFILER_H_ + +#include +#include +#include +#include + + +/** + * A simple class for managing stats of Bro script coverage across Bro runs. + */ +class Brofiler { +public: + Brofiler(); + virtual ~Brofiler(); + + /** + * Imports Bro script Stmt usage information from file pointed to by + * environment variable BROFILER_FILE. + */ + void ReadStats(); + + /** + * Combines usage stats from current run with any read from ReadStats(), + * then writes information to file pointed to by environment variable + * BROFILER_FILE. + */ + void WriteStats(); + + void SetDelim(char d) { delim = d; } + + /** + * The current, global Brofiler instance creates this list at parse-time. + */ + list stmts; + +private: + /** + * + * This maps Stmt location-desc pairs to the total number of times that + * Stmt has been executed. The map can be initialized from a file at + * startup time and modified at shutdown time before writing back + * to a file. + */ + map, uint64> usage_map; + + /** + * The character to use to delimit Brofiler output files. Default is '\t'. + */ + char delim; + + /** + * A canonicalization routine for Stmt descriptions containing characters + * that don't agree with the output format of Brofiler. + */ + struct canonicalize_desc { + void operator() (char& c) + { + if ( c == '\n' ) c = ' '; + } + }; +}; + +#endif /* BROFILER_H_ */ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 897acc9d37..f755895eae 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -281,6 +281,7 @@ set(bro_SRCS BPF_Program.cc BroDoc.cc BroDocObj.cc + Brofiler.cc BroString.cc CCL.cc ChunkedIO.cc diff --git a/src/Stmt.cc b/src/Stmt.cc index 6a83940b3b..669dc5565e 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -258,6 +258,7 @@ static BroFile* print_stdout = 0; Val* PrintStmt::DoExec(val_list* vals, stmt_flow_type& /* flow */) const { + RegisterAccess(); if ( ! print_stdout ) print_stdout = new BroFile(stdout); diff --git a/src/Stmt.h b/src/Stmt.h index 8e3a4b4118..7c3b42609b 100644 --- a/src/Stmt.h +++ b/src/Stmt.h @@ -52,6 +52,7 @@ public: void RegisterAccess() const { last_access = network_time; access_count++; } void AccessStats(ODesc* d) const; + uint32 GetAccessCount() const { return access_count; } virtual void Describe(ODesc* d) const; diff --git a/src/main.cc b/src/main.cc index dfa46c3050..68ab8b14a5 100644 --- a/src/main.cc +++ b/src/main.cc @@ -47,9 +47,12 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "ConnCompressor.h" #include "DPM.h" #include "BroDoc.h" +#include "Brofiler.h" #include "binpac_bro.h" +Brofiler brofiler; + #ifndef HAVE_STRSEP extern "C" { char* strsep(char**, const char*); @@ -260,6 +263,8 @@ void terminate_bro() terminating = true; + brofiler.WriteStats(); + EventHandlerPtr bro_done = internal_handler("bro_done"); if ( bro_done ) mgr.QueueEvent(bro_done, new val_list); @@ -335,6 +340,7 @@ static void bro_new_handler() int main(int argc, char** argv) { + brofiler.ReadStats(); bro_argc = argc; bro_argv = new char* [argc]; diff --git a/src/parse.y b/src/parse.y index 495931aae0..420e995ae1 100644 --- a/src/parse.y +++ b/src/parse.y @@ -80,10 +80,12 @@ #include "Reporter.h" #include "BroDoc.h" #include "BroDocObj.h" +#include "Brofiler.h" #include #include +extern Brofiler brofiler; extern BroDoc* current_reST_doc; extern int generate_documentation; extern std::list* reST_doc_comments; @@ -1330,93 +1332,111 @@ stmt: { set_location(@1, @3); $$ = new PrintStmt($2); + brofiler.stmts.push_back($$); } | TOK_EVENT event ';' { set_location(@1, @3); $$ = new EventStmt($2); + brofiler.stmts.push_back($$); } | TOK_IF '(' expr ')' stmt { set_location(@1, @4); $$ = new IfStmt($3, $5, new NullStmt()); + //brofiler.stmts.push_back($$); } | TOK_IF '(' expr ')' stmt TOK_ELSE stmt { set_location(@1, @4); $$ = new IfStmt($3, $5, $7); + //brofiler.stmts.push_back($$); } | TOK_SWITCH expr '{' case_list '}' { set_location(@1, @2); $$ = new SwitchStmt($2, $4); + //brofiler.stmts.push_back($$); } | for_head stmt - { $1->AsForStmt()->AddBody($2); } + { + $1->AsForStmt()->AddBody($2); + //brofiler.stmts.push_back($1); + } | TOK_NEXT ';' { set_location(@1, @2); $$ = new NextStmt; + brofiler.stmts.push_back($$); } | TOK_BREAK ';' { set_location(@1, @2); $$ = new BreakStmt; + brofiler.stmts.push_back($$); } | TOK_RETURN ';' { set_location(@1, @2); $$ = new ReturnStmt(0); + brofiler.stmts.push_back($$); } | TOK_RETURN expr ';' { set_location(@1, @2); $$ = new ReturnStmt($2); + brofiler.stmts.push_back($$); } | TOK_ADD expr ';' { set_location(@1, @3); $$ = new AddStmt($2); + brofiler.stmts.push_back($$); } | TOK_DELETE expr ';' { set_location(@1, @3); $$ = new DelStmt($2); + brofiler.stmts.push_back($$); } | TOK_LOCAL local_id opt_type init_class opt_init opt_attr ';' { set_location(@1, @7); $$ = add_local($2, $3, $4, $5, $6, VAR_REGULAR); + brofiler.stmts.push_back($$); } | TOK_CONST local_id opt_type init_class opt_init opt_attr ';' { set_location(@1, @6); $$ = add_local($2, $3, $4, $5, $6, VAR_CONST); + brofiler.stmts.push_back($$); } | TOK_WHEN '(' expr ')' stmt { set_location(@3, @5); $$ = new WhenStmt($3, $5, 0, 0, false); + brofiler.stmts.push_back($$); } | TOK_WHEN '(' expr ')' stmt TOK_TIMEOUT expr '{' stmt_list '}' { set_location(@3, @8); $$ = new WhenStmt($3, $5, $9, $7, false); + brofiler.stmts.push_back($$); } @@ -1424,18 +1444,21 @@ stmt: { set_location(@4, @6); $$ = new WhenStmt($4, $6, 0, 0, true); + brofiler.stmts.push_back($$); } | TOK_RETURN TOK_WHEN '(' expr ')' stmt TOK_TIMEOUT expr '{' stmt_list '}' { set_location(@4, @9); $$ = new WhenStmt($4, $6, $10, $8, true); + brofiler.stmts.push_back($$); } | expr ';' { set_location(@1, @2); $$ = new ExprStmt($1); + brofiler.stmts.push_back($$); } | ';' diff --git a/src/util.cc b/src/util.cc index f81eff8f22..a48feb9828 100644 --- a/src/util.cc +++ b/src/util.cc @@ -345,6 +345,7 @@ template int atoi_n(int len, const char* s, const char** end, int base, // Instantiate the ones we need. template int atoi_n(int len, const char* s, const char** end, int base, int& result); template int atoi_n(int len, const char* s, const char** end, int base, int64_t& result); +template int atoi_n(int len, const char* s, const char** end, int base, uint64_t& result); char* uitoa_n(uint64 value, char* str, int n, int base, const char* prefix) { From 5666448a482601a9890e0f495cc3ebcead950c0d Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Sun, 27 Nov 2011 15:11:13 -0800 Subject: [PATCH 095/964] Change some BiF return values from bool to any. The BiFs - do_profiling - make_connection_persistent - expect_connection used to unconditionally return true. Since such a return value is meaningless, returning 'any' is more appropriate. --- src/bro.bif | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index 65b9d9791a..16b18b0d48 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -2231,12 +2231,12 @@ function send_current_packet%(p: event_peer%) : bool return new Val(remote_serializer->SendPacket(&info, id, pkt), TYPE_BOOL); %} -function do_profiling%(%) : bool +function do_profiling%(%) : any %{ if ( profiling_logger ) profiling_logger->Log(); - return new Val(1, TYPE_BOOL); + return 0; %} function get_event_peer%(%) : event_peer @@ -2279,10 +2279,10 @@ function send_capture_filter%(p: event_peer, s: string%) : bool return new Val(remote_serializer->SendCaptureFilter(id, s->CheckString()), TYPE_BOOL); %} -function make_connection_persistent%(c: connection%) : bool +function make_connection_persistent%(c: connection%) : any %{ c->MakePersistent(); - return new Val(1, TYPE_BOOL); + return 0; %} function is_local_interface%(ip: addr%) : bool @@ -2959,11 +2959,11 @@ function continue_processing%(%) : any # Schedule analyzer for a future connection. function expect_connection%(orig: addr, resp: addr, resp_p: port, - analyzer: count, tout: interval%) : bool + analyzer: count, tout: interval%) : any %{ dpm->ExpectConnection(orig, resp, resp_p->Port(), resp_p->PortType(), (AnalyzerTag::Tag) analyzer, tout, 0); - return new Val(1, TYPE_BOOL); + return 0; %} # Disables the analyzer which raised the current event (if the analyzer From 286c8fd85978175e9be2ac05e499bae43babea98 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 28 Nov 2011 13:29:02 -0600 Subject: [PATCH 096/964] Add note about independent component releases to Broxygen index. --- doc/index.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/index.rst b/doc/index.rst index ba3df81e7d..0a6b225431 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -46,6 +46,12 @@ Script Reference Other Bro Components -------------------- +The following are snapshots of documentation for components that come +with this version of Bro (|version|). Since they can also be used +independently, see the `download page +`_ for documentation of any +current, independent component releases. + .. toctree:: :maxdepth: 1 From be1b3ce5e1f239861013517b8d8c5a5fafbdf0bd Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 28 Nov 2011 13:29:02 -0600 Subject: [PATCH 097/964] Add note about independent component releases to Broxygen index. --- doc/index.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/index.rst b/doc/index.rst index ba3df81e7d..0a6b225431 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -46,6 +46,12 @@ Script Reference Other Bro Components -------------------- +The following are snapshots of documentation for components that come +with this version of Bro (|version|). Since they can also be used +independently, see the `download page +`_ for documentation of any +current, independent component releases. + .. toctree:: :maxdepth: 1 From 4975584e01af095cf6a6d10a4c09eb88a3f198e0 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 28 Nov 2011 13:28:44 -0800 Subject: [PATCH 098/964] change Log enum to Input enum. --- scripts/base/frameworks/input/main.bro | 10 +++++----- src/input.bif | 10 +++++----- src/types.bif | 1 - testing/btest/scripts/base/frameworks/input/basic.bro | 8 ++++---- testing/btest/scripts/base/frameworks/input/event.bro | 8 ++++---- .../base/frameworks/input/onecolumn-norecord.bro | 8 ++++---- .../scripts/base/frameworks/input/onecolumn-record.bro | 8 ++++---- .../btest/scripts/base/frameworks/input/predicate.bro | 8 ++++---- .../btest/scripts/base/frameworks/input/twofilters.bro | 10 +++++----- 9 files changed, 35 insertions(+), 36 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index a036eeeebd..1a9e8b885c 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -58,28 +58,28 @@ module Input; #global filters: table[ID, string] of Filter; -function create_stream(id: Log::ID, description: Input::StreamDescription) : bool +function create_stream(id: Input::ID, description: Input::StreamDescription) : bool { return __create_stream(id, description); } -function remove_stream(id: Log::ID) : bool +function remove_stream(id: Input::ID) : bool { return __remove_stream(id); } -function force_update(id: Log::ID) : bool +function force_update(id: Input::ID) : bool { return __force_update(id); } -function add_tablefilter(id: Log::ID, filter: Input::TableFilter) : bool +function add_tablefilter(id: Input::ID, filter: Input::TableFilter) : bool { # filters[id, filter$name] = filter; return __add_tablefilter(id, filter); } -function remove_tablefilter(id: Log::ID, name: string) : bool +function remove_tablefilter(id: Input::ID, name: string) : bool { # delete filters[id, name]; return __remove_tablefilter(id, name); diff --git a/src/input.bif b/src/input.bif index b1d57b1df6..a7d561c060 100644 --- a/src/input.bif +++ b/src/input.bif @@ -11,31 +11,31 @@ type StreamDescription: record; type TableFilter: record; type EventFilter: record; -function Input::__create_stream%(id: Log::ID, description: Input::StreamDescription%) : bool +function Input::__create_stream%(id: Input::ID, description: Input::StreamDescription%) : bool %{ InputReader *the_reader = input_mgr->CreateStream(id->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} -function Input::__remove_stream%(id: Log::ID%) : bool +function Input::__remove_stream%(id: Input::ID%) : bool %{ bool res = input_mgr->RemoveStream(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__force_update%(id: Log::ID%) : bool +function Input::__force_update%(id: Input::ID%) : bool %{ bool res = input_mgr->ForceUpdate(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__add_tablefilter%(id: Log::ID, filter: Input::TableFilter%) : bool +function Input::__add_tablefilter%(id: Input::ID, filter: Input::TableFilter%) : bool %{ bool res = input_mgr->AddTableFilter(id->AsEnumVal(), filter->AsRecordVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_tablefilter%(id: Log::ID, name: string%) : bool +function Input::__remove_tablefilter%(id: Input::ID, name: string%) : bool %{ bool res = input_mgr->RemoveTableFilter(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL); diff --git a/src/types.bif b/src/types.bif index f90a954224..15fab1a7b1 100644 --- a/src/types.bif +++ b/src/types.bif @@ -180,7 +180,6 @@ enum Event %{ EVENT_REMOVED, %} - enum ID %{ Unknown, %} diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 3ad45eac69..10cc7376a8 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -15,7 +15,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -45,8 +45,8 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); - Input::force_update(A::LOG); + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::force_update(A::INPUT); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro index d9be733a1b..a07f0934a0 100644 --- a/testing/btest/scripts/base/frameworks/input/event.bro +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -20,7 +20,7 @@ module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Val: record { @@ -36,7 +36,7 @@ event line(tpe: Input::Event, i: int, b: bool) { event bro_init() { - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_eventfilter(A::LOG, [$name="input", $fields=Val, $ev=line]); - Input::force_update(A::LOG); + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_eventfilter(A::INPUT, [$name="input", $fields=Val, $ev=line]); + Input::force_update(A::INPUT); } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 134ceb49e6..88838cc8d6 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -15,7 +15,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -31,8 +31,8 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); - Input::force_update(A::LOG); + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); + Input::force_update(A::INPUT); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index c07c9c826c..fc4d862cd3 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -15,7 +15,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -31,8 +31,8 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers]); - Input::force_update(A::LOG); + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); + Input::force_update(A::INPUT); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index 769536d2a6..5e6bae7b62 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -21,7 +21,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -37,11 +37,11 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); - Input::force_update(A::LOG); + Input::force_update(A::INPUT); if ( 1 in servers ) { print "VALID"; } diff --git a/testing/btest/scripts/base/frameworks/input/twofilters.bro b/testing/btest/scripts/base/frameworks/input/twofilters.bro index 575665e6e5..5af664e0e9 100644 --- a/testing/btest/scripts/base/frameworks/input/twofilters.bro +++ b/testing/btest/scripts/base/frameworks/input/twofilters.bro @@ -21,7 +21,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -38,13 +38,13 @@ global destination2: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination1, $want_record=F, + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=destination1, $want_record=F, $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); - Input::add_tablefilter(A::LOG, [$name="input2",$idx=Idx, $val=Val, $destination=destination2]); + Input::add_tablefilter(A::INPUT, [$name="input2",$idx=Idx, $val=Val, $destination=destination2]); - Input::force_update(A::LOG); + Input::force_update(A::INPUT); if ( 1 in destination1 ) { print "VALID"; } From 1abb1424b8f1e050f24ffb9a9a7829fd8e10c3f0 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 28 Nov 2011 14:15:04 -0800 Subject: [PATCH 099/964] begin documenting... --- doc/index.rst | 1 + doc/input.rst | 190 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 doc/input.rst diff --git a/doc/index.rst b/doc/index.rst index 0a6b225431..975ee7def4 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -20,6 +20,7 @@ Frameworks notice logging + input cluster signatures diff --git a/doc/input.rst b/doc/input.rst new file mode 100644 index 0000000000..78e96fe06e --- /dev/null +++ b/doc/input.rst @@ -0,0 +1,190 @@ +===================== +Loading Data into Bro +===================== + +.. rst-class:: opening + + Bro comes with a flexible input interface that allows to read + previously stored data. Data is either read into bro tables or + sent to scripts using events. + This document describes how the input framework can be used. + +.. contents:: + +Terminology +=========== + +Bro's input framework is built around three main abstracts, that are +very similar to the abstracts used in the logging framework: + + Input Streams + An input stream corresponds to a single input source + (usually a textfile). It defined the information necessary + to find the source (e.g. the filename) + + Filters + Each input stream has a set of filters attached to it, that + determine exaclty what kind of information is read. + There are two different kind of streams, event streams and table + streams. + By default, event streams generate an event for each line read + from the input source. + Table streams on the other hand read the input source in a bro + table for easy later access. + + Readers + A reader defines the input format for the specific input stream. + At the moment, Bro comes with only one type of reader, which can + read the tab seperated ASCII logfiles that were generated by the + logging framework. + + +Basics +====== + +For examples, please look at the unit tests in +``testing/btest/scripts/base/frameworks/input/``. + +A very basic example to open an input stream is: + +.. code:: bro + + module Foo; + + export { + # Create an ID for our new stream + redef enum Input::ID += { INPUT }; + } + + event bro_init() { + Input::create_stream(FOO::INPUT, [$source="input.log"]); + } + +The fields that can be set when creating a stream are: + + ``source`` + A mandatory string identifying the source of the data. + For the ASCII reader this is the filename. + + ``reader`` + The reader used for this stream. Default is ``READER_ASCII``. + + +Filters +======= + +Each filter defines the data fields that it wants to receive from the respective +input file. Depending on the type of filter, events or a table are created from +the data in the source file. + +Event Filters +------------- + +Event filters are filters that generate an event for each line in of the input source. + +For example, a simple filter retrieving the fields ``i`` and ``b`` from an inputSource +could be defined as follows: + +.. code:: bro + + type Val: record { + i: int; + b: bool; + }; + + event line(tpe: Input::Event, i: int, b: bool) { + # work with event data + } + + event bro_init { + # Input stream definition, etc + ... + + Input::add_eventfilter(Foo::INPUT, [$name="input", $fields=Val, $ev=line]); + + # read the file after all filters have been set + Input::force_update(Foo::INPUT); + } + +The fields that can be set for an event filter are: + + ``name`` + A mandatory name for the filter that can later be used + to manipulate it further. + + ``fields`` + Name of a record type containing the fields, which should be retrieved from + the input stream. + + ``ev`` + The event which is fired, after a line has been read from the input source. + The first argument that is passed to the event is an Input::Event structure, + followed by the data, either inside of a record (if ``want_record is set``) or as + individual fields. + The Input::Event structure can contain information, if the received line is ``NEW``, has + been ``CHANGED`` or ``DELETED``. Singe the ascii reader cannot track this information + for event filters, the value is always ``NEW`` at the moment. + + ``want_record`` + Boolean value, that defines if the event wants to receive the fields inside of + a single record value, or individually (default). + +Table Filters +------------- + +Table filters are the second, more complex type of filter. + +Table filters store the information they read from an input source in a bro table. For example, +when reading a file that contains ip addresses and connection attemt information one could use +an approach similar to this: + +.. code:: bro + + type Idx: record { + a: addr; + }; + + type Val: record { + tries: count; + }; + + global conn_attempts: table[addr] of count = table(); + + event bro_init { + # Input stream definitions, etc. + ... + + Input::add_tablefilter(Foo::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=conn_attempts]); + + # read the file after all filters have been set + Input::force_update(Foo::INPUT); + } + +The table conn_attempts will then contain the information about connection attemps. + +The possible fields that can be set for an table filter are: + + ``name`` + A mandatory name for the filter that can later be used + to manipulate it further. + + ``idx`` + Record type that defines the index of the table + + ``val`` + Record type that defines the values of the table + + ``want_record`` + Defines if the values of the table should be stored as a record (default), + or as a simple value. Has to be set if Val contains more than one element. + + ``destination`` + The destination table + + ``ev`` + Optional event that is raised, when values are added to, changed in or deleted from the table. + Events are passed an Input::Event description as the first argument, the index record as the second argument + and the values as the third argument. + + ``pred`` + Optional predicate, that can prevent entries from being added to the table and events from being sent. From 26296f9fb18319c5a16fdb46d7f9827501230066 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 28 Nov 2011 16:41:36 -0600 Subject: [PATCH 100/964] Add header/footer iframes to Broxygen docs to mimic website --- doc/_templates/layout.html | 12 ++++++++++++ doc/index.rst | 2 ++ 2 files changed, 14 insertions(+) diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html index 518718c55c..0c58f34e17 100644 --- a/doc/_templates/layout.html +++ b/doc/_templates/layout.html @@ -7,4 +7,16 @@ {% endblock %} +{% block header %} + +{% endblock %} + {% block relbar2 %}{% endblock %} +{% block relbar1 %}{% endblock %} + +{% block footer %} +{{ super() }} + +{% endblock %} diff --git a/doc/index.rst b/doc/index.rst index 0a6b225431..9e9aabc625 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -3,6 +3,8 @@ Bro Documentation ================= +Documentation for version: |version| + .. toctree:: :maxdepth: 1 From 5991eaa564c1eeafa0cec815a68d07899ededc0f Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 28 Nov 2011 18:12:46 -0600 Subject: [PATCH 101/964] Turn off Broxygen's sticky/floating sidebar. --- doc/conf.py.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/conf.py.in b/doc/conf.py.in index 72337b930f..8844370a07 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -99,7 +99,7 @@ html_last_updated_fmt = '%B %d, %Y' # documentation. html_theme_options = { "rightsidebar": "true", -"stickysidebar": "true", +"stickysidebar": "false", "externalrefs": "false", "footerbgcolor": "#333", "footertextcolor": "#ddd", From fa9125f61a113f7a00e8e6ac716dc5ad48bc8e39 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 17 Nov 2011 16:59:53 -0800 Subject: [PATCH 102/964] Adding some first doc strings to event.bif. Still trying to find the right style. --- src/event.bif | 188 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 186 insertions(+), 2 deletions(-) diff --git a/src/event.bif b/src/event.bif index d953ac78fe..1da2420083 100644 --- a/src/event.bif +++ b/src/event.bif @@ -1,35 +1,219 @@ +# Documentation conventions: +# +# - Short initial sentence (which doesn't need to be a sentence), starting +# with "Generated ..." +# +# - Use past tense for activity that has already occured. + +## Generated at Bro initialization time. The event engine generates this +## event just before normal input processing begins. It can be used to execute +## one-time initialization code at startup. At the time a handler runs, Bro will +## have executed any global initializations and statements. +## +## .. note:: When a ``bro_init`` handler executes, Bro has not yet seen any +## input packets and therefore :bro:id:`network_time` is not initialized yet. An +## artifact of that is that any timer installed in a ``bro_init`` handler will fire +## immediately with the first packet. The standard way to work around that is to +## ignore the first time the timer fires and immediately reschedule. +## +## see:: bro_done event bro_init%(%); + +## Generated at Bro termination time. The event engine generates this event when +## Bro is about to terminate, either due to having exhausted reading its input +## trace file(s), receiving a termination signal, or because Bro was run without +## a network input source and has finished executing any global statements. +## This event is generated after :bro:id:`net_done`, which gets only raised when +## processing network traffic. +## +## .. note:: If Bro terminates due to an invocation of :bro:id:`exit`, then this +## event is not generated. +## +## see:: net_done bro_init event bro_done%(%); +# TODO. event dns_mapping_valid%(dm: dns_mapping%); event dns_mapping_unverified%(dm: dns_mapping%); event dns_mapping_new_name%(dm: dns_mapping%); event dns_mapping_lost_name%(dm: dns_mapping%); -event dns_mapping_name_changed%(old_dm: dns_mapping, new_dm: dns_mapping%); +# event dns_mapping_name_changed%(old_dm: dns_mapping, new_dm: dns_mapping%); event dns_mapping_altered%(dm: dns_mapping, old_addrs: addr_set, new_addrs: addr_set%); +## Generated for every new connection. The event is raised with the first packet +## of a previously unknown connection. Bro uses a flow-based definition of +## "connection" here that includes not only TCP sessions but also UDP and ICMP +## flows. +## +## c: The connection. +## +## .. note:: Handling this event is potentially expensive. For example, during a +## SYN flooding attack, every spoofed SYN packet will lead to a new new_connection +## event. +## +## see:: *connection* event new_connection%(c: connection%); + +## Generated when reassembly starts for a TCP connection. The event is raised +## at the moment when Bro's TCP analyzer enables stream reassembly for a +## connection. +## +## c: The connection. +## +## see:: *connection* event new_connection_contents%(c: connection%); -event new_packet%(c: connection, p: pkt_hdr%); + +## Generated for an unsuccessful connection attempt. The event is raised when an +## originator unsuccessfully attempted to establish a connection. "Unsuccessful" +## is defined as at least :bro:id:`tcp_attempt_delay` seconds having elapsed since +## the originator first sent a connection establishment packet to the destination +## without seeing a reply. +## +## c: The connection. +## +## see:: connection_*, new_connection*, partial_connection. event connection_attempt%(c: connection%); + +## Generated for an established TCP connection. The event is raised when the +## initial 3-way TCP handshake has successfully finished for a connection. +## +## c: The connection. +## +## see:: *connection* event connection_established%(c: connection%); + +## Generated for a new active TCP connection if Bro did not see the initial +## handshake. The event is raised when Bro has observed traffic from each endpoint, +## but the activity did not begin with the usual connection establishment. +## +## c: The connection. +## +## see:: *connection* event partial_connection%(c: connection%); + +## TODO. +## +## see:: *connection* event connection_partial_close%(c: connection%); + +## Generated for a TCP connection that finished normally. The event is raised +## when a regular FIN handshake from both endpoints was observed. +## +## c: The connection. +## +## see:: *connection* event connection_finished%(c: connection%); + +## TODO. +## +## see:: *connection* event connection_half_finished%(c: connection%); + +## Generated for a rejected TCP connection. The event is raised when an originator +## attempted to setup a TCP connection but the responder replied with a RST packet +## denying it. +## +## c: The connection. +## +## .. note:: If the responder does not respond at all, :bro:id:`connection_attempt` +## is raised instead. If the responder initially accepts the connection but +## aborts it later, Bro first generates :bro:id:`connection_established` and then +## :bro:id:`connection_reset`. +## +## see:: *connection* event connection_rejected%(c: connection%); + +## Generated when an endpoint aborted a TCP connection. The event is raised +## when one endpoint of an established TCP connection aborted by sending a RST +## packet. +## +## c: The connection. +## +## see:: *connection* event connection_reset%(c: connection%); + +## TODO +## +## see:: *connection* event connection_pending%(c: connection%); + +## Generated when a connection's internal state is about to be removed from +## memory. Bro generates this event reliably once for every connection when it +## is about to delete the internal state. As such, the event is well-suited for +## scrip-level cleanup that needs to be performed for every connection. The +## ``connection_state_remove`` event is generated not only for TCP sessions but +## also for UDP and ICMP flows. +## +## c: The connection. +## +## see:: *connection* event connection_state_remove%(c: connection%); + +## Generated for a SYN packet. Bro raises this event for every SYN packet seen by +## its TCP analyzer. +## +## c: The connection. +## +## pkt: Information extracted from the SYN packet. +## +## .. note:: This event has quite low-level semantics and can potentially be +## expensive to generate. It should only be used if one really needs the +## specific information passed into the handler via the ``pkt`` argument. If +## not, handling one of the other ``connection_*`` events is typically the +## better approach. +## +## see:: *connection* event connection_SYN_packet%(c: connection, pkt: SYN_packet%); + +## Generated for the first ACK packet seen for a TCP connection from +## its *orginator*. +## +## c: The connection. +## +## pkt: Information extracted from the SYN packet. +## +## .. note:: This event has quite low-level semantics and should be used only +## rarely. +## +## see:: *connection* event connection_first_ACK%(c: connection%); + +## Generated when a TCP connection timed out. This event is raised when no activity +## was seen for an interval of at least :bro:id:`tcp_connection_linger`, and +## either one endpoint has already closed the connection or one side never +## never became active. +## +## c: The connection. +## +## .. note:: The precise semantics of this event can be unintuitive as it +## only covers a subset of cases where a connection times out. Often, handling +## :bro:id:`connection_state_remove` is the better option. That one will be +## generated reliably when an interval of ``tcp_inactivity_timeout`` has passed +## with out any activity seen (but also for all other ways a connection may +## terminate). +## +## see:: *connection* event connection_timeout%(c: connection%); + +## Generated when a connection 4-tuple is reused. The event is raised when Bro +## sees a new TCP session or UDP flow using a 4-tuple matching that of an earlier +## connection it still consideres active. +## +## c: The connection. +## +## see:: *connection* event connection_reused%(c: connection%); + + + event connection_status_update%(c: connection%); event connection_EOF%(c: connection, is_orig: bool%); event connection_external%(c: connection, tag: string%); + event expected_connection_seen%(c: connection, a: count%); +event new_packet%(c: connection, p: pkt_hdr%); + event protocol_confirmation%(c: connection, atype: count, aid: count%); event protocol_violation%(c: connection, atype: count, aid: count, reason: string%); From 59a6187f42c21720428c4dfc6cdd0b93ae94efc7 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 18 Nov 2011 14:47:28 -0800 Subject: [PATCH 103/964] Checkpointing work on event.bif. Much there, not all yet. --- doc/conf.py.in | 5 +- src/event.bif | 5280 +++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 5034 insertions(+), 251 deletions(-) diff --git a/doc/conf.py.in b/doc/conf.py.in index 8844370a07..8148bd0f42 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -24,7 +24,7 @@ sys.path.insert(0, os.path.abspath('sphinx-sources/ext')) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['bro', 'rst_directive'] +extensions = ['bro', 'rst_directive', 'sphinx.ext.todo'] # Add any paths that contain templates here, relative to this directory. templates_path = ['sphinx-sources/_templates', 'sphinx-sources/_static'] @@ -241,3 +241,6 @@ man_pages = [ ('index', 'bro', u'Bro Documentation', [u'The Bro Project'], 1) ] + +# -- Options for todo plugin -------------------------------------------- +todo_include_todos=True diff --git a/src/event.bif b/src/event.bif index 1da2420083..a8d858de57 100644 --- a/src/event.bif +++ b/src/event.bif @@ -1,140 +1,252 @@ -# Documentation conventions: -# -# - Short initial sentence (which doesn't need to be a sentence), starting -# with "Generated ..." -# -# - Use past tense for activity that has already occured. - +# +# Documentation conventions: +# +# - Use past tense for activity that has already occured. +# +# - List parameters with an empty line in between. +# +# - Within the description, reference other parameters of the same events +# with *arg*. +# +# - Order: +# +# - Short initial sentence (which doesn't need to be a sentence), +# starting with "Generated ..." +# +# - Description. +# +# - Parameters +# +# - .. bro:see:: +# +# - .. note:: +# +# - .. todo:: +# +# Templates: +# +# See `Wikipedia `__ for more +# information about the XXX protocol. +# +# See `RFC XXX `__ for more information about +# the XXX protocol. + ## Generated at Bro initialization time. The event engine generates this ## event just before normal input processing begins. It can be used to execute ## one-time initialization code at startup. At the time a handler runs, Bro will -## have executed any global initializations and statements. -## -## .. note:: When a ``bro_init`` handler executes, Bro has not yet seen any -## input packets and therefore :bro:id:`network_time` is not initialized yet. An -## artifact of that is that any timer installed in a ``bro_init`` handler will fire +## have executed any global initializations and statements. +## +## .. bro:see:: bro_done +## +## .. note:: +## +## When a ``bro_init`` handler executes, Bro has not yet seen any input packets +## and therefore :bro:id:`network_time` is not initialized yet. An artifact +## of that is that any timer installed in a ``bro_init`` handler will fire ## immediately with the first packet. The standard way to work around that is to ## ignore the first time the timer fires and immediately reschedule. ## -## see:: bro_done event bro_init%(%); ## Generated at Bro termination time. The event engine generates this event when ## Bro is about to terminate, either due to having exhausted reading its input ## trace file(s), receiving a termination signal, or because Bro was run without ## a network input source and has finished executing any global statements. -## This event is generated after :bro:id:`net_done`, which gets only raised when -## processing network traffic. -## -## .. note:: If Bro terminates due to an invocation of :bro:id:`exit`, then this -## event is not generated. ## -## see:: net_done bro_init +## .. bro:see:: bro_init +## +## .. note:: +## +## If Bro terminates due to an invocation of :bro:id:`exit`, then this event is +## not generated. event bro_done%(%); -# TODO. +## TODO. +## +## .. bro:see:: dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified event dns_mapping_valid%(dm: dns_mapping%); + +## TODO. +## +## .. bro:see:: dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_valid event dns_mapping_unverified%(dm: dns_mapping%); + +## TODO. +## +## .. bro:see:: dns_mapping_altered dns_mapping_lost_name dns_mapping_unverified +## dns_mapping_valid event dns_mapping_new_name%(dm: dns_mapping%); + +## TODO. +## +## .. bro:see:: dns_mapping_altered dns_mapping_new_name dns_mapping_unverified +## dns_mapping_valid event dns_mapping_lost_name%(dm: dns_mapping%); -# event dns_mapping_name_changed%(old_dm: dns_mapping, new_dm: dns_mapping%); + +## TODO. +## +## .. bro:see:: dns_mapping_lost_name dns_mapping_new_name dns_mapping_unverified +## dns_mapping_valid event dns_mapping_altered%(dm: dns_mapping, old_addrs: addr_set, new_addrs: addr_set%); ## Generated for every new connection. The event is raised with the first packet ## of a previously unknown connection. Bro uses a flow-based definition of ## "connection" here that includes not only TCP sessions but also UDP and ICMP -## flows. -## -## c: The connection. -## -## .. note:: Handling this event is potentially expensive. For example, during a -## SYN flooding attack, every spoofed SYN packet will lead to a new new_connection -## event. +## flows. ## -## see:: *connection* +## c: The connection. +## +## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt +## connection_established connection_external connection_finished +## connection_first_ACK connection_half_finished connection_partial_close +## connection_pending connection_rejected connection_reset connection_reused +## connection_state_remove connection_status_update connection_timeout +## expected_connection_seen new_connection_contents partial_connection +## +## .. note:: +## +## Handling this event is potentially expensive. For example, during a SYN +## flooding attack, every spoofed SYN packet will lead to a new +## event. event new_connection%(c: connection%); ## Generated when reassembly starts for a TCP connection. The event is raised ## at the moment when Bro's TCP analyzer enables stream reassembly for a -## connection. +## connection. ## -## c: The connection. +## c: The connection. ## -## see:: *connection* +## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt +## connection_established connection_external connection_finished +## connection_first_ACK connection_half_finished connection_partial_close +## connection_pending connection_rejected connection_reset connection_reused +## connection_state_remove connection_status_update connection_timeout +## expected_connection_seen new_connection partial_connection event new_connection_contents%(c: connection%); ## Generated for an unsuccessful connection attempt. The event is raised when an ## originator unsuccessfully attempted to establish a connection. "Unsuccessful" ## is defined as at least :bro:id:`tcp_attempt_delay` seconds having elapsed since ## the originator first sent a connection establishment packet to the destination -## without seeing a reply. -## -## c: The connection. -## -## see:: connection_*, new_connection*, partial_connection. +## without seeing a reply. +## +## c: The connection. +## +## .. bro:see:: connection_EOF connection_SYN_packet connection_established +## connection_external connection_finished connection_first_ACK +## connection_half_finished connection_partial_close connection_pending +## connection_rejected connection_reset connection_reused connection_state_remove +## connection_status_update connection_timeout expected_connection_seen +## new_connection new_connection_contents partial_connection event connection_attempt%(c: connection%); ## Generated for an established TCP connection. The event is raised when the -## initial 3-way TCP handshake has successfully finished for a connection. -## -## c: The connection. -## -## see:: *connection* +## initial 3-way TCP handshake has successfully finished for a connection. +## +## c: The connection. +## +## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt +## connection_external connection_finished connection_first_ACK +## connection_half_finished connection_partial_close connection_pending +## connection_rejected connection_reset connection_reused connection_state_remove +## connection_status_update connection_timeout expected_connection_seen +## new_connection new_connection_contents partial_connection event connection_established%(c: connection%); ## Generated for a new active TCP connection if Bro did not see the initial ## handshake. The event is raised when Bro has observed traffic from each endpoint, -## but the activity did not begin with the usual connection establishment. +## but the activity did not begin with the usual connection establishment. ## ## c: The connection. -## -## see:: *connection* +## +## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt +## connection_established connection_external connection_finished +## connection_first_ACK connection_half_finished connection_partial_close +## connection_pending connection_rejected connection_reset connection_reused +## connection_state_remove connection_status_update connection_timeout +## expected_connection_seen new_connection new_connection_contents +## event partial_connection%(c: connection%); ## TODO. ## -## see:: *connection* +## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt +## connection_established connection_external connection_finished +## connection_first_ACK connection_half_finished connection_pending +## connection_rejected connection_reset connection_reused connection_state_remove +## connection_status_update connection_timeout expected_connection_seen +## new_connection new_connection_contents partial_connection event connection_partial_close%(c: connection%); -## Generated for a TCP connection that finished normally. The event is raised -## when a regular FIN handshake from both endpoints was observed. -## +## Generated for a TCP connection that finished normally. The event is raised +## when a regular FIN handshake from both endpoints was observed. +## ## c: The connection. -## -## see:: *connection* +## +## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt +## connection_established connection_external connection_first_ACK +## connection_half_finished connection_partial_close connection_pending +## connection_rejected connection_reset connection_reused connection_state_remove +## connection_status_update connection_timeout expected_connection_seen +## new_connection new_connection_contents partial_connection event connection_finished%(c: connection%); ## TODO. ## -## see:: *connection* +## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt +## connection_established connection_external connection_finished +## connection_first_ACK connection_partial_close connection_pending +## connection_rejected connection_reset connection_reused connection_state_remove +## connection_status_update connection_timeout expected_connection_seen +## new_connection new_connection_contents partial_connection event connection_half_finished%(c: connection%); ## Generated for a rejected TCP connection. The event is raised when an originator ## attempted to setup a TCP connection but the responder replied with a RST packet -## denying it. -## -## c: The connection. -## -## .. note:: If the responder does not respond at all, :bro:id:`connection_attempt` -## is raised instead. If the responder initially accepts the connection but -## aborts it later, Bro first generates :bro:id:`connection_established` and then -## :bro:id:`connection_reset`. +## denying it. ## -## see:: *connection* +## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt +## connection_established connection_external connection_finished +## connection_first_ACK connection_half_finished connection_partial_close +## connection_pending connection_reset connection_reused connection_state_remove +## connection_status_update connection_timeout expected_connection_seen +## new_connection new_connection_contents partial_connection +## +## c: The connection. +## +## .. note:: +## +## If the responder does not respond at all, :bro:id:`connection_attempt` is +## raised instead. If the responder initially accepts the connection but aborts +## it later, Bro first generates :bro:id:`connection_established` and then +## :bro:id:`connection_reset`. event connection_rejected%(c: connection%); ## Generated when an endpoint aborted a TCP connection. The event is raised ## when one endpoint of an established TCP connection aborted by sending a RST -## packet. -## -## c: The connection. +## packet. ## -## see:: *connection* +## c: The connection. +## +## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt +## connection_established connection_external connection_finished +## connection_first_ACK connection_half_finished connection_partial_close +## connection_pending connection_rejected connection_reused +## connection_state_remove connection_status_update connection_timeout +## expected_connection_seen new_connection new_connection_contents +## partial_connection event connection_reset%(c: connection%); ## TODO ## -## see:: *connection* +## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt +## connection_established connection_external connection_finished +## connection_first_ACK connection_half_finished connection_partial_close +## connection_rejected connection_reset connection_reused connection_state_remove +## connection_status_update connection_timeout expected_connection_seen +## new_connection new_connection_contents partial_connection event connection_pending%(c: connection%); ## Generated when a connection's internal state is about to be removed from @@ -142,524 +254,5192 @@ event connection_pending%(c: connection%); ## is about to delete the internal state. As such, the event is well-suited for ## scrip-level cleanup that needs to be performed for every connection. The ## ``connection_state_remove`` event is generated not only for TCP sessions but -## also for UDP and ICMP flows. -## +## also for UDP and ICMP flows. +## ## c: The connection. -## -## see:: *connection* +## +## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt +## connection_established connection_external connection_finished +## connection_first_ACK connection_half_finished connection_partial_close +## connection_pending connection_rejected connection_reset connection_reused +## connection_status_update connection_timeout expected_connection_seen +## new_connection new_connection_contents partial_connection udp_inactivity_timeout +## tcp_inactivity_timeout icmp_inactivity_timeout event connection_state_remove%(c: connection%); ## Generated for a SYN packet. Bro raises this event for every SYN packet seen by -## its TCP analyzer. -## -## c: The connection. -## -## pkt: Information extracted from the SYN packet. -## -## .. note:: This event has quite low-level semantics and can potentially be -## expensive to generate. It should only be used if one really needs the -## specific information passed into the handler via the ``pkt`` argument. If -## not, handling one of the other ``connection_*`` events is typically the -## better approach. +## its TCP analyzer. ## -## see:: *connection* +## c: The connection. +## +## pkt: Information extracted from the SYN packet. +## +## .. bro:see:: connection_EOF connection_attempt connection_established +## connection_external connection_finished connection_first_ACK +## connection_half_finished connection_partial_close connection_pending +## connection_rejected connection_reset connection_reused connection_state_remove +## connection_status_update connection_timeout expected_connection_seen +## new_connection new_connection_contents partial_connection +## +## .. note:: +## +## This event has quite low-level semantics and can potentially be expensive to +## generate. It should only be used if one really needs the specific information +## passed into the handler via the ``pkt`` argument. If not, handling one of the +## other ``connection_*`` events is typically the better approach. event connection_SYN_packet%(c: connection, pkt: SYN_packet%); ## Generated for the first ACK packet seen for a TCP connection from -## its *orginator*. -## -## c: The connection. -## -## pkt: Information extracted from the SYN packet. -## -## .. note:: This event has quite low-level semantics and should be used only -## rarely. +## its *orginator*. ## -## see:: *connection* +## c: The connection. +## +## pkt: Information extracted from the SYN packet. +## +## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt +## connection_established connection_external connection_finished +## connection_half_finished connection_partial_close connection_pending +## connection_rejected connection_reset connection_reused connection_state_remove +## connection_status_update connection_timeout expected_connection_seen +## new_connection new_connection_contents partial_connection +## +## .. note:: +## +## This event has quite low-level semantics and should be used only rarely. event connection_first_ACK%(c: connection%); ## Generated when a TCP connection timed out. This event is raised when no activity ## was seen for an interval of at least :bro:id:`tcp_connection_linger`, and ## either one endpoint has already closed the connection or one side never -## never became active. -## +## never became active. +## ## c: The connection. -## -## .. note:: The precise semantics of this event can be unintuitive as it -## only covers a subset of cases where a connection times out. Often, handling +## +## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt +## connection_established connection_external connection_finished +## connection_first_ACK connection_half_finished connection_partial_close +## connection_pending connection_rejected connection_reset connection_reused +## connection_state_remove connection_status_update expected_connection_seen +## new_connection new_connection_contents partial_connection +## +## .. note:: +## +## The precise semantics of this event can be unintuitive as it only +## covers a subset of cases where a connection times out. Often, handling ## :bro:id:`connection_state_remove` is the better option. That one will be ## generated reliably when an interval of ``tcp_inactivity_timeout`` has passed ## with out any activity seen (but also for all other ways a connection may ## terminate). -## -## see:: *connection* event connection_timeout%(c: connection%); ## Generated when a connection 4-tuple is reused. The event is raised when Bro ## sees a new TCP session or UDP flow using a 4-tuple matching that of an earlier -## connection it still consideres active. +## connection it still consideres active. ## -## c: The connection. -## -## see:: *connection* +## c: The connection. +## +## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt +## connection_established connection_external connection_finished +## connection_first_ACK connection_half_finished connection_partial_close +## connection_pending connection_rejected connection_reset connection_state_remove +## connection_status_update connection_timeout expected_connection_seen +## new_connection new_connection_contents partial_connection event connection_reused%(c: connection%); - - +## Generated in regular intervals during the life time of a connection. The +## events is raised each :bro:id:`connection_status_update_interval` seconds +## and can be used to check conditions on a regular basis. +## +## c: The connection. +## +## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt +## connection_established connection_external connection_finished +## connection_first_ACK connection_half_finished connection_partial_close +## connection_pending connection_rejected connection_reset connection_reused +## connection_state_remove connection_timeout expected_connection_seen +## new_connection new_connection_contents partial_connection event connection_status_update%(c: connection%); + +## Generated at the end of reassembled TCP connections. The TCP reassembler +## raised the event once for each endpoint of a connection when it finished +## reassembling the corresponding side of the communication. +## +## c: The connection. +## +## is_orig: True if the event is raised for the originator side. +## +## .. bro:see:: connection_SYN_packet connection_attempt connection_established +## connection_external connection_finished connection_first_ACK +## connection_half_finished connection_partial_close connection_pending +## connection_rejected connection_reset connection_reused connection_state_remove +## connection_status_update connection_timeout expected_connection_seen +## new_connection new_connection_contents partial_connection event connection_EOF%(c: connection, is_orig: bool%); + +## Generated for a new connection received from the communication subsystem. Remote +## peers can inject packets into Bro's packet loop, for example via TODO_Broccoli. +## The communication systems raises this event with the first packet of a +## connection coming in this way. event connection_external%(c: connection, tag: string%); +## Generated when a connected is seen that has previously marked as being expected. +## The function :bro:id:`expect_connection` tells Bro to expect a particular +## connection to come up, and which analyzer to associate with it. Once the +## first packet of such a connection is indeed seen, this event is raised. +## +## c: The connection. +## +## a: The analyzer that was scheduled for the connection with the +## :bro:id:`expect_connection` call. When the event is raised, that +## analyzer will already have been activated to process the connection. The +## ``count`` is one of the ``ANALYZER_*`` constants, see TODO. +## +## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt +## connection_established connection_external connection_finished +## connection_first_ACK connection_half_finished connection_partial_close +## connection_pending connection_rejected connection_reset connection_reused +## connection_state_remove connection_status_update connection_timeout +## new_connection new_connection_contents partial_connection event expected_connection_seen%(c: connection, a: count%); +## Generated for every packet Bro sees. This is a very low-level and expensive +## event that should be avoided when at all possible. Is's usually infeasible to +## handle when processing even medium volumes of traffic in real-time. That +## said, if you work from a trace and want to do some packet-level analysis, +## it may come in handy. +## +## c: The connection the packet is part of. +## +## p: Informattion from the header of the packet that triggered the event. +## +## .. bro:see:: tcp_packet packet_contents event new_packet%(c: connection, p: pkt_hdr%); +## Generated for every packet that has non-empty transport-layer payload. This is a +## very low-level and expensive event that should be avoided when at all possible. +## It's usually infeasible to handle when processing even medium volumes of +## traffic in real-time. It's even worse than :bro:id:`new_packet`. That said, if +## you work from a trace and want to do some packet-level analysis, it may come in +## handy. +## +## c: The connection the packet is part of. +## +## contants: The raw transport-layer payload. +## +## .. bro:see:: new_packet tcp_packet +event packet_contents%(c: connection, contents: string%); + +## Generated for every TCP packet. This is a very low-level and expensive event +## that should be avoided when at all possible. It's usually infeasible to handle +## when processing even medium volumes of traffic in real-time. It's slightly +## better than :bro:id:`new_packet` because it affects only TCP, but not much. That +## said, if you work from a trace and want to do some packet-level analysis, it may +## come in handy. +## +## c: The connection the packet is part of. +## +## is_orig: True if the packet was sent by the connection's originator. +## +## flags: A string with the packet's TCP flags. In the string, each character +## corresponds to one set flag, as follows: ``S`` -> SYN; ``F`` -> FIN; +## ``R`` -> RST; ``A`` -> ACK; ``P`` -> PUSH. +## +## seq: The packet's TCP sequence number. +## +## ack: The packet's ACK number. +## +## len: The length of the TCP payload, as specified in the packet header. +## +## payload: The raw TCP payload. Note that this may less than *len* if the packet +## was not fully captured. +## +## .. bro:see:: new_packet packet_contents tcp_option tcp_contents tcp_rexmit +event tcp_packet%(c: connection, is_orig: bool, flags: string, seq: count, ack: count, len: count, payload: string%); + +## Generated for each option found in a TCP header. Like many of the ``tcp_*`` +## events, this is a very low-level event and potentially expensive as it may +## be raised very often. +## +## c: The connection the packet is part of. +## +## is_orig: True if the packet was sent by the connection's originator. +## +## opt: The numerical option number, as found in the TCP header. +## +## optlen: The length of the options value. +## +## .. bro:see:: tcp_packet tcp_contents tcp_rexmit +## +## .. note:: There is currently no way to get the actual option value, if any. +event tcp_option%(c: connection, is_orig: bool, opt: count, optlen: count%); + +## Generated for each chunk of reassembled TCP payload. When reassembly is enabled +## for a TCP connection (as it is automatically whenever there is TCP protocol +## analyzer parsing it), this event is raised for each chunk of in-order payload +## reconstructed from the packet stream. Note that this event is potentially +## expensive if many connections carry signficant amounts of data as then all +## that needs to be passed on to the scripting layer. +## +## TODO: add link to options (or discussionof options) for enabling TCP +## reaseembly in other ways. +## +## c: The connection the payload is part of. +## +## is_orig: True if the packet was sent by the connection's originator. +## +## seq: The sequence number corresponding to the first byte of the payload +## chunk. +## +## payload: The raw payload, which will be non-empty. +## +## .. bro:see:: tcp_packet tcp_option tcp_rexmit +## +## .. note:: +## +## The payload received by this event is the same that is also passed into +## application-layer protocol analyzers internally. Subsequent invocations of +## this event for the same connection receive non-overlapping in-order chunks +## of its TCP payload stream. It is however undefined what size each chunk +## has; while Bro passes the data on as soon as possible, specifics depend +## on network-level effects such as latency, acknowledgements, reordering, +## etc. Also note that Bro's reassembler holds on to any data until it has seen +## the other side's acknowledgement. +event tcp_contents%(c: connection, is_orig: bool, seq: count, contents: string%); + +## Generated +event tcp_rexmit%(c: connection, is_orig: bool, seq: count, len: count, data_in_flight: count, window: count%); + +## Generated when Bro detects a TCP retransmission inconsistency. When +## reassemling TCP stream, Bro buffers all payload until it seens the responder +## acking it. If during time, the sender resends a chunk of payload but with +## content than originally, this event will be raised. +## +## c: The connection showing the inconsistency. +## +## t1: The original payload. +## +## t2: The new payload. +## +## .. bro:see:: tcp_rexmit tcp_contents +event rexmit_inconsistency%(c: connection, t1: string, t2: string%); + +## Generated when a TCP endpoint acknowledges payload that Bro did never see. +## +## c: The connection. +## +## .. bro:see:: content_gap +## +## .. note:: +## +## Seeing an acknowledgment indicates that the responder of the connection +## says it has received the corresponding data. If Bro did not, it must have +## either missed one or more packets, or the responder's TCP stack is broken +## (which isn't unheard of). In practice, one will always see a few of these +## events in any larger volume of network traffic. If there are lots of them, +## however, that typically means that there is a problem with the monitoring +## infrastructure such as a tap dropping packets, split routing on the path, or +## reordering at the tap. +## +## This event reports similar situations as :bro:id:`content_gap`, though their +## specifics differ slightly. Often, however, both will be raised for the same +## connection if some of its data is missing. We should eventually merge +## the two. +event ack_above_hole%(c: connection%); + +## Generated when Bro detects a gap in a reassembled TCP payload stream. This event +## is raised when Bro, while reassemling a payload stream, determines that a chunk +## of payload is missing (e.g., because the responder has already acknowledged it, +## even though Bro didn't see it). +## +## c: The connection. +## +## is_orig: True if the gap is on the originator's side. +## +## seq: The sequence number where the gap starts. +## +## length: The number of bytes missing. +## +## .. bro:see:: ack_above_hole +## +## .. note:: +## +## Content gaps tend to occur occasionally for various reasons, including broken +## TCP stacks. If, however, one finds lots of them, that typically means that +## there is a problem with the monitoring infrastructure such as a tap dropping +## packets, split routing on the path, or reordering at the tap. +## +## This event reports similar situations as :bro:id:`ack_above_hole`, though +## their specifics differ slightly. Often, however, both will be raised for +## connection if some of its data is missing. We should eventually merge the +## two. +event content_gap%(c: connection, is_orig: bool, seq: count, length: count%); + +## Summarizes the amount of missing TCP payload at regular intervals. Internally, +## Bro tracks (1) the number of :bro:id:`ack_above_hole` events, including the +## numer of bytes missing; and (2) the total number of TCP acks seen, with the +## total volume of bytes that have been acked. This event reports these statistics +## in :bro:id:`gap_report_freq` intervals for the purpose of determining packet +## loss. +## +## dt: The time that has past since the last ``gap_report`` interval. +## +## info: The gap statistics. +## +## .. bro:see:: content_gap ack_above_hole +## +## .. note:: +## +## Bro comes with a script :doc:`/scripts/policy/misc/capture-loss.bro` that +## uses this event to estimate packet loss and report when a predefined +## threshold is exceeded. +event gap_report%(dt: interval, info: gap_info%); + + +## Generated when a protocol analyzer confirms that a connection is indeed +## using that protocol. Bro's TODO_dynamic protocol detection heuristically +## activates analyzers as soon as it believe a connection *could* be using a +## particular protocol. It is then left to the corresponding analyzer to +## verify whether that is indeed the case; if so, this event will be +## generated. +## +## c: The connection. +## +## atype: The type of the analyzer confirming that its protocol is in +## use. The value is one of the ``ANALYZER_*`` constants. For example, +## :bro:id:`ANALYZER_HTTP` means the HTTP analyzers determined that it's indeed +## parsing an HTTP connection. +## +## aid: A unique integer ID identifying the specific *instance* of the +## analyzer *atype* that is analyzing the connection ``c``. The ID can +## be used to reference the analyzer when using builtin functions like +## :bro:id:`disable_analyzer`. +## +## .. bro:see:: protocol_violation +## +## .. note:: +## +## Bro's default scripts use this event to determine the ``service`` column of +## TODO``conn.log``: once confirmed, the protocol will be listed there. event protocol_confirmation%(c: connection, atype: count, aid: count%); + +## Generated when a protocol analyzer determines that a connection it is parsing is +## not conforming to the protocol it expects. Bro's TODO_dynamic protocol detection +## heuristically activates analyzers as soon as it believe a connection *could* be +## using a particular protocol. It is then left to the corresponding analyzer to +## verify whether that is indeed the case; if not, the analyzer will +## trigger this event. +## +## c: The connection. +## +## atype: The type of the analyzer confirming that its protocol is in +## use. The value is one of the ``ANALYZER_*`` constants. For example, +## :bro:id:`ANALYZER_HTTP` means the HTTP analyzers determined that it's indeed +## parsing an HTTP connection. +## +## aid: A unique integer ID identifying the specific *instance* of the +## analyzer *atype* that is analyzing the connection ``c``. The ID can +## be used to reference the analyzer when using builtin functions like +## :bro:id:`disable_analyzer`. +## +## .. bro:see:: protocol_confirmation +## +## .. note:: +## +## Bro's default scripts use this event to disable an analyzer via +## :bro:id:`disable_analyzer` if it's parsing the wrong protocol. That's however +## a script-level decision and not done automatically by the event eninge. event protocol_violation%(c: connection, atype: count, aid: count, reason: string%); -event packet_contents%(c: connection, contents: string%); -event tcp_packet%(c: connection, is_orig: bool, flags: string, seq: count, ack: count, len: count, payload: string%); -event tcp_option%(c: connection, is_orig: bool, opt: count, optlen: count%); -event tcp_contents%(c: connection, is_orig: bool, seq: count, contents: string%); -event tcp_rexmit%(c: connection, is_orig: bool, seq: count, len: count, data_in_flight: count, window: count%); +## Generated for each packet sent by a UDP flow's originator. This a potentially +## expsensive event due to the volume of UDP traffic and should be used with care. +## +## u: The connection record for the corresponding UDP flow. +## +## .. bro:see:: udp_contents udp_reply udp_session_done event udp_request%(u: connection%); -event udp_reply%(u: connection%); -event udp_contents%(u: connection, is_orig: bool, contents: string%); -event udp_session_done%(u: connection%); -event icmp_sent%(c: connection, icmp: icmp_conn%); -event icmp_echo_request%(c: connection, icmp: icmp_conn, id: count, seq: count, payload: string%); -event icmp_echo_reply%(c: connection, icmp: icmp_conn, id: count, seq: count, payload: string%); -event icmp_unreachable%(c: connection, icmp: icmp_conn, code: count, context: icmp_context%); -event icmp_time_exceeded%(c: connection, icmp: icmp_conn, code: count, context: icmp_context%); -event icmp_redirect%(c: connection, icmp: icmp_conn, a: addr%); -event conn_stats%(c: connection, os: endpoint_stats, rs: endpoint_stats%); -event conn_weird%(name: string, c: connection, addl: string%); -event flow_weird%(name: string, src: addr, dst: addr%); -event net_weird%(name: string%); -event load_sample%(samples: load_sample_info, CPU: interval, dmem: int%); -event rexmit_inconsistency%(c: connection, t1: string, t2: string%); -event ack_above_hole%(c: connection%); -event content_gap%(c: connection, is_orig: bool, seq: count, length: count%); -event gap_report%(dt: interval, info: gap_info%); -event inconsistent_option%(c: connection%); -event bad_option%(c: connection%); -event bad_option_termination%(c: connection%); +## Generated for each packet sent by a UDP flow's responder. This a potentially +## expsensive event due to the volume of UDP traffic and should be used with care. +## +## u: The connection record for the corresponding UDP flow. +## +## .. bro:see:: udp_contents udp_request udp_session_done +event udp_reply%(u: connection%); + +## Generated for UDP packets to pass on their payload. As the number of UDP +## packets can be very large, this event is normally raised only for those on +## ports configured in :bro:id:`udp_content_delivery_ports_orig` (for packets sent +## by the flow's orgininator) or :bro:id:`udp_content_delivery_ports_resp` (for +## packets sent by the flow's responder). However, delivery can be enabled for all +## UDP request and reply packets by setting :bro:id:`udp_content_deliver_all_orig` +## or :bro:id:`udp_content_deliver_all_resp`, respectively. Note that this event is +## also raised for all matching UDP packets, including empty ones. +## +## u: The connection record for the corresponding UDP flow. +## +## is_orig: True if the event is raised for the originator side. +## +## .. bro:see:: udp_reply udp_request udp_session_done +## udp_content_deliver_all_orig udp_content_deliver_all_resp +## udp_content_delivery_ports_orig udp_content_delivery_ports_resp +event udp_contents%(u: connection, is_orig: bool, contents: string%); + +## Generated when a UDP session for a supported protocol has finished. Some of +## Bro's application-layer UDP analyzers flag the end of a session by raising this +## event. Currently, the analyzers for DNS, NTP, Netbios, and Syslog support this. +## +## u: The connection record for the corresponding UDP flow. +## +## .. bro:see:: udp_contents udp_reply udp_request +event udp_session_done%(u: connection%); + +## Generated for all ICMP messages that are not handled separetely with dedicated +## ICMP events. Bro's ICMP analyzer handles a number of ICMP messages directly +## with dedicated events. This handlers acts as a fallback for those it doesn't. +## The *icmp* record provides more information about the message. +## +## See `Wikipedia +## `__ for more +## information about the ICMP protocol. +## +## c: The connection record for the corresponding ICMP flow. +## +## icmp: Additional ICMP-specific information augmenting the standard +## connection record *c*. +## +## .. bro:see:: icmp_echo_reply icmp_echo_request icmp_redirect +## icmp_time_exceeded icmp_unreachable +event icmp_sent%(c: connection, icmp: icmp_conn%); + +## Generated for ICMP *echo request* messages. +## +## See `Wikipedia +## `__ for more +## information about the ICMP protocol. +## +## c: The connection record for the corresponding ICMP flow. +## +## icmp: Additional ICMP-specific information augmenting the standard +## connection record *c*. +## +## id: The *echo request* identifier. +## +## seq: The *echo request* sequence number. +## +## payload: The message-specific data of the packet payload, i.e., everything after +## the first 8 bytes of the ICMP header. +## +## .. bro:see:: icmp_echo_reply icmp_redirect icmp_sent +## icmp_time_exceeded icmp_unreachable +event icmp_echo_request%(c: connection, icmp: icmp_conn, id: count, seq: count, payload: string%); + +## Generated for ICMP *echo reply* messages. +## +## See `Wikipedia +## `__ for more +## information about the ICMP protocol. +## +## c: The connection record for the corresponding ICMP flow. +## +## icmp: Additional ICMP-specific information augmenting the standard connection +## record *c*. +## +## id: The *echo reply* identifier. +## +## seq: The *echo reply* sequence number. +## +## payload: The message-specific data of the packet payload, i.e., everything after +## the first 8 bytes of the ICMP header. +## +## .. bro:see:: icmp_echo_request icmp_redirect icmp_sent +## icmp_time_exceeded icmp_unreachable +event icmp_echo_reply%(c: connection, icmp: icmp_conn, id: count, seq: count, payload: string%); + +## Generated for ICMP *destination unreachable* messages. +## +## See `Wikipedia +## `__ for more +## information about the ICMP protocol. +## +## c: The connection record for the corresponding ICMP flow. +## +## icmp: Additional ICMP-specific information augmenting the standard connection +## record *c*. +## +## code: The ICMP code of the *unreachable* message. +## +## context: A record with specifics of the original packet that the message refers +## to. *Unreachable* messages should include the original IP header from the packet +## that triggered them, and Bro parses that into the *context* structure. Note +## that if the *unreachable* includes only a partial IP header for some reason, no +## fields of *context* will be filled out. +## +## .. bro:see:: icmp_echo_reply icmp_echo_request icmp_redirect icmp_sent +## icmp_time_exceeded +event icmp_unreachable%(c: connection, icmp: icmp_conn, code: count, context: icmp_context%); + +## Generated for ICMP *time exceeded* messages. +## +## See `Wikipedia +## `__ for more +## information about the ICMP protocol. +## +## c: The connection record for the corresponding ICMP flow. +## +## icmp: Additional ICMP-specific information augmenting the standard connection +## record *c*. +## +## code: The ICMP code of the *exceeded* message. +## +## context: A record with specifics of the original packet that the message refers +## to. *Unreachable* messages should include the original IP header from the packet +## that triggered them, and Bro parses that into the *context* structure. Note that +## if the *exceeded* includes only a partial IP header for some reason, no fields +## of *context* will be filled out. +## +## .. bro:see:: icmp_echo_reply icmp_echo_request icmp_redirect icmp_sent +## icmp_unreachable +event icmp_time_exceeded%(c: connection, icmp: icmp_conn, code: count, context: icmp_context%); + +## Generated for ICMP *redirect* messages. +## +## See `Wikipedia +## `__ for more +## information about the ICMP protocol. +## +## c: The connection record for the corresponding ICMP flow. +## +## icmp: Additional ICMP-specific information augmenting the standard connection +## record *c*. +## +## a: The new destination address the message is redirecting to. +## +## .. bro:see:: icmp_echo_reply icmp_echo_request icmp_sent +## icmp_time_exceeded icmp_unreachable +event icmp_redirect%(c: connection, icmp: icmp_conn, a: addr%); + +## TODO. +event conn_stats%(c: connection, os: endpoint_stats, rs: endpoint_stats%); + +## Generated for unexpected activity related to a specific connection. When +## Bro's packet analysis encounters activity that does not conform to a protocol's +## specification, it raises one of the ``*_weird`` events to report that. This +## event is raised if the activity is tied directly to a specific connection. +## +## name: A unique name for the specific type of "weird" situation. Bro's default +## scripts use this name in filtering policies that specify which "weirds" are +## worth reporting. +## +## c: The corresponding connection. +## +## addl: Optional additional context further describing the situation. +## +## .. bro:see:: flow_weird net_weird +## +## .. note:: "Weird" activity is much more common in real-world network traffic +## than one would intuitively expect. While in principle, any protocol violation +## could be an attack attempt, it's much more likely that an endpoint's +## implementation interprets an RFC quite liberally. +event conn_weird%(name: string, c: connection, addl: string%); + +## Generated for unexpected activity related to a pair of hosts, but independent +## of a specific connection. When Bro's packet analysis encounters activity that +## does not conform to a protocol's specification, it raises one of the ``*_weird`` +## event to report that. This event is raised if the activity is related to a +## pair of hosts, yet not to a specific connection between them. +## +## name: A unique name for the specific type of "weird" situation. Bro's default +## scripts use this name in filtering policies that specify which "weirds" are +## worth reporting. +## +## src: The source address corresponding to the activity. +## +## dst: The destination address corresponding to the activity. +## +## .. bro:see:: conn_weird net_weird +## +## .. note:: "Weird" activity is much more common in real-world network traffic +## than one would intuitively expect. While in principle, any protocol violation +## could be an attack attempt, it's much more likely that an endpoint's +## implementation interprets an RFC quite liberally. +event flow_weird%(name: string, src: addr, dst: addr%); + +## Generated for unexpected activity that is not tied to a specific connection +## or pair of hosts. When Bro's packet analysis encounters activity that +## does not conform to a protocol's specification, it raises one of the +## ``*_weird`` event to report that. This event is raised if the activity is +## not tied directly to a specific connection or pair of hosts. +## +## name: A unique name for the specific type of "weird" situation. Bro's default +## scripts use this name in filtering policies that specify which "weirds" are +## worth reporting. +## +## .. bro:see:: flow_weird +## +## .. note:: "Weird" activity is much more common in real-world network traffic +## than one would intuitively expect. While in principle, any protocol violation +## could be an attack attempt, it's much more likely that an endpoint's +## implementation interprets an RFC quite liberally. +event net_weird%(name: string%); + +## Generated regularly for the purpose of profiling Bro's processing. This event +## is raised for every :bro:id:`load_sample_freq` packet. For these packets, +## Bro records script-level functions executed during their processing as well as +## further internal locations. By sampling the processing in this form, one can +## understand where Bro spends its time. +## +## samples: A set with functions and locations seens during the processing of +## the sampled packet. +## +## CPU: The CPU time spent on processing the sampled. +## +## dmem: The difference in memory usage caused by processing the sampled packet. +event load_sample%(samples: load_sample_info, CPU: interval, dmem: int%); + +## Generated for ARP requests. +## +## See `Wikipedia `__ for +## more information about the ARP protocol. +## +## mac_src: The request's source MAC address. +## +## mac_dst: The request's destination MAC address. +## +## SPA: The sender protocol address. +## +## SHA: The sender hardware address. +## +## TPA: The target protocol address. +## +## THA: The target hardware address. +## +## .. bro:see:: arp_reply bad_arp event arp_request%(mac_src: string, mac_dst: string, SPA: addr, SHA: string, TPA: addr, THA: string%); + +## Generated for ARP replies. +## +## See `Wikipedia `__ for +## more information about the ARP protocol. +## +## mac_src: The replies's source MAC address. +## +## mac_dst: The replies's destination MAC address. +## +## SPA: The sender protocol address. +## +## SHA: The sender hardware address. +## +## TPA: The target protocol address. +## +## THA: The target hardware address. +## +## .. bro:see:: arp_request bad_arp event arp_reply%(mac_src: string, mac_dst: string, SPA: addr, SHA: string, TPA: addr, THA: string%); + +## Generated for ARP packets that Bro cannot interpret. Examples are packets with +## non-standard hardware address formats or hardware addresses that not match the +## originator of the packet. +## +## SPA: The sender protocol address. +## +## SHA: The sender hardware address. +## +## TPA: The target protocol address. +## +## THA: The target hardware address. +## +## explanation: A short description of why the ARP packet is considered "bad". +## +## .. bro:see:: arp_reply arp_request event bad_arp%(SPA: addr, SHA: string, TPA: addr, THA: string, explanation: string%); +## TODO. +## +## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke +## bittorrent_peer_have bittorrent_peer_interested bittorrent_peer_keep_alive +## bittorrent_peer_not_interested bittorrent_peer_piece bittorrent_peer_port +## bittorrent_peer_request bittorrent_peer_unchoke bittorrent_peer_unknown +## bittorrent_peer_weird event bittorrent_peer_handshake%(c: connection, is_orig: bool, reserved: string, info_hash: string, peer_id: string%); + +## TODO. +## +## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke +## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested +## bittorrent_peer_not_interested bittorrent_peer_piece bittorrent_peer_port +## bittorrent_peer_request bittorrent_peer_unchoke bittorrent_peer_unknown +## bittorrent_peer_weird event bittorrent_peer_keep_alive%(c: connection, is_orig: bool%); + +## TODO. +## +## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel +## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested +## bittorrent_peer_keep_alive bittorrent_peer_not_interested bittorrent_peer_piece +## bittorrent_peer_port bittorrent_peer_request bittorrent_peer_unchoke +## bittorrent_peer_unknown bittorrent_peer_weird event bittorrent_peer_choke%(c: connection, is_orig: bool%); + +## TODO. +## +## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke +## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested +## bittorrent_peer_keep_alive bittorrent_peer_not_interested bittorrent_peer_piece +## bittorrent_peer_port bittorrent_peer_request +## bittorrent_peer_unknown bittorrent_peer_weird event bittorrent_peer_unchoke%(c: connection, is_orig: bool%); + +## TODO. +## +## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke +## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_keep_alive +## bittorrent_peer_not_interested bittorrent_peer_piece bittorrent_peer_port +## bittorrent_peer_request bittorrent_peer_unchoke bittorrent_peer_unknown +## bittorrent_peer_weird event bittorrent_peer_interested%(c: connection, is_orig: bool%); + +## TODO. +## +## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke +## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested +## bittorrent_peer_keep_alive bittorrent_peer_piece bittorrent_peer_port +## bittorrent_peer_request bittorrent_peer_unchoke bittorrent_peer_unknown +## bittorrent_peer_weird event bittorrent_peer_not_interested%(c: connection, is_orig: bool%); + +## TODO. +## +## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke +## bittorrent_peer_handshake bittorrent_peer_interested bittorrent_peer_keep_alive +## bittorrent_peer_not_interested bittorrent_peer_piece bittorrent_peer_port +## bittorrent_peer_request bittorrent_peer_unchoke bittorrent_peer_unknown +## bittorrent_peer_weird event bittorrent_peer_have%(c: connection, is_orig: bool, piece_index: count%); + +## TODO. +## +## .. bro:see:: bittorrent_peer_cancel bittorrent_peer_choke bittorrent_peer_handshake +## bittorrent_peer_have bittorrent_peer_interested bittorrent_peer_keep_alive +## bittorrent_peer_not_interested bittorrent_peer_piece bittorrent_peer_port +## bittorrent_peer_request bittorrent_peer_unchoke bittorrent_peer_unknown +## bittorrent_peer_weird event bittorrent_peer_bitfield%(c: connection, is_orig: bool, bitfield: string%); + +## TODO. +## +## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke +## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested +## bittorrent_peer_keep_alive bittorrent_peer_not_interested bittorrent_peer_piece +## bittorrent_peer_port bittorrent_peer_unchoke bittorrent_peer_unknown +## bittorrent_peer_weird event bittorrent_peer_request%(c: connection, is_orig: bool, index: count, begin: count, length: count%); + +## TODO. +## +## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke +## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested +## bittorrent_peer_keep_alive bittorrent_peer_not_interested bittorrent_peer_port +## bittorrent_peer_request bittorrent_peer_unchoke bittorrent_peer_unknown +## bittorrent_peer_weird event bittorrent_peer_piece%(c: connection, is_orig: bool, index: count, begin: count, piece_length: count%); + +## TODO. +## +## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_choke +## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested +## bittorrent_peer_keep_alive bittorrent_peer_not_interested bittorrent_peer_piece +## bittorrent_peer_port bittorrent_peer_request bittorrent_peer_unchoke +## bittorrent_peer_unknown bittorrent_peer_weird event bittorrent_peer_cancel%(c: connection, is_orig: bool, index: count, begin: count, length: count%); + +## TODO. +## +## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke +## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested +## bittorrent_peer_keep_alive bittorrent_peer_not_interested bittorrent_peer_piece +## bittorrent_peer_request bittorrent_peer_unchoke bittorrent_peer_unknown +## bittorrent_peer_weird event bittorrent_peer_port%(c: connection, is_orig: bool, listen_port: port%); + +## TODO. +## +## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke +## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested +## bittorrent_peer_keep_alive bittorrent_peer_not_interested bittorrent_peer_piece +## bittorrent_peer_port bittorrent_peer_request bittorrent_peer_unchoke +## bittorrent_peer_weird event bittorrent_peer_unknown%(c: connection, is_orig: bool, message_id: count, data: string%); + +## TODO. +## +## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke +## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested +## bittorrent_peer_keep_alive bittorrent_peer_not_interested bittorrent_peer_piece +## bittorrent_peer_port bittorrent_peer_request bittorrent_peer_unchoke +## bittorrent_peer_unknown event bittorrent_peer_weird%(c: connection, is_orig: bool, msg: string%); + +## TODO. +## +## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke +## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested +## bittorrent_peer_keep_alive bittorrent_peer_not_interested bittorrent_peer_piece +## bittorrent_peer_port bittorrent_peer_request bittorrent_peer_unchoke +## bittorrent_peer_unknown bittorrent_peer_weird event bt_tracker_request%(c: connection, uri: string, headers: bt_tracker_headers%); + +## TODO. +## +## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke +## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested +## bittorrent_peer_keep_alive bittorrent_peer_not_interested bittorrent_peer_piece +## bittorrent_peer_port bittorrent_peer_request bittorrent_peer_unchoke +## bittorrent_peer_unknown bittorrent_peer_weird event bt_tracker_response%(c: connection, status: count, headers: bt_tracker_headers, peers: bittorrent_peer_set, benc: bittorrent_benc_dir%); + +## TODO. +## +## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke +## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested +## bittorrent_peer_keep_alive bittorrent_peer_not_interested bittorrent_peer_piece +## bittorrent_peer_port bittorrent_peer_request bittorrent_peer_unchoke +## bittorrent_peer_unknown bittorrent_peer_weird event bt_tracker_response_not_ok%(c: connection, status: count, headers: bt_tracker_headers%); + +## TODO. +## +## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke +## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested +## bittorrent_peer_keep_alive bittorrent_peer_not_interested bittorrent_peer_piece +## bittorrent_peer_port bittorrent_peer_request bittorrent_peer_unchoke +## bittorrent_peer_unknown bittorrent_peer_weird event bt_tracker_weird%(c: connection, is_orig: bool, msg: string%); +## Generated for Finger requests. +## +## See `Wikipedia `__ for more +## information about the Finger protocol. +## +## c: The connection. +## +## full: True if verbose information is requested (``/W`` switch). +## +## username: The request's user name. +## +## hostname: The request's host name. +## +## .. bro:see:: finger_reply event finger_request%(c: connection, full: bool, username: string, hostname: string%); + +## Generated for Finger replies. +## +## See `Wikipedia `__ for more +## information about the Finger protocol. +## +## c: The connection. +## +## reply_line: The reply as returned by the server +## +## .. bro:see:: finger_request event finger_reply%(c: connection, reply_line: string%); + +# Gnutella analyzer. + +## TODO. +## +## .. bro:see:: gnutella_binary_msg gnutella_establish gnutella_http_notify +## gnutella_not_establish gnutella_partial_binary_msg gnutella_signature_found +## event gnutella_text_msg%(c: connection, orig: bool, headers: string%); + +## TODO. +## +## .. bro:see:: gnutella_establish gnutella_http_notify gnutella_not_establish +## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg event gnutella_binary_msg%(c: connection, orig: bool, msg_type: count, ttl: count, hops: count, msg_len: count, payload: string, payload_len: count, trunc: bool, complete: bool%); + +## TODO. +## +## .. bro:see:: gnutella_binary_msg gnutella_establish gnutella_http_notify +## gnutella_not_establish gnutella_signature_found gnutella_text_msg event gnutella_partial_binary_msg%(c: connection, orig: bool, msg: string, len: count%); + +## TODO. +## +## .. bro:see:: gnutella_binary_msg gnutella_http_notify gnutella_not_establish +## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg event gnutella_establish%(c: connection%); + +## TODO. +## +## .. bro:see:: gnutella_binary_msg gnutella_establish gnutella_http_notify +## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg event gnutella_not_establish%(c: connection%); + +## TODO. +## +## .. bro:see:: gnutella_binary_msg gnutella_establish gnutella_not_establish +## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg event gnutella_http_notify%(c: connection%); +## Generated for Ident requests. +## +## See `Wikipedia `__ for more +## information about the Ident protocol. +## +## c: The connection. +## +## lport: The request's local port. +## +## rport: The request's remote port. +## +## .. bro:see:: ident_error ident_reply event ident_request%(c: connection, lport: port, rport: port%); + +## Generated for Ident replies. +## +## See `Wikipedia `__ for more +## information about the Ident protocol. +## +## c: The connection. +## +## lport: The corresponding request's local port. +## +## rport: The corresponding request's remote port. +## +## user_id: The user id returned by the reply. +## +## system: The operating system returned by the reply. +## +## .. bro:see:: ident_error ident_request event ident_reply%(c: connection, lport: port, rport: port, user_id: string, system: string%); + +## Generated for Ident error replies. +## +## See `Wikipedia `__ for more +## information about the Ident protocol. +## +## c: The connection. +## +## lport: The corresponding request's local port. +## +## rport: The corresponding request's remote port. +## +## line: The error description returned by the reply. +## +## .. bro:see:: ident_reply ident_request event ident_error%(c: connection, lport: port, rport: port, line: string%); +## Generated for Telnet/Rlogin login failures. The *login* analyzer inspects +## Telnet/Rlogin sessions to heuristically extract username and password +## information as well as the text returned by the login server. This event is +## raised if a login attempt appears to have been unsuccessful. +## +## c: The connection. +## +## user: The user name tried. +## +## client_user: For Telnet connections, this is an empty string, but for Rlogin +## connections, it is the client name passed in the initial authentication +## information (to check against .rhosts). +## +## password: The password tried. +## +## line: line is the line of text that led the analyzer to conclude that the +## authentication had failed. +## +## .. bro:see:: login_confused login_confused_text login_display login_input_line +## login_output_line login_prompt login_success login_terminal direct_login_prompts +## get_login_state login_failure_msgs login_non_failure_msgs login_prompts login_success_msgs +## login_timeouts set_login_state +## +## .. note:: The login analyzer depends on a set of script-level variables that +## need to configured with patterns identifying login attempts. This configuration +## has not yet been ported over from Bro 1.5 to Bro 2.x, and the analyzer is +## therefore not directly usable at the moment. event login_failure%(c: connection, user: string, client_user: string, password: string, line: string%); + +## Generated for successful Telnet/Rlogin logins. The *login* analyzer inspects +## Telnet/Rlogin sessions to heuristically extract username and password +## information as well as the text returned by the login server. This event is +## raised if a login attempt appears to have been successful. +## +## c: The connection. +## +## user: The user name used. +## +## client_user: For Telnet connections, this is an empty string, but for Rlogin +## connections, it is the client name passed in the initial authentication +## information (to check against .rhosts). +## +## password: The password used. +## +## line: line is the line of text that led the analyzer to conclude that the +## authentication had succeeded. +## +## .. bro:see:: login_confused login_confused_text login_display login_failure +## login_input_line login_output_line login_prompt login_terminal +## direct_login_prompts get_login_state login_failure_msgs login_non_failure_msgs +## login_prompts login_success_msgs login_timeouts set_login_state +## +## .. note:: The login analyzer depends on a set of script-level variables that +## need to configured with patterns identifying login attempts. This configuration +## has not yet been ported over from Bro 1.5 to Bro 2.x, and the analyzer is +## therefore not directly usable at the moment. event login_success%(c: connection, user: string, client_user: string, password: string, line: string%); + +## Generated for lines of input on Telnet/Rlogin sessions. The line will have +## control characters (such as in-band Telnet options) removed. +## +## c: The connection. +## +## line: The input line. +## +## .. bro:see:: login_confused login_confused_text login_display login_failure +## login_output_line login_prompt login_success login_terminal rsh_request event login_input_line%(c: connection, line: string%); + +## Generated for lines of output on Telnet/Rlogin sessions. The line will have +## control characters (such as in-band Telnet options) removed. +## +## c: The connection. +## +## line: The ouput line. +## +## .. bro:see:: login_confused login_confused_text login_display login_failure +## login_input_line login_prompt login_success login_terminal rsh_reply event login_output_line%(c: connection, line: string%); + +## Generated when tracking of Telnet/Rlogin authentication failed. As Bro's *login* +## analyzer uses a number of heuristics to extract authentication information, it +## may become confused. If it can no longer correctly track the authentication +## dialog, it raised this event. +## +## c: The connection. +## +## msg: Gives the particular problem the heuristics detected (for example, +## ``multiple_login_prompts`` means that the engine saw several login prompts in +## a row, without the type-ahead from the client side presumed necessary to cause +## them) +## +## line: The line of text that caused the heuristics to conclude they were +## confused. +## +## .. bro:see:: login_confused_text login_display login_failure login_input_line login_output_line +## login_prompt login_success login_terminal direct_login_prompts get_login_state +## login_failure_msgs login_non_failure_msgs login_prompts login_success_msgs +## login_timeouts set_login_state event login_confused%(c: connection, msg: string, line: string%); + +## Generated after getting confused while tracking a Telnet/Rlogin authentication +## dialog. The *login* analyzer generates this even for every line of user input +## after it has reported :bro:id:`login_confused` for a connection. +## +## c: The connection. +## +## line: The line the user typed. +## +## .. bro:see:: login_confused login_display login_failure login_input_line +## login_output_line login_prompt login_success login_terminal direct_login_prompts +## get_login_state login_failure_msgs login_non_failure_msgs login_prompts +## login_success_msgs login_timeouts set_login_state event login_confused_text%(c: connection, line: string%); + +## Generated for clients transmitting a terminal type in an Telnet session. This +## information is extracted out of environment variables sent as Telnet options. +## +## c: The connection. +## +## terminal: The TERM value transmitted. +## +## .. bro:see:: login_confused login_confused_text login_display login_failure +## login_input_line login_output_line login_prompt login_success event login_terminal%(c: connection, terminal: string%); + +## Generated for clients transmitting a X11 DISPLAY in a Telnet session. This +## information is extracted out of environment variables sent as Telnet options. +## +## c: The connection. +## +## terminal: The DISPLAY transmitted. +## +## .. bro:see:: login_confused login_confused_text login_failure login_input_line +## login_output_line login_prompt login_success login_terminal event login_display%(c: connection, display: string%); -event login_prompt%(c: connection, prompt: string%); -event rsh_request%(c: connection, client_user: string, server_user: string, line: string, new_session: bool%); -event rsh_reply%(c: connection, client_user: string, server_user: string, line: string%); -event excessive_line%(c: connection%); + +## Generated when a Telnet authentication has been successful. The Telnet protocol +## includes options for negotiating authentication. When such an option is sent +## from client to server and the server replies that it accepts the authentication, +## then the event engine generates this event. +## +## name: The authenticated name. +## +## c: The connection. +## +## .. bro:see:: authentication_rejected authentication_skipped login_success +## +## .. note:: This event inspects the corresponding Telnet option while :bro:id:`login_success` +## heuristically determines success by watching session data. event authentication_accepted%(name: string, c: connection%); + +## Generated when a Telnet authentication has been unsuccessful. The Telnet +## protocol includes options for negotiating authentication. When such an option +## is sent from client to server and the server replies that it did not accept the +## authentication, then the event engine generates this event. +## +## name: The attempted authentication name. +## +## c: The connection. +## +## .. bro:see:: authentication_accepted authentication_skipped login_failure +## +## .. note:: This event inspects the corresponding Telnet option while :bro:id:`login_success` +## heuristically determines failure by watching session +## data. event authentication_rejected%(name: string, c: connection%); + +## Generated when for Telnet/Rlogin sessions when a pattern match indicates +## that no authentication is performed. +## +## c: The connection. +## +## .. bro:see:: authentication_accepted authentication_rejected direct_login_prompts +## get_login_state login_failure_msgs login_non_failure_msgs login_prompts +## login_success_msgs login_timeouts set_login_state +## +## .. note:: The login analyzer depends on a set of script-level variables that +## need to be configured with patterns identifying actvity. This configuration has +## not yet been ported over from Bro 1.5 to Bro 2.x, and the analyzer is therefore +## not directly usable at the moment. event authentication_skipped%(c: connection%); + +## Generated for clients transmitting a terminal prompt in a Telnet session. This +## information is extracted out of environment variables sent as Telnet options. +## +## c: The connection. +## +## terminal: The TTYPROMPT transmitted. +## +## .. bro:see:: login_confused login_confused_text login_display login_failure +## login_input_line login_output_line login_success login_terminal +event login_prompt%(c: connection, prompt: string%); + +## Generated for Telnet sessions when encryption is activated. The Telnet protoco; +## includes options for negotiating encryption. When such a series of options is +## successfully negotiated, the event engine generates this event. +## +## c: The connection. +## +## .. bro:see:: authentication_accepted authentication_rejected authentication_skipped +## login_confused login_confused_text login_display login_failure login_input_line +## login_output_line login_prompt login_success login_terminal event activating_encryption%(c: connection%); +## TODO. +## +## .. bro:see:: login_confused login_confused_text login_display login_failure +## login_input_line login_output_line login_prompt login_success login_terminal +event inconsistent_option%(c: connection%); + +## TODO. +## +## .. bro:see:: login_confused login_confused_text login_display login_failure +## login_input_line login_output_line login_prompt login_success login_terminal +event bad_option%(c: connection%); + +## TODO. +## +## .. bro:see:: login_confused login_confused_text login_display login_failure +## login_input_line login_output_line login_prompt login_success login_terminal +event bad_option_termination%(c: connection%); + +## Generated for client side commands on an RSH connection. +## +## See `RFC 1258 `__ for more information about +## the Rlogin/Rsh protocol. +## +## c: The connection. +## +## client_user: The client-side user name as sent in the initial protocol +## handshake. +## +## client_user: The server-side user name as sent in the initial protocol +## handshake. +## +## line: The command line sent in the request. +## +## new_session: True if this is the first command of the Rsh session. +## +## .. bro:see:: rsh_reply login_confused login_confused_text login_display +## login_failure login_input_line login_output_line login_prompt login_success +## login_terminal +## +## .. note: For historical reasons, these events are separate from the ``login_`` +## events. Ideally, they would all be handled uniquely. +event rsh_request%(c: connection, client_user: string, server_user: string, line: string, new_session: bool%); + +## Generated for client side commands on an RSH connection. +## +## See `RFC 1258 `__ for more information about +## the Rlogin/Rsh protocol. +## +## c: The connection. +## +## client_user: The client-side user name as sent in the initial protocol +## handshake. +## +## client_user: The server-side user name as sent in the initial protocol +## handshake. +## +## line: The command line sent in the request. +## +## new_session: True if this is the first command of the Rsh session. +## +## .. bro:see:: rsh_request login_confused login_confused_text login_display +## login_failure login_input_line login_output_line login_prompt login_success +## login_terminal +## +## .. note: For historical reasons, these events are separate from the ``login_`` +## events. Ideally, they would all be handled uniquely. +event rsh_reply%(c: connection, client_user: string, server_user: string, line: string%); + +## Generated for client-side FTP commands. +## +## See `Wikipedia `__ for more +## information about the FTP protocol. +## +## c: The connection. +## +## command: The FTP command issued by the client (without any arguments). +## +## arg: The arguments going with the command. +## +## .. bro:see:: ftp_reply fmt_ftp_port parse_eftp_port +## parse_ftp_epsv parse_ftp_pasv parse_ftp_port event ftp_request%(c: connection, command: string, arg: string%) &group="ftp"; + +## Generated for server-side FTP replies. +## +## See `Wikipedia `__ for more +## information about the FTP protocol. +## +## c: The connection. +## +## code: The numerical response code the server responded with. +## +## msg: The textual message of the response. +## +## cont_resp: True if the reply line is tagged as being continued to the next line. +## If so, further events will be raised and a handler may want to reassemle the +## pieces before processing the response any further. +## +## .. bro:see:: ftp_request fmt_ftp_port parse_eftp_port +## parse_ftp_epsv parse_ftp_pasv parse_ftp_port event ftp_reply%(c: connection, code: count, msg: string, cont_resp: bool%) &group="ftp"; +## Generated for client-side SMTP commands. +## +## See `Wikipedia `__ +## for more information about the SMTP protocol. +## +## c: The connection. +## +## is_orig: True if the sender of the command is the originator of the TCP +## connection. Note that this is not redundant: the SMTP ``TURN`` command allows +## client and server to flip roles on established SMTP sessions, and hence a +## "request" might still come from the TCP-level responder. In practice, however, +## that will rarely happen as TURN is considered insecure and rarely used. +## +## command: The request's command, without any arguments. +## +## arg: The request command's arguments. +## +## .. bro:see:: mime_all_data mime_all_headers mime_begin_entity mime_content_hash +## mime_end_entity mime_entity_data mime_event mime_one_header mime_segment_data +## smtp_data smtp_reply +## +## .. note:: Bro does not support the newer ETRN extension yet. event smtp_request%(c: connection, is_orig: bool, command: string, arg: string%) &group="smtp"; + +## Generated for server-side SMTP commands. +## +## See `Wikipedia `__ +## for more information about the SMTP protocol. +## +## c: The connection. +## +## is_orig: True if the sender of the command is the originator of the TCP +## connection. Note that this is not redundant: the SMTP ``TURN`` command +## allows client and server to flip roles on established SMTP sessions, +## and hence a "reply" might still come from the TCP-level originator. In +## practice, however, that will rarely happen as TURN is considered insecure +## and rarely used. +## +## code: The reply's numerical code. +## +## msg: The reply's textual description. +## +## cont_resp: True if the reply line is tagged as being continued to the next line. +## If so, further events will be raised and a handler may want to reassemle the +## pieces before processing the response any further. +## +## .. bro:see:: mime_all_data mime_all_headers mime_begin_entity mime_content_hash +## mime_end_entity mime_entity_data mime_event mime_one_header mime_segment_data +## smtp_data smtp_request +## +## .. note:: Bro doesn't support the newer ETRN extension yet. event smtp_reply%(c: connection, is_orig: bool, code: count, cmd: string, msg: string, cont_resp: bool%) &group="smtp"; + +## Generated for DATA transmitted on SMTP sessions. This event is raised for +## subsequent chunks of raw data following the ``DATA`` SMTP command until the +## corresponding end marker ``.`` is seen. A handler may want to reassembly +## the pieces as they come in if stream-analysis is required. +## +## See `Wikipedia `__ +## for more information about the SMTP protocol. +## +## c: The connection. +## +## is_orig: True if the sender of the data is the originator of the TCP +## connection. +## +## data: The raw data. Note that the size of each chunk is undefined and +## depends on specifics of the underlying TCP connection. +## +## .. bro:see:: mime_all_data mime_all_headers mime_begin_entity mime_content_hash +## mime_end_entity mime_entity_data mime_event mime_one_header mime_segment_data +## smtp_reply smtp_request skip_smtp_data +## +## .. note:: This event received the unprocessed raw data. There is a separate +## set ``mime_*`` events that strip out the outer MIME-layer of emails and provide +## structured access to their content. event smtp_data%(c: connection, is_orig: bool, data: string%) &group="smtp"; + +## Generated for unexpected activity on SMTP sessions. The SMTP analyzer tracks the +## state of SMTP sessions and reports commands and other activity with this event +## that it sees even though it would not expect so at the current point of the +## communication. +## +## See `Wikipedia `__ +## for more information about the SMTP protocol. +## +## c: The connection. +## +## is_orig: True if the sender of the unexpected activity is the originator of the +## TCP connection. +## +## msg: A descriptive message of what was unexpected. +## +## detail: The actual SMTP line triggering the event. +## +## .. bro:see:: smtp_data smtp_request smtp_reply event smtp_unexpected%(c: connection, is_orig: bool, msg: string, detail: string%) &group="smtp"; +## Generated when starting to parse a email MIME entity. MIME is a +## protocol-independent data format for encoding text and files, along with +## corresponding meta-data, for transmission. Bro raises this event when it begin +## parsing a MIME entity extracted from an email protocol. +## +## Bro's MIME analyzer for emails currently supports SMTP and POP3. See `Wikipedia +## `__ for more information about the ARP +## protocol. +## +## c: The connection. +## +## .. bro:see:: mime_all_data mime_all_headers mime_content_hash mime_end_entity +## mime_entity_data mime_event mime_one_header mime_segment_data smtp_data +## http_begin_entity +## +## .. note:: Bro also extracts MIME entities from HTTP session. For those, however, +## it raises :bro:id:`http_begin_entity` instead. event mime_begin_entity%(c: connection%); -event mime_next_entity%(c: connection%); + +## Generated when finishing parsing an email MIME entity. MIME is a +## protocol-independent data format for encoding text and files, along with +## corresponding meta-data, for transmission. Bro raises this event when it +## finished parsing a MIME entity extracted from an email protocol. +## +## Bro's MIME analyzer for emails currently supports SMTP and POP3. See `Wikipedia +## `__ for more information about the ARP +## protocol. +## +## c: The connection. +## +## .. bro:see:: mime_all_data mime_all_headers mime_begin_entity mime_content_hash +## mime_entity_data mime_event mime_one_header mime_segment_data smtp_data +## http_end_entity +## +## .. note:: Bro also extracts MIME entities from HTTP session. For those, however, +## it raises :bro:id:`http_end_entity` instead. event mime_end_entity%(c: connection%); + +## Generated for individual MIME headers extracted from email MIME +## entities. MIME is a protocol-independent data format for encoding text and +## files, along with corresponding meta-data, for transmission. +## +## Bro's MIME analyzer for emails currently supports SMTP and POP3. See `Wikipedia +## `__ for more information about the ARP +## protocol. +## +## c: The connection. +## +## h: The parsed MIME header. +## +## .. bro:see:: mime_all_data mime_all_headers mime_begin_entity mime_content_hash +## mime_end_entity mime_entity_data mime_event mime_segment_data +## http_header http_all_headers +## +## .. note:: Bro also extracts MIME headers from HTTP sessions. For those, however, +## it raises :bro:id:`http_header` instead. event mime_one_header%(c: connection, h: mime_header_rec%); + +## Generated for MIME headers extracted from email MIME entities, passing all +## headers at once. MIME is a protocol-independent data format for encoding text +## and files, along with corresponding meta-data, for transmission. +## +## Bro's MIME analyzer for emails currently supports SMTP and POP3. See `Wikipedia +## `__ for more information about the ARP +## protocol. +## +## c: The connection. +## +## hlist: A *table* containing all headers extracted from the current entity. +## The table is indexed by the position of the header (1 for the first, 2 for the +## second, etc.). +## +## .. bro:see:: mime_all_data mime_begin_entity mime_content_hash mime_end_entity +## mime_entity_data mime_event mime_one_header mime_segment_data +## http_header http_all_headers +## +## .. note:: Bro also extracts MIME headers from HTTP sessions. For those, however, +## it raises :bro:id:`http_header` instead. event mime_all_headers%(c: connection, hlist: mime_header_list%); + +## Generated for chunks of decoded MIME data from email MIME entities. MIME +## is a protocol-independent data format for encoding text and files, along with +## corresponding meta-data, for transmission. As Bro parses the data of an entity, +## it raises a sequence of these events, each coming as soon as a new chunk of +## data is available. In contrast, there is also :bro:id:`mime_entity_data`, which +## passes all of an entities data at once in a single block. While the latter is +## more convinient to handle, ``mime_segment_data`` is more efficient as Bro does +## not need to buffer the data. Thus, if possible, this event should be prefered. +## +## Bro's MIME analyzer for emails currently supports SMTP and POP3. See `Wikipedia +## `__ for more information about the ARP +## protocol. +## +## c: The connection. +## +## length: The length of *data*. +## +## data: The raw data of one segment of the current entity. +## +## .. bro:see:: mime_all_data mime_all_headers mime_begin_entity mime_content_hash +## mime_end_entity mime_entity_data mime_event mime_one_header http_entity_data +## mime_segment_length mime_segment_overlap_length +## +## .. note:: Bro also extracts MIME data from HTTP sessions. For those, however, it +## raises :bro:id:`http_entity_data` (sic!) instead. event mime_segment_data%(c: connection, length: count, data: string%); + +## Generated for data decoded from an email MIME entity. This event delivers +## the complete content of a single MIME entity. In contrast, there is also +## :bro:id:`mime_segment_data`, which passes on a sequence of data chunks as +## they. come in. While ``mime_entity_data`` is more convinient to handle, +## ``mime_segment_data`` is more efficient as Bro does not need to buffer the data. +## Thus, if possible, the latter should be prefered. +## +## Bro's MIME analyzer for emails currently supports SMTP and POP3. See `Wikipedia +## `__ for more information about the ARP +## protocol. +## +## c: The connection. +## +## length: The length of *data*. +## +## data: The raw data of the complete entity. +## +## .. bro:see:: mime_all_data mime_all_headers mime_begin_entity mime_content_hash +## mime_end_entity mime_event mime_one_header mime_segment_data +## +## .. note:: While Bro also decodes MIME entities extracted from HTTP +## sessions, there's no corresponding event for that currently. event mime_entity_data%(c: connection, length: count, data: string%); + +## Generated for passing on all data decoded from an single email MIME +## message. If an email message has more than one MIME entity, this event +## combines all their data into a single value for analysis. Note that because +## of the potentially significant buffering necessary, using this event can be +## expensive. +## +## Bro's MIME analyzer for emails currently supports SMTP and POP3. See `Wikipedia +## `__ for more information about the ARP +## protocol. +## +## c: The connection. +## +## length: The length of *data*. +## +## data: The raw data of all MIME entities concatenated. +## +## .. bro:see:: mime_all_headers mime_begin_entity mime_content_hash mime_end_entity +## mime_entity_data mime_event mime_one_header mime_segment_data +## +## .. note:: While Bro also decodes MIME entities extracted from HTTP +## sessions, there's no corresponding event for that currently. event mime_all_data%(c: connection, length: count, data: string%); + +## Generated for errors found when decoding email MIME entities. +## +## Bro's MIME analyzer for emails currently supports SMTP and POP3. See `Wikipedia +## `__ for more information about the ARP +## protocol. +## +## event_type: A string describing the general category of the problem found (e.g., +## ``illegal format``). +## +## detail: Further more detailed description of the error. +## +## .. bro:see:: mime_all_data mime_all_headers mime_begin_entity mime_content_hash +## mime_end_entity mime_entity_data mime_one_header mime_segment_data http_event +## +## .. note:: Bro also extracts MIME headers from HTTP sessions. For those, however, +## it raises :bro:id:`http_event` instead. event mime_event%(c: connection, event_type: string, detail: string%); + +## Generated for decoded MIME entities extracted from email meessage, passing on +## their MD5 checksums. Bro computes the MD5 over the complete decoded data of +## each MIME entity. +## +## Bro's MIME analyzer for emails currently supports SMTP and POP3. See `Wikipedia +## `__ for more information about the ARP +## protocol. +## +## c: The connection. +## +## content_len: The length of entity being hashed. +## +## hash_value: The MD5 hash. +## +## .. bro:see:: mime_all_data mime_all_headers mime_begin_entity mime_end_entity +## mime_entity_data mime_event mime_one_header mime_segment_data +## +## .. note:: While Bro also decodes MIME entities extracted from HTTP +## sessions, there's no corresponding event for that currently. event mime_content_hash%(c: connection, content_len: count, hash_value: string%); -# Generated for each RPC request / reply *pair* (if there is no reply, the event -# will be generated on timeout). +## Generated for RPC request/reply *pairs*. The RPC analyzer associates request +## and reply by their transactions identifiers and raise this event once both +## have been seen. If there's not reply, the will still be generated eventually +## on timeout. In that case, *status* will be set to :bro:enum:`RPC_TIMEOUT`. +## +## See `Wikipedia `__ for more information +## about the ONC RPC protocol. +## c: The connection. +## +## xid: The transaction identifier allowing to match requests with replies. +## +## prog: The remote program to call. +## +## ver: The version of the remote program to call. +## +## proc: The procedure of the remote program to call. +## +## status: The status of the reply, which should be one of the index values of +## :bro:id:`RPC_status`. +## +## start_time: Then time when the *call* was seen. +## +## call_len: The size of the *call_body* PDU. +## +## reply_len: The size of the *reply_body* PDU. +## +## .. bro:see:: rpc_call rpc_reply dce_rpc_bind dce_rpc_message dce_rpc_request +## dce_rpc_response rpc_timeout event rpc_dialogue%(c: connection, prog: count, ver: count, proc: count, status: rpc_status, start_time: time, call_len: count, reply_len: count%); -# Generated for each (correctly formed) RPC_CALL message received. + +## Generated for RPC *call* messages. +## +## See `Wikipedia `__ for more information +## about the ONC RPC protocol. +## +## c: The connection. +## +## xid: The transaction identifier allowing to match requests with replies. +## +## prog: The remote program to call. +## +## ver: The version of the remote program to call. +## +## proc: The procedure of the remote program to call. +## +## call_len: The size of the *call_body* PDU. +## +## .. bro:see:: rpc_dialogue rpc_reply dce_rpc_bind dce_rpc_message dce_rpc_request +## dce_rpc_response rpc_timeout event rpc_call%(c: connection, xid: count, prog: count, ver: count, proc: count, call_len: count%); -# Generated for each (correctly formed) RPC_REPLY message received. + +## Generated for RPC *reply* messages. +## +## See `Wikipedia `__ for more information +## about the ONC RPC protocol. +## +## c: The connection. +## +## xid: The transaction identifier allowing to match requests with replies. +## +## status: The status of the reply, which should be one of the index values of +## :bro:id:`RPC_status`. +## +## reply_len: The size of the *reply_body* PDU. +## +## .. bro:see:: rpc_call rpc_dialogue dce_rpc_bind dce_rpc_message dce_rpc_request +## dce_rpc_response rpc_timeout event rpc_reply%(c: connection, xid: count, status: rpc_status, reply_len: count%); +## Generated for Portmapper requests of type *null*. +## +## Portmapper is a service running on top of RPC. See `Wikipedia +## `__ for more information about the service. +## +## r: The RPC connection. +## +## .. bro:see:: epm_map_response pm_attempt_callit pm_attempt_dump pm_attempt_getport +## pm_attempt_null pm_attempt_set pm_attempt_unset pm_bad_port pm_request_callit +## pm_request_dump pm_request_getport pm_request_set pm_request_unset rpc_call +## rpc_dialogue rpc_reply event pm_request_null%(r: connection%); + +## Generated for Portmapper request/reply dialogues of type *set*. +## +## Portmapper is a service running on top of RPC. See `Wikipedia +## `__ for more information about the service. +## +## r: The RPC connection. +## +## m: The argument to the request. +## +## success: True if the request was successful, according to the corresponding +## reply. If no reply was seen, this will be false once the request times out. +## +## .. bro:see:: epm_map_response pm_attempt_callit pm_attempt_dump pm_attempt_getport +## pm_attempt_null pm_attempt_set pm_attempt_unset pm_bad_port pm_request_callit +## pm_request_dump pm_request_getport pm_request_null pm_request_unset rpc_call +## rpc_dialogue rpc_reply event pm_request_set%(r: connection, m: pm_mapping, success: bool%); + +## Generated for Portmapper request/reply dialogues of type *unset*. +## +## Portmapper is a service running on top of RPC. See `Wikipedia +## `__ for more information about the service. +## +## r: The RPC connection. +## +## m: The argument to the request. +## +## success: True if the request was successful, according to the corresponding +## reply. If no reply was seen, this will be false once the request times out. +## +## .. bro:see:: epm_map_response pm_attempt_callit pm_attempt_dump pm_attempt_getport +## pm_attempt_null pm_attempt_set pm_attempt_unset pm_bad_port pm_request_callit +## pm_request_dump pm_request_getport pm_request_null pm_request_set rpc_call +## rpc_dialogue rpc_reply event pm_request_unset%(r: connection, m: pm_mapping, success: bool%); + +## Generated for Portmapper request/reply dialogues of type *getport*. +## +## Portmapper is a service running on top of RPC. See `Wikipedia +## `__ for more information about the service. +## +## r: The RPC connection. +## +## pr: The argument to the request. +## +## p: The port returned by the server. +## +## success: True if the request was successful, according to the corresponding +## reply. If no reply was seen, this will be false once the request times out. +## +## .. bro:see:: epm_map_response pm_attempt_callit pm_attempt_dump pm_attempt_getport +## pm_attempt_null pm_attempt_set pm_attempt_unset pm_bad_port pm_request_callit +## pm_request_dump pm_request_null pm_request_set pm_request_unset rpc_call +## rpc_dialogue rpc_reply event pm_request_getport%(r: connection, pr: pm_port_request, p: port%); + +## Generated for Portmapper request/reply dialogues of type *dump*. +## +## Portmapper is a service running on top of RPC. See `Wikipedia +## `__ for more information about the service. +## +## r: The RPC connection. +## +## m: The mappings returned by the server. +## +## .. bro:see:: epm_map_response pm_attempt_callit pm_attempt_dump pm_attempt_getport +## pm_attempt_null pm_attempt_set pm_attempt_unset pm_bad_port pm_request_callit +## pm_request_getport pm_request_null pm_request_set pm_request_unset rpc_call +## rpc_dialogue rpc_reply event pm_request_dump%(r: connection, m: pm_mappings%); + +## Generated for Portmapper request/reply dialogues of type *callit*. +## +## Portmapper is a service running on top of RPC. See `Wikipedia +## `__ for more information about the +## service. +## +## r: The RPC connection. +## +## m: The argument to the request. +## +## p: The port value returned by the call. +## +## .. bro:see:: epm_map_response pm_attempt_callit pm_attempt_dump pm_attempt_getport +## pm_attempt_null pm_attempt_set pm_attempt_unset pm_bad_port pm_request_dump +## pm_request_getport pm_request_null pm_request_set pm_request_unset rpc_call +## rpc_dialogue rpc_reply event pm_request_callit%(r: connection, call: pm_callit_request, p: port%); + +## Generated for failed Portmapper requests of type *null*. +## +## Portmapper is a service running on top of RPC. See `Wikipedia +## `__ for more information about the +## service. +## +## r: The RPC connection. +## +## status: The status of the reply, which should be one of the index values of +## :bro:id:`RPC_status`. +## +## .. bro:see:: epm_map_response pm_attempt_callit pm_attempt_dump pm_attempt_getport +## pm_attempt_set pm_attempt_unset pm_bad_port pm_request_callit pm_request_dump +## pm_request_getport pm_request_null pm_request_set pm_request_unset rpc_call +## rpc_dialogue rpc_reply event pm_attempt_null%(r: connection, status: rpc_status%); + +## Generated for failed Portmapper requests of type *set*. +## +## Portmapper is a service running on top of RPC. See `Wikipedia +## `__ for more information about the +## service. +## +## r: The RPC connection. +## +## status: The status of the reply, which should be one of the index values of +## :bro:id:`RPC_status`. +## +## m: The argument to the original request. +## +## .. bro:see:: epm_map_response pm_attempt_callit pm_attempt_dump pm_attempt_getport +## pm_attempt_null pm_attempt_unset pm_bad_port pm_request_callit pm_request_dump +## pm_request_getport pm_request_null pm_request_set pm_request_unset rpc_call +## rpc_dialogue rpc_reply event pm_attempt_set%(r: connection, status: rpc_status, m: pm_mapping%); + +## Generated for failed Portmapper requests of type *unset*. +## +## Portmapper is a service running on top of RPC. See `Wikipedia +## `__ for more information about the +## service. +## +## r: The RPC connection. +## +## status: The status of the reply, which should be one of the index values of +## :bro:id:`RPC_status`. +## +## m: The argument to the original request. +## +## .. bro:see:: epm_map_response pm_attempt_callit pm_attempt_dump pm_attempt_getport +## pm_attempt_null pm_attempt_set pm_bad_port pm_request_callit pm_request_dump +## pm_request_getport pm_request_null pm_request_set pm_request_unset rpc_call +## rpc_dialogue rpc_reply event pm_attempt_unset%(r: connection, status: rpc_status, m: pm_mapping%); + +## Generated for failed Portmapper requests of type *getport*. +## +## Portmapper is a service running on top of RPC. See `Wikipedia +## `__ for more information about the +## service. +## +## r: The RPC connection. +## +## status: The status of the reply, which should be one of the index values of +## :bro:id:`RPC_status`. +## +## pr: The argument to the original request. +## +## .. bro:see:: epm_map_response pm_attempt_callit pm_attempt_dump pm_attempt_null +## pm_attempt_set pm_attempt_unset pm_bad_port pm_request_callit pm_request_dump +## pm_request_getport pm_request_null pm_request_set pm_request_unset rpc_call +## rpc_dialogue rpc_reply event pm_attempt_getport%(r: connection, status: rpc_status, pr: pm_port_request%); + +## Generated for failed Portmapper requests of type *dump*. +## +## Portmapper is a service running on top of RPC. See `Wikipedia +## `__ for more information about the +## service. +## +## r: The RPC connection. +## +## status: The status of the reply, which should be one of the index values of +## :bro:id:`RPC_status`. +## +## .. bro:see:: epm_map_response pm_attempt_callit pm_attempt_getport pm_attempt_null +## pm_attempt_set pm_attempt_unset pm_bad_port pm_request_callit pm_request_dump +## pm_request_getport pm_request_null pm_request_set pm_request_unset rpc_call +## rpc_dialogue rpc_reply event pm_attempt_dump%(r: connection, status: rpc_status%); + +## Generated for failed Portmapper requests of type *callit*. +## +## Portmapper is a service running on top of RPC. See `Wikipedia +## `__ for more information about the +## service. +## +## r: The RPC connection. +## +## status: The status of the reply, which should be one of the index values of +## :bro:id:`RPC_status`. +## +## call: The argument to the original request. +## +## .. bro:see:: epm_map_response pm_attempt_dump pm_attempt_getport pm_attempt_null +## pm_attempt_set pm_attempt_unset pm_bad_port pm_request_callit pm_request_dump +## pm_request_getport pm_request_null pm_request_set pm_request_unset rpc_call +## rpc_dialogue rpc_reply event pm_attempt_callit%(r: connection, status: rpc_status, call: pm_callit_request%); + +## Generated for Portmapper requests or replies that include an invalid port +## number. Since ports are represented by unsigned 4-byte integers, they can stray +## outside the allowed range of 0--65535 by being >= 65536. If so, this event is +## generated. +## +## Portmapper is a service running on top of RPC. See `Wikipedia +## `__ for more information about the +## service. +## +## r: The RPC connection. +## +## bad_p: The invalid port value. +## +## .. bro:see:: epm_map_response pm_attempt_callit pm_attempt_dump pm_attempt_getport +## pm_attempt_null pm_attempt_set pm_attempt_unset pm_request_callit +## pm_request_dump pm_request_getport pm_request_null pm_request_set +## pm_request_unset rpc_call rpc_dialogue rpc_reply event pm_bad_port%(r: connection, bad_p: count%); -# Events for the NFS analyzer. An event is generated if we have received a -# Call (request) / Response pair (or in case of a time out). info$rpc_stat and -# info$nfs_stat show whether the request was successful. The request record is -# always filled out, however, the reply record(s) might not be set or might only -# be partially set. See the comments for the record types in bro.init to see which -# reply fields are set when. +## Generated for NFSv3 request/reply dialogues of type *null*. The event is +## generated once we have either seen both the request and its corresponding reply, +## or an unanswered request has timed out. +## +## NFS is a service running on top of RPC. See `Wikipedia +## `__ for more +## information about the service. +## +## c: The RPC connection. +## +## info: Reports the status of the dialogue, along with some meta information. +## +## .. bro:see:: nfs_proc_create nfs_proc_getattr nfs_proc_lookup nfs_proc_mkdir +## nfs_proc_not_implemented nfs_proc_read nfs_proc_readdir nfs_proc_readlink +## nfs_proc_remove nfs_proc_rmdir nfs_proc_write nfs_reply_status rpc_call +## rpc_dialogue rpc_reply event nfs_proc_null%(c: connection, info: NFS3::info_t%); -event nfs_proc_not_implemented%(c: connection, info: NFS3::info_t, proc: NFS3::proc_t%); +## Generated for NFSv3 request/reply dialogues of type *getattr*. The event is +## generated once we have either seen both the request and its corresponding reply, +## or an unanswered request has timed out. +## +## NFS is a service running on top of RPC. See `Wikipedia +## `__ for more +## information about the service. +## +## c: The RPC connection. +## +## info: Reports the status of the dialogue, along with some meta information. +## +## attr: The attributes returned in the reply. The values may not be valid if the +## request was unsuccessful. +## +## .. bro:see:: nfs_proc_create nfs_proc_lookup nfs_proc_mkdir +## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readdir +## nfs_proc_readlink nfs_proc_remove nfs_proc_rmdir nfs_proc_write nfs_reply_status +## rpc_call rpc_dialogue rpc_reply NFS3::mode2string event nfs_proc_getattr%(c: connection, info: NFS3::info_t, fh: string, attrs: NFS3::fattr_t%); + +## Generated for NFSv3 request/reply dialogues of type *lookup*. The event is +## generated once we have either seen both the request and its corresponding reply, +## or an unanswered request has timed out. +## +## NFS is a service running on top of RPC. See `Wikipedia +## `__ for more +## information about the service. +## +## c: The RPC connection. +## +## info: Reports the status of the dialogue, along with some meta information. +## +## req: The arguments passed in the request. +## +## rep: The response returned in the reply. The values may not be valid if the +## request was unsuccessful. +## +## .. bro:see:: nfs_proc_create nfs_proc_getattr nfs_proc_mkdir +## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readdir +## nfs_proc_readlink nfs_proc_remove nfs_proc_rmdir nfs_proc_write nfs_reply_status +## rpc_call rpc_dialogue rpc_reply event nfs_proc_lookup%(c: connection, info: NFS3::info_t, req: NFS3::diropargs_t, rep: NFS3::lookup_reply_t%); + +## Generated for NFSv3 request/reply dialogues of type *read*. The event is +## generated once we have either seen both the request and its corresponding reply, +## or an unanswered request has timed out. +## +## NFS is a service running on top of RPC. See `Wikipedia +## `__ for more +## information about the service. +## +## c: The RPC connection. +## +## info: Reports the status of the dialogue, along with some meta information. +## +## req: The arguments passed in the request. +## +## rep: The response returned in the reply. The values may not be valid if the +## request was unsuccessful. +## +## .. bro:see:: nfs_proc_create nfs_proc_getattr nfs_proc_lookup nfs_proc_mkdir +## nfs_proc_not_implemented nfs_proc_null nfs_proc_remove nfs_proc_rmdir +## nfs_proc_write nfs_reply_status rpc_call rpc_dialogue rpc_reply +## NFS3::return_data NFS3::return_data_first_only NFS3::return_data_max event nfs_proc_read%(c: connection, info: NFS3::info_t, req: NFS3::readargs_t, rep: NFS3::read_reply_t%); + +## Generated for NFSv3 request/reply dialogues of type *readlink*. The event is +## generated once we have either seen both the request and its corresponding reply, +## or an unanswered request has timed out. +## +## NFS is a service running on top of RPC. See `Wikipedia +## `__ for more +## information about the service. +## +## c: The RPC connection. +## +## info: Reports the status of the dialogue, along with some meta information. +## +## fh: The file handle passed in the request. +## +## rep: The response returned in the reply. The values may not be valid if the +## request was unsuccessful. +## +## .. bro:see:: nfs_proc_create nfs_proc_getattr nfs_proc_lookup nfs_proc_mkdir +## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readdir +## nfs_proc_remove nfs_proc_rmdir nfs_proc_write nfs_reply_status rpc_call +## rpc_dialogue rpc_reply event nfs_proc_readlink%(c: connection, info: NFS3::info_t, fh: string, rep: NFS3::readlink_reply_t%); + +## Generated for NFSv3 request/reply dialogues of type *write*. The event is +## generated once we have either seen both the request and its corresponding reply, +## or an unanswered request has timed out. +## +## NFS is a service running on top of RPC. See `Wikipedia +## `__ for more +## information about the service. +## +## c: The RPC connection. +## +## info: Reports the status of the dialogue, along with some meta information. +## +## fh: The file handle passed in the request. +## +## rep: The response returned in the reply. The values may not be valid if the +## request was unsuccessful. +## +## .. bro:see:: nfs_proc_create nfs_proc_getattr nfs_proc_lookup nfs_proc_mkdir +## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readdir +## nfs_proc_readlink nfs_proc_remove nfs_proc_rmdir nfs_reply_status rpc_call +## rpc_dialogue rpc_reply NFS3::return_data NFS3::return_data_first_only +## NFS3::return_data_max event nfs_proc_write%(c: connection, info: NFS3::info_t, req: NFS3::writeargs_t, rep: NFS3::write_reply_t%); + +## Generated for NFSv3 request/reply dialogues of type *create*. The event is +## generated once we have either seen both the request and its corresponding reply, +## or an unanswered request has timed out. +## +## NFS is a service running on top of RPC. See `Wikipedia +## `__ for more +## information about the service. +## +## c: The RPC connection. +## +## info: Reports the status of the dialogue, along with some meta information. +## +## fh: The file handle passed in the request. +## +## rep: The response returned in the reply. The values may not be valid if the +## request was unsuccessful. +## +## .. bro:see:: nfs_proc_getattr nfs_proc_lookup nfs_proc_mkdir +## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readdir +## nfs_proc_readlink nfs_proc_remove nfs_proc_rmdir nfs_proc_write nfs_reply_status +## rpc_call rpc_dialogue rpc_reply event nfs_proc_create%(c: connection, info: NFS3::info_t, req: NFS3::diropargs_t, rep: NFS3::newobj_reply_t%); + +## Generated for NFSv3 request/reply dialogues of type *mkdir*. The event is +## generated once we have either seen both the request and its corresponding reply, +## or an unanswered request has timed out. +## +## NFS is a service running on top of RPC. See `Wikipedia +## `__ for more +## information about the service. +## +## c: The RPC connection. +## +## info: Reports the status of the dialogue, along with some meta information. +## +## fh: The file handle passed in the request. +## +## rep: The response returned in the reply. The values may not be valid if the +## request was unsuccessful. +## +## .. bro:see:: nfs_proc_create nfs_proc_getattr nfs_proc_lookup +## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readdir +## nfs_proc_readlink nfs_proc_remove nfs_proc_rmdir nfs_proc_write nfs_reply_status +## rpc_call rpc_dialogue rpc_reply event nfs_proc_mkdir%(c: connection, info: NFS3::info_t, req: NFS3::diropargs_t, rep: NFS3::newobj_reply_t%); + +## Generated for NFSv3 request/reply dialogues of type *remove*. The event is +## generated once we have either seen both the request and its corresponding reply, +## or an unanswered request has timed out. +## +## NFS is a service running on top of RPC. See `Wikipedia +## `__ for more +## information about the service. +## +## c: The RPC connection. +## +## info: Reports the status of the dialogue, along with some meta information. +## +## fh: The file handle passed in the request. +## +## rep: The response returned in the reply. The values may not be valid if the +## request was unsuccessful. +## +## .. bro:see:: nfs_proc_create nfs_proc_getattr nfs_proc_lookup nfs_proc_mkdir +## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readdir +## nfs_proc_readlink nfs_proc_rmdir nfs_proc_write nfs_reply_status rpc_call +## rpc_dialogue rpc_reply event nfs_proc_remove%(c: connection, info: NFS3::info_t, req: NFS3::diropargs_t, rep: NFS3::delobj_reply_t%); + +## Generated for NFSv3 request/reply dialogues of type *rmdir*. The event is +## generated once we have either seen both the request and its corresponding reply, +## or an unanswered request has timed out. +## +## NFS is a service running on top of RPC. See `Wikipedia +## `__ for more +## information about the service. +## +## c: The RPC connection. +## +## info: Reports the status of the dialogue, along with some meta information. +## +## fh: The file handle passed in the request. +## +## rep: The response returned in the reply. The values may not be valid if the +## request was unsuccessful. +## +## .. bro:see:: nfs_proc_create nfs_proc_getattr nfs_proc_lookup nfs_proc_mkdir +## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readdir +## nfs_proc_readlink nfs_proc_remove nfs_proc_write nfs_reply_status rpc_call +## rpc_dialogue rpc_reply event nfs_proc_rmdir%(c: connection, info: NFS3::info_t, req: NFS3::diropargs_t, rep: NFS3::delobj_reply_t%); + +## Generated for NFSv3 request/reply dialogues of type *readdir*. The event is +## generated once we have either seen both the request and its corresponding reply, +## or an unanswered request has timed out. +## +## NFS is a service running on top of RPC. See `Wikipedia +## `__ for more +## information about the service. +## +## c: The RPC connection. +## +## info: Reports the status of the dialogue, along with some meta information. +## +## fh: The file handle passed in the request. +## +## rep: The response returned in the reply. The values may not be valid if the +## request was unsuccessful. +## +## .. bro:see:: nfs_proc_create nfs_proc_getattr nfs_proc_lookup nfs_proc_mkdir +## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readlink +## nfs_proc_remove nfs_proc_rmdir nfs_proc_write nfs_reply_status rpc_call +## rpc_dialogue rpc_reply event nfs_proc_readdir%(c: connection, info: NFS3::info_t, req: NFS3::readdirargs_t, rep: NFS3::readdir_reply_t%); -# Generated for each NFS reply message we receive, giving just gives the status. +## Generated for NFS3 request/reply dialogues of a type that Bro's NFS3 analyzer +## does not implement. +## +## NFS is a service running on top of RPC. See `Wikipedia +## `__ for more +## information about the service. +## +## c: The RPC connection. +## +## info: Reports the status of the dialogue, along with some meta information. +## +## proc: The procedure called that Bro does not implement. +## +## .. bro:see:: nfs_proc_create nfs_proc_getattr nfs_proc_lookup nfs_proc_mkdir +## nfs_proc_null nfs_proc_read nfs_proc_readdir nfs_proc_readlink nfs_proc_remove +## nfs_proc_rmdir nfs_proc_write nfs_reply_status rpc_call rpc_dialogue rpc_reply +event nfs_proc_not_implemented%(c: connection, info: NFS3::info_t, proc: NFS3::proc_t%); + +## Generated for each NFS3 reply message received, reporting just the +## status included. +## +## info: Reports the status included in the reply. +## +## .. bro:see:: nfs_proc_create nfs_proc_getattr nfs_proc_lookup nfs_proc_mkdir +## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readdir +## nfs_proc_readlink nfs_proc_remove nfs_proc_rmdir nfs_proc_write rpc_call +## rpc_dialogue rpc_reply event nfs_reply_status%(n: connection, info: NFS3::info_t%); +## Generated for all NTP messages. Different from many other of Bro's events, this +## one is generated for both client-side and server-side messages. +## +## See `Wikipedia `__ for more +## information about the NTP protocol. +## +## u: The connection record describing the corresponding UDP flow. +## +## msg: The parsed NTP message. +## +## excess: The raw bytes of any optional parts of the NTP packet. Bro does not +## further parse any optional fields. +## +## .. bro:see:: ntp_session_timeout event ntp_message%(u: connection, msg: ntp_msg, excess: string%); +## Generated for all NetBIOS SSN and DGM messages. Bro's NetBIOS analyzer processes +## the NetBIOS session service running on TCP port 139, and (despite its name!) the +## NetBIOS datagram service on UDP port 138. +## +## See `Wikipedia `__ for more information +## about NetBIOS. `RFC 1002 `__ describes +## the packet format for NetBIOS over TCP/IP, which Bro parses. +## +## c: The connection, which may be a TCP or UDP, depending on the type of the +## NetBIOS session. +## +## is_orig: True if the message was sent by the originator of the connection. +## +## msg_type: The general type of message, as defined in Section 4.3.1 of `RFC 1002 +## `__. +## +## data_len: The length of the message's payload. +## +## .. bro:see:: netbios_session_accepted netbios_session_keepalive +## netbios_session_raw_message netbios_session_rejected netbios_session_request +## netbios_session_ret_arg_resp decode_netbios_name decode_netbios_name_type +## +## .. note:: These days, NetBIOS is primarily used as a transport mechanism for +## `SMB/CIFS `__. Bro's SMB +## anlyzer parses both SMB-over-NetBIOS and SMB-over-TCP on port 445. event netbios_session_message%(c: connection, is_orig: bool, msg_type: count, data_len: count%); + +## Generated for NetBIOS messages of type *session request*. Bro's NetBIOS analyzer +## processes the NetBIOS session service running on TCP port 139, and (despite its +## name!) the NetBIOS datagram service on UDP port 138. +## +## See `Wikipedia `__ for more information +## about NetBIOS. `RFC 1002 `__ describes +## the packet format for NetBIOS over TCP/IP, which Bro parses. +## +## c: The connection, which may be a TCP or UDP, depending on the type of the +## NetBIOS session. +## +## msg: The raw payload of the message sent, excluding the common NetBIOS +## header. +## +## .. bro:see:: netbios_session_accepted netbios_session_keepalive +## netbios_session_message netbios_session_raw_message netbios_session_rejected +## netbios_session_ret_arg_resp decode_netbios_name decode_netbios_name_type +## +## .. note:: These days, NetBIOS is primarily used as a transport mechanism for +## `SMB/CIFS `__. Bro's SMB +## anlyzer parses both SMB-over-NetBIOS and SMB-over-TCP on port 445. event netbios_session_request%(c: connection, msg: string%); + +## Generated for NetBIOS messages of type *positive session response*. Bro's +## NetBIOS analyzer processes the NetBIOS session service running on TCP port 139, +## and (despite its name!) the NetBIOS datagram service on UDP port 138. +## +## See `Wikipedia `__ for more information +## about NetBIOS. `RFC 1002 `__ describes +## the packet format for NetBIOS over TCP/IP, which Bro parses. +## +## c: The connection, which may be a TCP or UDP, depending on the type of the +## NetBIOS session. +## +## msg: The raw payload of the message sent, excluding the common NetBIOS +## header. +## +## .. bro:see:: netbios_session_keepalive netbios_session_message +## netbios_session_raw_message netbios_session_rejected netbios_session_request +## netbios_session_ret_arg_resp decode_netbios_name decode_netbios_name_type +## +## .. note:: These days, NetBIOS is primarily used as a transport mechanism for +## `SMB/CIFS `__. Bro's SMB +## anlyzer parses both SMB-over-NetBIOS and SMB-over-TCP on port 445. event netbios_session_accepted%(c: connection, msg: string%); + +## Generated for NetBIOS messages of type *negative session response*. Bro's +## NetBIOS analyzer processes the NetBIOS session service running on TCP port 139, +## and (despite its name!) the NetBIOS datagram service on UDP port 138. +## +## See `Wikipedia `__ for more information +## about NetBIOS. `RFC 1002 `__ describes +## the packet format for NetBIOS over TCP/IP, which Bro parses. +## +## c: The connection, which may be a TCP or UDP, depending on the type of the +## NetBIOS session. +## +## msg: The raw payload of the message sent, excluding the common NetBIOS +## header. +## +## .. bro:see:: netbios_session_accepted netbios_session_keepalive +## netbios_session_message netbios_session_raw_message netbios_session_request +## netbios_session_ret_arg_resp decode_netbios_name decode_netbios_name_type +## +## .. note:: These days, NetBIOS is primarily used as a transport mechanism for +## `SMB/CIFS `__. Bro's SMB +## anlyzer parses both SMB-over-NetBIOS and SMB-over-TCP on port 445. event netbios_session_rejected%(c: connection, msg: string%); + +## Generated for NetBIOS message of type *session message* that are not carrying +## SMB payload. +## +## NetBIOS analyzer processes the NetBIOS session service running on TCP port 139, +## and (despite its name!) the NetBIOS datagram service on UDP port 138. +## +## See `Wikipedia `__ for more information +## about NetBIOS. `RFC 1002 `__ describes +## the packet format for NetBIOS over TCP/IP, which Bro parses. +## +## c: The connection, which may be a TCP or UDP, depending on the type of the +## NetBIOS session. +## +## is_orig: True if the message was sent by the originator of the connection. +## +## msg: The raw payload of the message sent, excluding the common NetBIOS +## header (i.e., the ``user_data``). +## +## .. bro:see:: netbios_session_accepted netbios_session_keepalive +## netbios_session_message netbios_session_rejected netbios_session_request +## netbios_session_ret_arg_resp decode_netbios_name decode_netbios_name_type +## +## .. note:: These days, NetBIOS is primarily used as a transport mechanism for +## `SMB/CIFS `__. Bro's SMB +## anlyzer parses both SMB-over-NetBIOS and SMB-over-TCP on port 445. +## +## .. todo:: This is an oddly named event. In fact, it's probably an odd event to +## have to begin with. event netbios_session_raw_message%(c: connection, is_orig: bool, msg: string%); + +## Generated for NetBIOS messages of type *retarget response*. Bro's NetBIOS +## analyzer processes the NetBIOS session service running on TCP port 139, and +## (despite its name!) the NetBIOS datagram service on UDP port 138. +## +## See `Wikipedia `__ for more information +## about NetBIOS. `RFC 1002 `__ describes +## the packet format for NetBIOS over TCP/IP, which Bro parses. +## +## c: The connection, which may be a TCP or UDP, depending on the type of the +## NetBIOS session. +## +## msg: The raw payload of the message sent, excluding the common NetBIOS header. +## +## .. bro:see:: netbios_session_accepted netbios_session_keepalive +## netbios_session_message netbios_session_raw_message netbios_session_rejected +## netbios_session_request decode_netbios_name decode_netbios_name_type +## +## .. note:: These days, NetBIOS is primarily used as a transport mechanism for +## `SMB/CIFS `__. Bro's SMB +## anlyzer parses both SMB-over-NetBIOS and SMB-over-TCP on port 445. +## +## .. todo: This is an oddly named event. event netbios_session_ret_arg_resp%(c: connection, msg: string%); + +## Generated for NetBIOS messages of type *keep-alive*. Bro's NetBIOS analyzer +## processes the NetBIOS session service running on TCP port 139, and (despite its +## name!) the NetBIOS datagram service on UDP port 138. +## +## See `Wikipedia `__ for more information +## about NetBIOS. `RFC 1002 `__ describes +## the packet format for NetBIOS over TCP/IP, which Bro parses. +## +## c: The connection, which may be a TCP or UDP, depending on the type of the +## NetBIOS session. +## +## msg: The raw payload of the message sent, excluding the common NetBIOS header. +## +## .. bro:see:: netbios_session_accepted netbios_session_message +## netbios_session_raw_message netbios_session_rejected netbios_session_request +## netbios_session_ret_arg_resp decode_netbios_name decode_netbios_name_type +## +## .. note:: These days, NetBIOS is primarily used as a transport mechanism for +## `SMB/CIFS `__. Bro's SMB +## anlyzer parses both SMB-over-NetBIOS and SMB-over-TCP on port 445. event netbios_session_keepalive%(c: connection, msg: string%); +## Generated for all SMB/CIFS messages. +## +## See `Wikipedia `__ for more +## information about the SMB/CIFS protocol. Bro's SMB/CIFS analyzer parses both +## SMB-over-NetBIOS on ports 138/139 and SMB-over-TCP on port 445. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## is_orig: True if the message was sent by the originator of the underlying +## transport-level connection. +## +## cmd: A string mmenonic of the SMB command code. +## +## body_length: The length of the SMB message body, i.e. the data starting after +## the SMB header. +## +## body: The raw SMB message body, i.e., the data starting after the SMB header. +## +## .. bro:see:: smb_com_close smb_com_generic_andx smb_com_logoff_andx +## smb_com_negotiate smb_com_negotiate_response smb_com_nt_create_andx +## smb_com_read_andx smb_com_setup_andx smb_com_trans_mailslot +## smb_com_trans_pipe smb_com_trans_rap smb_com_transaction smb_com_transaction2 +## smb_com_tree_connect_andx smb_com_tree_disconnect smb_com_write_andx smb_error +## smb_get_dfs_referral event smb_message%(c: connection, hdr: smb_hdr, is_orig: bool, cmd: string, body_length: count, body: string%); + +## Generated for SMB/CIFS messages of type *tree connect andx*. +## +## See `Wikipedia `__ for more +## information about the SMB/CIFS protocol. Bro's SMB/CIFS analyzer parses both +## SMB-over-NetBIOS on ports 138/139 and SMB-over-TCP on port 445. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## path: The ``path`` attribute specified in the message. +## +## service: The ``service`` attribute specified in the message. +## +## .. bro:see:: smb_com_close smb_com_generic_andx smb_com_logoff_andx +## smb_com_negotiate smb_com_negotiate_response smb_com_nt_create_andx +## smb_com_read_andx smb_com_setup_andx smb_com_trans_mailslot +## smb_com_trans_pipe smb_com_trans_rap smb_com_transaction smb_com_transaction2 +## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral +## smb_message event smb_com_tree_connect_andx%(c: connection, hdr: smb_hdr, path: string, service: string%); + +## Generated for SMB/CIFS messages of type *tree disconnect*. +## +## See `Wikipedia `__ for more +## information about the SMB/CIFS protocol. Bro's SMB/CIFS analyzer parses both +## SMB-over-NetBIOS on ports 138/139 and SMB-over-TCP on port 445. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## path: The ``path`` attribute specified in the message. +## +## .. bro:see:: smb_com_close smb_com_generic_andx smb_com_logoff_andx +## smb_com_negotiate smb_com_negotiate_response smb_com_nt_create_andx +## smb_com_read_andx smb_com_setup_andx smb_com_trans_mailslot +## smb_com_trans_pipe smb_com_trans_rap smb_com_transaction smb_com_transaction2 +## smb_com_tree_connect_andx smb_com_write_andx smb_error smb_get_dfs_referral +## smb_message event smb_com_tree_disconnect%(c: connection, hdr: smb_hdr%); + +## Generated for SMB/CIFS messages of type *nt create andx*. +## +## See `Wikipedia `__ for more +## information about the SMB/CIFS protocol. Bro's SMB/CIFS analyzer parses both +## SMB-over-NetBIOS on ports 138/139 and SMB-over-TCP on port 445. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## name: The ``name`` attribute specified in the message. +## +## .. bro:see:: smb_com_close smb_com_generic_andx smb_com_logoff_andx +## smb_com_negotiate smb_com_negotiate_response smb_com_read_andx +## smb_com_setup_andx smb_com_trans_mailslot smb_com_trans_pipe smb_com_trans_rap +## smb_com_transaction smb_com_transaction2 smb_com_tree_connect_andx +## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral +## smb_message event smb_com_nt_create_andx%(c: connection, hdr: smb_hdr, name: string%); + +## Generated for SMB/CIFS messages of type *nt transaction*. +## +## See `Wikipedia `__ for more +## information about the SMB/CIFS protocol. Bro's SMB/CIFS analyzer parses both +## SMB-over-NetBIOS on ports 138/139 and SMB-over-TCP on port 445. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## trans: The parsed transaction header. +## +## data: The raw transaction data. +## +## is_orig: True if the message was sent by the originator of the connection. +## +## .. bro:see:: smb_com_close smb_com_generic_andx smb_com_logoff_andx +## smb_com_negotiate smb_com_negotiate_response smb_com_nt_create_andx +## smb_com_read_andx smb_com_setup_andx smb_com_trans_mailslot smb_com_trans_pipe +## smb_com_trans_rap smb_com_transaction2 smb_com_tree_connect_andx +## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral +## smb_message event smb_com_transaction%(c: connection, hdr: smb_hdr, trans: smb_trans, data: smb_trans_data, is_orig: bool%); + +## Generated for SMB/CIFS messages of type *nt transaction 2*. +## +## See `Wikipedia `__ for more +## information about the SMB/CIFS protocol. Bro's SMB/CIFS analyzer parses both +## SMB-over-NetBIOS on ports 138/139 and SMB-over-TCP on port 445. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## trans: The parsed transaction header. +## +## data: The raw transaction data. +## +## is_orig: True if the message was sent by the originator of the connection. +## +## .. bro:see:: smb_com_close smb_com_generic_andx smb_com_logoff_andx +## smb_com_negotiate smb_com_negotiate_response smb_com_nt_create_andx +## smb_com_read_andx smb_com_setup_andx smb_com_trans_mailslot smb_com_trans_pipe +## smb_com_trans_rap smb_com_transaction smb_com_tree_connect_andx +## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral +## smb_message event smb_com_transaction2%(c: connection, hdr: smb_hdr, trans: smb_trans, data: smb_trans_data, is_orig: bool%); + +## Generated for SMB/CIFS messages of type *transaction mailslot*. +## +## See `Wikipedia `__ for more +## information about the SMB/CIFS protocol. Bro's SMB/CIFS analyzer parses both +## SMB-over-NetBIOS on ports 138/139 and SMB-over-TCP on port 445. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## trans: The parsed transaction header. +## +## data: The raw transaction data. +## +## is_orig: True if the message was sent by the originator of the connection. +## +## .. bro:see:: smb_com_close smb_com_generic_andx smb_com_logoff_andx +## smb_com_negotiate smb_com_negotiate_response smb_com_nt_create_andx +## smb_com_read_andx smb_com_setup_andx smb_com_trans_pipe smb_com_trans_rap +## smb_com_transaction smb_com_transaction2 smb_com_tree_connect_andx +## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral +## smb_message event smb_com_trans_mailslot%(c: connection, hdr: smb_hdr, trans: smb_trans, data: smb_trans_data, is_orig: bool%); + +## Generated for SMB/CIFS messages of type *transaction rap*. +## +## See `Wikipedia `__ for more +## information about the SMB/CIFS protocol. Bro's SMB/CIFS analyzer parses both +## SMB-over-NetBIOS on ports 138/139 and SMB-over-TCP on port 445. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## trans: The parsed transaction header. +## +## data: The raw transaction data. +## +## is_orig: True if the message was sent by the originator of the connection. +## +## .. bro:see:: smb_com_close smb_com_generic_andx smb_com_logoff_andx +## smb_com_negotiate smb_com_negotiate_response smb_com_nt_create_andx +## smb_com_read_andx smb_com_setup_andx smb_com_trans_mailslot +## smb_com_trans_pipe smb_com_transaction smb_com_transaction2 +## smb_com_tree_connect_andx smb_com_tree_disconnect smb_com_write_andx smb_error +## smb_get_dfs_referral smb_message event smb_com_trans_rap%(c: connection, hdr: smb_hdr, trans: smb_trans, data: smb_trans_data, is_orig: bool%); + +## Generated for SMB/CIFS messages of type *transaction pipe*. +## +## See `Wikipedia `__ for more +## information about the SMB/CIFS protocol. Bro's SMB/CIFS analyzer parses both +## SMB-over-NetBIOS on ports 138/139 and SMB-over-TCP on port 445. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## trans: The parsed transaction header. +## +## data: The raw transaction data. +## +## is_orig: True if the message was sent by the originator of the connection. +## +## .. bro:see:: smb_com_close smb_com_generic_andx smb_com_logoff_andx +## smb_com_negotiate smb_com_negotiate_response smb_com_nt_create_andx +## smb_com_read_andx smb_com_setup_andx smb_com_trans_mailslot smb_com_trans_rap +## smb_com_transaction smb_com_transaction2 smb_com_tree_connect_andx +## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral +## smb_message event smb_com_trans_pipe%(c: connection, hdr: smb_hdr, trans: smb_trans, data: smb_trans_data, is_orig: bool%); + +## Generated for SMB/CIFS messages of type *read andx*. +## +## See `Wikipedia `__ for more +## information about the SMB/CIFS protocol. Bro's SMB/CIFS analyzer parses both +## SMB-over-NetBIOS on ports 138/139 and SMB-over-TCP on port 445. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## data: Always empty. +## +## is_orig: True if the message was sent by the originator of the connection. +## +## .. bro:see:: smb_com_close smb_com_generic_andx smb_com_logoff_andx +## smb_com_negotiate smb_com_negotiate_response smb_com_nt_create_andx +## smb_com_setup_andx smb_com_trans_mailslot smb_com_trans_pipe smb_com_trans_rap +## smb_com_transaction smb_com_transaction2 smb_com_tree_connect_andx +## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral +## smb_message event smb_com_read_andx%(c: connection, hdr: smb_hdr, data: string%); + +## Generated for SMB/CIFS messages of type *read andx*. +## +## See `Wikipedia `__ for more +## information about the SMB/CIFS protocol. Bro's SMB/CIFS analyzer parses both +## SMB-over-NetBIOS on ports 138/139 and SMB-over-TCP on port 445. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## data: Always empty. +## +## is_orig: True if the message was sent by the originator of the connection. +## +## .. bro:see:: smb_com_close smb_com_generic_andx smb_com_logoff_andx +## smb_com_negotiate smb_com_negotiate_response smb_com_nt_create_andx +## smb_com_read_andx smb_com_setup_andx smb_com_trans_mailslot +## smb_com_trans_pipe smb_com_trans_rap smb_com_transaction smb_com_transaction2 +## smb_com_tree_connect_andx smb_com_tree_disconnect smb_error +## smb_get_dfs_referral smb_message event smb_com_write_andx%(c: connection, hdr: smb_hdr, data: string%); + +## Generated for SMB/CIFS messages of type *get dfs referral*. +## +## See `Wikipedia `__ for more +## information about the SMB/CIFS protocol. Bro's SMB/CIFS analyzer parses both +## SMB-over-NetBIOS on ports 138/139 and SMB-over-TCP on port 445. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## max_referral_level: The ``max_referral_level`` attribute specified in the +## message. +## +## file_name: The ``filene_name`` attribute specified in the message. +## +## .. bro:see:: smb_com_close smb_com_generic_andx smb_com_logoff_andx +## smb_com_negotiate smb_com_negotiate_response smb_com_nt_create_andx +## smb_com_read_andx smb_com_setup_andx smb_com_trans_mailslot +## smb_com_trans_pipe smb_com_trans_rap smb_com_transaction smb_com_transaction2 +## smb_com_tree_connect_andx smb_com_tree_disconnect smb_com_write_andx smb_error +## smb_message event smb_get_dfs_referral%(c: connection, hdr: smb_hdr, max_referral_level: count, file_name: string%); + +## Generated for SMB/CIFS messages of type *negotiate*. +## +## See `Wikipedia `__ for more +## information about the SMB/CIFS protocol. Bro's SMB/CIFS analyzer parses both +## SMB-over-NetBIOS on ports 138/139 and SMB-over-TCP on port 445. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## .. bro:see:: smb_com_close smb_com_generic_andx smb_com_logoff_andx +## smb_com_negotiate_response smb_com_nt_create_andx smb_com_read_andx smb_com_setup_andx +## smb_com_trans_mailslot smb_com_trans_pipe smb_com_trans_rap smb_com_transaction +## smb_com_transaction2 smb_com_tree_connect_andx smb_com_tree_disconnect +## smb_com_write_andx smb_error smb_get_dfs_referral smb_message event smb_com_negotiate%(c: connection, hdr: smb_hdr%); + +## Generated for SMB/CIFS messages of type *negotiate response*. +## +## See `Wikipedia `__ for more +## information about the SMB/CIFS protocol. Bro's SMB/CIFS analyzer parses both +## SMB-over-NetBIOS on ports 138/139 and SMB-over-TCP on port 445. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## dialect_index: The ``dialect`` indicated in the message. +## +## .. bro:see:: smb_com_close smb_com_generic_andx smb_com_logoff_andx +## smb_com_negotiate smb_com_nt_create_andx smb_com_read_andx smb_com_setup_andx +## smb_com_trans_mailslot smb_com_trans_pipe smb_com_trans_rap smb_com_transaction +## smb_com_transaction2 smb_com_tree_connect_andx smb_com_tree_disconnect +## smb_com_write_andx smb_error smb_get_dfs_referral smb_message event smb_com_negotiate_response%(c: connection, hdr: smb_hdr, dialect_index: count%); + +## Generated for SMB/CIFS messages of type *setup andx*. +## +## See `Wikipedia `__ for more +## information about the SMB/CIFS protocol. Bro's SMB/CIFS analyzer parses both +## SMB-over-NetBIOS on ports 138/139 and SMB-over-TCP on port 445. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## .. bro:see:: smb_com_close smb_com_generic_andx smb_com_logoff_andx +## smb_com_negotiate smb_com_negotiate_response smb_com_nt_create_andx +## smb_com_read_andx smb_com_trans_mailslot smb_com_trans_pipe smb_com_trans_rap +## smb_com_transaction smb_com_transaction2 smb_com_tree_connect_andx +## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral +## smb_message event smb_com_setup_andx%(c: connection, hdr: smb_hdr%); + +## Generated for SMB/CIFS messages of type *generic andx*. +## +## See `Wikipedia `__ for more +## information about the SMB/CIFS protocol. Bro's SMB/CIFS analyzer parses both +## SMB-over-NetBIOS on ports 138/139 and SMB-over-TCP on port 445. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## .. bro:see:: smb_com_close smb_com_logoff_andx smb_com_negotiate +## smb_com_negotiate_response smb_com_nt_create_andx smb_com_read_andx +## smb_com_setup_andx smb_com_trans_mailslot smb_com_trans_pipe smb_com_trans_rap +## smb_com_transaction smb_com_transaction2 smb_com_tree_connect_andx +## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral +## smb_message event smb_com_generic_andx%(c: connection, hdr: smb_hdr%); + +## Generated for SMB/CIFS messages of type *close*. +## +## See `Wikipedia `__ for more +## information about the SMB/CIFS protocol. Bro's SMB/CIFS analyzer parses both +## SMB-over-NetBIOS on ports 138/139 and SMB-over-TCP on port 445. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## .. bro:see:: smb_com_generic_andx smb_com_logoff_andx smb_com_negotiate +## smb_com_negotiate_response smb_com_nt_create_andx smb_com_read_andx +## smb_com_setup_andx smb_com_trans_mailslot smb_com_trans_pipe smb_com_trans_rap +## smb_com_transaction smb_com_transaction2 smb_com_tree_connect_andx +## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral +## smb_message event smb_com_close%(c: connection, hdr: smb_hdr%); + +## Generated for SMB/CIFS messages of type *logoff andx*. +## +## See `Wikipedia `__ for more +## information about the SMB/CIFS protocol. Bro's SMB/CIFS analyzer parses both +## SMB-over-NetBIOS on ports 138/139 and SMB-over-TCP on port 445. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## .. bro:see:: smb_com_close smb_com_generic_andx smb_com_negotiate +## smb_com_negotiate_response smb_com_nt_create_andx smb_com_read_andx +## smb_com_setup_andx smb_com_trans_mailslot smb_com_trans_pipe smb_com_trans_rap +## smb_com_transaction smb_com_transaction2 smb_com_tree_connect_andx +## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral +## smb_message event smb_com_logoff_andx%(c: connection, hdr: smb_hdr%); + +## Generated for SMB/CIFS messages that indicate an error. This event is triggered +## by an SMB header including a status that signals an error. +## +## c: The connection. +## +## hdr: The parsed header of the SMB message. +## +## cmd: The SMB command code. +## +## cmd_str: A string mmenonic of the SMB command code. +## +## body: The raw SMB message body, i.e., the data starting after the SMB header. +## +## .. bro:see:: smb_com_close smb_com_generic_andx smb_com_logoff_andx +## smb_com_negotiate smb_com_negotiate_response smb_com_nt_create_andx +## smb_com_read_andx smb_com_setup_andx smb_com_trans_mailslot +## smb_com_trans_pipe smb_com_trans_rap smb_com_transaction smb_com_transaction2 +## smb_com_tree_connect_andx smb_com_tree_disconnect smb_com_write_andx +## smb_get_dfs_referral smb_message event smb_error%(c: connection, hdr: smb_hdr, cmd: count, cmd_str: string, data: string%); +## Generated for all DNS messages. +## +## See `Wikipedia `__ for more +## information about the DNS protocol. Bro analyzes both UDP and TCP DNS sessions. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## is_orig: True if the message was sent by the originator of the connection. +## +## msg: The parsed DNS message header. +## +## len: The length of the message's raw representation (i.e, the DNS payload). +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end +## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_query_reply dns_rejected +## dns_request non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_message%(c: connection, is_orig: bool, msg: dns_msg, len: count%) &group="dns"; + +## Generated for DNS requests. For requests with multiple queries, this event +## is raised once for each. +## +## See `Wikipedia `__ for more +## information about the DNS protocol. Bro analyzes both UDP and TCP DNS sessions. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## query: The queried name. +## +## qtype: The queried resource record type. +## +## qclass: The queried resource record class. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end +## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_request%(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count%) &group="dns"; -event dns_full_request%(%) &group="dns"; + +## Generated for DNS replies that reject a query. This event is raised if a DNS +## reply either indicates failure via its status code or does not pass on any +## answers to a query. Note that all of the event's paramaters are parsed out of +## the reply; there's no stateful correlation with the query. +## +## See `Wikipedia `__ for more +## information about the DNS protocol. Bro analyzes both UDP and TCP DNS sessions. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## query: The queried name. +## +## qtype: The queried resource record type. +## +## qclass: The queried resource record class. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end +## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_request non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_rejected%(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count%) &group="dns"; -event dns_A_reply%(c: connection, msg: dns_msg, ans: dns_answer, a: addr%) &group="dns"; -event dns_AAAA_reply%(c: connection, msg: dns_msg, ans: dns_answer, a: addr, astr: string%) &group="dns"; -event dns_NS_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string%) &group="dns"; -event dns_CNAME_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string%) &group="dns"; -event dns_PTR_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string%) &group="dns"; -event dns_SOA_reply%(c: connection, msg: dns_msg, ans: dns_answer, soa: dns_soa%) &group="dns"; -event dns_WKS_reply%(c: connection, msg: dns_msg, ans: dns_answer%) &group="dns"; -event dns_HINFO_reply%(c: connection, msg: dns_msg, ans: dns_answer%) &group="dns"; -event dns_MX_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string, preference: count%) &group="dns"; -event dns_TXT_reply%(c: connection, msg: dns_msg, ans: dns_answer, str: string%) &group="dns"; -event dns_SRV_reply%(c: connection, msg: dns_msg, ans: dns_answer%) &group="dns"; -event dns_EDNS%(c: connection, msg: dns_msg, ans: dns_answer%) &group="dns"; -event dns_EDNS_addl%(c: connection, msg: dns_msg, ans: dns_edns_additional%) &group="dns"; -event dns_TSIG_addl%(c: connection, msg: dns_msg, ans: dns_tsig_additional%) &group="dns"; - -# Generated at the end of processing a DNS packet. -event dns_end%(c: connection, msg: dns_msg%) &group="dns"; +## Generated for DNS replies with an *ok* status code but no question section. +## +## See `Wikipedia `__ for more +## information about the DNS protocol. Bro analyzes both UDP and TCP DNS sessions. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## query: The queried name. +## +## qtype: The queried resource record type. +## +## qclass: The queried resource record class. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end +## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_rejected +## dns_request non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_query_reply%(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count%) &group="dns"; -# Generated when a port 53 UDP message cannot be parsed as a DNS request. +## Generated when the DNS analyzer processes what seems to be a non-DNS packets. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The raw DNS payload. +## +## .. note:: This event is deprecated and superseded by Bro's dynamic protocol +## detection framework. event non_dns_request%(c: connection, msg: string%) &group="dns"; +## Generated for DNS replies of type *A*. For replies with multiple answers, an +## individual event of the corresponding type is raised for each. +## +## See `Wikipedia `__ for more +## information about the DNS protocol. Bro analyzes both UDP and TCP DNS sessions. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## ans: The type-independent part of the parsed answer record. +## +## a: The address returned by the reply. +## +## .. bro:see:: dns_AAAA_reply dns_CNAME_reply dns_EDNS_addl dns_HINFO_reply +## dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply +## dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end dns_full_request +## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth +## +## .. note: This event is currently also raised for ``AAAA`` records. In that +## case, the address *a* will correspond to the lower-order 4 bytes of the +## IPv6 address. This will go away once IPv6 support is improved. +## +## .. todo: IPv6 handling is obviously very broken here ... +event dns_A_reply%(c: connection, msg: dns_msg, ans: dns_answer, a: addr%) &group="dns"; + +## Generated for DNS replies of type *AAAA*. For replies with multiple answers, an +## individual event of the corresponding type is raised for each. +## +## See `Wikipedia `__ for more +## information about the DNS protocol. Bro analyzes both UDP and TCP DNS sessions. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## ans: The type-independent part of the parsed answer record. +## +## a: The address returned by the reply. +## +## .. bro:see:: dns_A_reply dns_CNAME_reply dns_EDNS_addl dns_HINFO_reply dns_MX_reply +## dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply dns_TSIG_addl +## dns_TXT_reply dns_WKS_reply dns_end dns_full_request dns_mapping_altered +## dns_mapping_lost_name dns_mapping_new_name dns_mapping_unverified +## dns_mapping_valid dns_message dns_query_reply dns_rejected dns_request +## non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_skip_all_addl dns_skip_all_auth dns_skip_auth +## +## .. todo: Raising this event is not implemented currently, not even when +## Bro's compiled IPv6 support. ``AAAA`` are currently always turned into +## :bro:id:`dns_A_reply` events. +event dns_AAAA_reply%(c: connection, msg: dns_msg, ans: dns_answer, a: addr, astr: string%) &group="dns"; + +## Generated for DNS replies of type *NS*. For replies with multiple answers, an +## individual event of the corresponding type is raised for each. +## +## See `Wikipedia `__ for more +## information about the DNS protocol. Bro analyzes both UDP and TCP DNS sessions. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## ans: The type-independent part of the parsed answer record. +## +## name: The name returned by the reply. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply +## dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end dns_full_request +## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth +event dns_NS_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string%) &group="dns"; + +## Generated for DNS replies of type *CNAME*. For replies with multiple answers, +## an individual event of the corresponding type is raised for each. +## +## See `Wikipedia `__ for more +## information about the DNS protocol. Bro analyzes both UDP and TCP DNS sessions. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## ans: The type-independent part of the parsed answer record. +## +## name: The name returned by the reply. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_EDNS_addl dns_HINFO_reply dns_MX_reply +## dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply dns_TSIG_addl +## dns_TXT_reply dns_WKS_reply dns_end dns_full_request dns_mapping_altered +## dns_mapping_lost_name dns_mapping_new_name dns_mapping_unverified +## dns_mapping_valid dns_message dns_query_reply dns_rejected dns_request +## non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_skip_all_addl dns_skip_all_auth dns_skip_auth +event dns_CNAME_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string%) &group="dns"; + +## Generated for DNS replies of type *PTR*. For replies with multiple answers, +## an individual event of the corresponding type is raised for each. +## +## See `Wikipedia `__ for more +## information about the DNS protocol. Bro analyzes both UDP and TCP DNS sessions. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## ans: The type-independent part of the parsed answer record. +## +## name: The name returned by the reply. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_SOA_reply dns_SRV_reply +## dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end dns_full_request +## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth +event dns_PTR_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string%) &group="dns"; + +## Generated for DNS replies of type *CNAME*. For replies with multiple answers, +## an individual event of the corresponding type is raised for each. +## +## See `Wikipedia `__ for more +## information about the DNS protocol. Bro analyzes both UDP and TCP DNS sessions. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## ans: The type-independent part of the parsed answer record. +## +## soa: The parsed SOA value +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SRV_reply +## dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end dns_full_request +## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth +event dns_SOA_reply%(c: connection, msg: dns_msg, ans: dns_answer, soa: dns_soa%) &group="dns"; + +## Generated for DNS replies of type *WKS*. For replies with multiple answers, an +## individual event of the corresponding type is raised for each. +## +## See `Wikipedia `__ for more +## information about the DNS protocol. Bro analyzes both UDP and TCP DNS sessions. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## ans: The type-independent part of the parsed answer record. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_end dns_full_request +## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth +event dns_WKS_reply%(c: connection, msg: dns_msg, ans: dns_answer%) &group="dns"; + +## Generated for DNS replies of type *HINFO*. For replies with multiple answers, an +## individual event of the corresponding type is raised for each. +## +## See `Wikipedia `__ for more +## information about the DNS protocol. Bro analyzes both UDP and TCP DNS sessions. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## ans: The type-independent part of the parsed answer record. +## +## name: The name returned by the reply. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl dns_MX_reply +## dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply dns_TSIG_addl +## dns_TXT_reply dns_WKS_reply dns_end dns_full_request dns_mapping_altered +## dns_mapping_lost_name dns_mapping_new_name dns_mapping_unverified +## dns_mapping_valid dns_message dns_query_reply dns_rejected dns_request +## non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_skip_all_addl dns_skip_all_auth dns_skip_auth +event dns_HINFO_reply%(c: connection, msg: dns_msg, ans: dns_answer%) &group="dns"; + +## Generated for DNS replies of type *MX*. For replies with multiple answers, an +## individual event of the corresponding type is raised for each. +## +## See `Wikipedia `__ for more +## information about the DNS protocol. Bro analyzes both UDP and TCP DNS sessions. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## ans: The type-independent part of the parsed answer record. +## +## name: The name returned by the reply. +## +## preference: The preference for *name* specificed by the reply. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply +## dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end dns_full_request +## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth +event dns_MX_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string, preference: count%) &group="dns"; + +## Generated for DNS replies of type *TXT*. For replies with multiple answers, an +## individual event of the corresponding type is raised for each. +## +## See `Wikipedia `__ for more +## information about the DNS protocol. Bro analyzes both UDP and TCP DNS sessions. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## ans: The type-independent part of the parsed answer record. +## +## str: The textual information returned by the reply. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply +## dns_SRV_reply dns_TSIG_addl dns_WKS_reply dns_end dns_full_request +## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth +event dns_TXT_reply%(c: connection, msg: dns_msg, ans: dns_answer, str: string%) &group="dns"; + +## Generated for DNS replies of type *SRV*. For replies with multiple answers, an +## individual event of the corresponding type is raised for each. +## +## See `Wikipedia `__ for more +## information about the DNS protocol. Bro analyzes both UDP and TCP DNS sessions. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## ans: The type-independent part of the parsed answer record. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply +## dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end dns_full_request +## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth +event dns_SRV_reply%(c: connection, msg: dns_msg, ans: dns_answer%) &group="dns"; + +## Generated for DNS replies of type *EDNS*. For replies with multiple answers, an +## individual event of the corresponding type is raised for each. +## +## See `Wikipedia `__ for more +## information about the DNS protocol. Bro analyzes both UDP and TCP DNS sessions. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## ans: The parsed EDNS reply. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_HINFO_reply dns_MX_reply +## dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply dns_TSIG_addl +## dns_TXT_reply dns_WKS_reply dns_end dns_full_request dns_mapping_altered +## dns_mapping_lost_name dns_mapping_new_name dns_mapping_unverified +## dns_mapping_valid dns_message dns_query_reply dns_rejected dns_request +## non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_skip_all_addl dns_skip_all_auth dns_skip_auth +event dns_EDNS_addl%(c: connection, msg: dns_msg, ans: dns_edns_additional%) &group="dns"; + +## Generated for DNS replies of type *TSIG*. For replies with multiple answers, an +## individual event of the corresponding type is raised for each. +## +## See `Wikipedia `__ for more +## information about the DNS protocol. Bro analyzes both UDP and TCP DNS sessions. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## ans: The parsed TSIG reply. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply +## dns_SRV_reply dns_TXT_reply dns_WKS_reply dns_end dns_full_request +## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth +event dns_TSIG_addl%(c: connection, msg: dns_msg, ans: dns_tsig_additional%) &group="dns"; + +## Generated at the end of processing a DNS packet. This event is the last +## ``dns_*`` event that will be raised for a DNS query/reply and signals that +## all resource records have been passed on. +## +## See `Wikipedia `__ for more +## information about the DNS protocol. Bro analyzes both UDP and TCP DNS sessions. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_full_request +## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth +event dns_end%(c: connection, msg: dns_msg%) &group="dns"; + +## Generated for DHCP messages of type *discover*. +## +## See `Wikipedia +## `__ for more +## information about the DHCP protocol. +## +## c: The connection record describing the underlying UDP flow.. +## +## msg: The parsed type-indepedent part of the DHCP message. +## +## req_addr: The specific address requested by the client. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end +## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth +## +## .. note: Bro does not support broadcast packets (as used by the DHCP protocol). +## It treats broadcast addresses just like any other and associates packets into +## transport-level flows in the same way as usual. event dhcp_discover%(c: connection, msg: dhcp_msg, req_addr: addr%); + +## Generated for DHCP messages of type *offer*. +## +## See `Wikipedia +## `__ for more +## information about the DHCP protocol. +## +## c: The connection record describing the underlying UDP flow.. +## +## mask: The subnet mask specified by the mesage. +## +## router: The list of routers specified by the message. +## +## lease: The least interval specificed by the message. +## +## serv_addr: The server address specified by the message. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end +## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected dns_request non_dns_request +## +## .. note: Bro does not support broadcast packets (as used by the DHCP protocol). +## It treats broadcast addresses just like any other and associates packets into +## transport-level flows in the same way as usual. event dhcp_offer%(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr%); + +## Generated for DHCP messages of type *request*. +## +## See `Wikipedia +## `__ for more +## information about the DHCP protocol. +## +## c: The connection record describing the underlying UDP flow.. +## +## msg: The parsed type-indepedent part of the DHCP message. +## +## req_addr: The client address specified by the message. +## +## serv_addr: The server address specified by the message. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end +## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected dns_request non_dns_request +## +## .. note: Bro does not support broadcast packets (as used by the DHCP protocol). +## It treats broadcast addresses just like any other and associates packets into +## transport-level flows in the same way as usual. event dhcp_request%(c: connection, msg: dhcp_msg, req_addr: addr, serv_addr: addr%); + +## Generated for DHCP messages of type *decline*. +## +## See `Wikipedia +## `__ for more +## information about the DHCP protocol. +## +## c: The connection record describing the underlying UDP flow.. +## +## msg: The parsed type-indepedent part of the DHCP message. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end +## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected dns_request non_dns_request +## +## .. note: Bro does not support broadcast packets (as used by the DHCP protocol). +## It treats broadcast addresses just like any other and associates packets into +## transport-level flows in the same way as usual. event dhcp_decline%(c: connection, msg: dhcp_msg%); + +## Generated for DHCP messages of type *acknowledgment*. +## +## See `Wikipedia +## `__ for more +## information about the DHCP protocol. +## +## c: The connection record describing the underlying UDP flow.. +## +## msg: The parsed type-indepedent part of the DHCP message. +## +## mask: The subnet mask specified by the mesage. +## +## router: The list of routers specified by the message. +## +## lease: The least interval specificed by the message. +## +## serv_addr: The server address specified by the message. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end +## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected dns_request non_dns_request +## +## .. note: Bro does not support broadcast packets (as used by the DHCP protocol). +## It treats broadcast addresses just like any other and associates packets into +## transport-level flows in the same way as usual. event dhcp_ack%(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr%); + +## Generated for DHCP messages of type *negative acknowledgment*. +## +## See `Wikipedia +## `__ for more +## information about the DHCP protocol. +## +## c: The connection record describing the underlying UDP flow.. +## +## msg: The parsed type-indepedent part of the DHCP message. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end +## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected dns_request non_dns_request +## +## .. note: Bro does not support broadcast packets (as used by the DHCP protocol). +## It treats broadcast addresses just like any other and associates packets into +## transport-level flows in the same way as usual. event dhcp_nak%(c: connection, msg: dhcp_msg%); + +## Generated for DHCP messages of type *release*. +## +## See `Wikipedia +## `__ for more +## information about the DHCP protocol. +## +## c: The connection record describing the underlying UDP flow.. +## +## msg: The parsed type-indepedent part of the DHCP message. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end +## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected dns_request non_dns_request +## +## .. note: Bro does not support broadcast packets (as used by the DHCP protocol). +## It treats broadcast addresses just like any other and associates packets into +## transport-level flows in the same way as usual. event dhcp_release%(c: connection, msg: dhcp_msg%); + +## Generated for DHCP messages of type *inform*. +## +## See `Wikipedia +## `__ for more +## information about the DHCP protocol. +## +## c: The connection record describing the underlying UDP flow.. +## +## msg: The parsed type-indepedent part of the DHCP message. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end +## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply +## dns_rejected dns_request non_dns_request +## +## .. note: Bro does not support broadcast packets (as used by the DHCP protocol). +## It treats broadcast addresses just like any other and associates packets into +## transport-level flows in the same way as usual. event dhcp_inform%(c: connection, msg: dhcp_msg%); +## Generated for HTTP requests. Bro supports persistent and pipelined HTTP sessions +## and raises corresponding events as it parses client/server dialogues. This event +## is generated as soon as a request's initial line has been parsed, and before any +## :bro:id:`http_header` events are raised. +## +## See `Wikipedia `__ for +## more information about the HTTP protocol. +## +## c: The connection. +## +## method: The HTTP method extracted from the request (e.g., ``GET``, ``POST``). +## +## original_URI: The unprocessed URI as specified in the request. +## +## unescaped_URI: The URI with all percent-encodings decoded. +## +## version: The version number specified in the request (e.g., ``1.1``). +## +## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity +## http_entity_data http_event http_header http_message_done http_reply http_stats +## truncate_http_URI event http_request%(c: connection, method: string, original_URI: string, unescaped_URI: string, version: string%) &group="http-request"; + +## Generated for HTTP replies. Bro supports persistent and pipelined HTTP sessions +## and raises corresponding events as it parses client/server dialogues. This event +## is generated as soon as a reply's initial line has been parsed, and before any +## :bro:id:`http_header` events are raised. +## +## See `Wikipedia `__ for +## more information about the HTTP protocol. +## +## c: The connection. +## +## version: The version number specified in the reply (e.g., ``1.1``). +## +## code: The numerical response code returned by the server. +## +## reason: The textual description returned by the server along with *code*. +## +## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity +## http_entity_data http_event http_header http_message_done http_request +## http_stats event http_reply%(c: connection, version: string, code: count, reason: string%) &group="http-reply"; + +## Generated for HTTP headers. Bro supports persistent and pipelined HTTP sessions +## and raises corresponding events as it parses client/server dialogues. +## +## See `Wikipedia `__ for +## more information about the HTTP protocol. +## +## c: The connection. +## +## is_orig: True if the header was sent by the originator of the TCP connection. +## +## name: The name of the header. +## +## value: The value of the header. +## +## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity +## http_entity_data http_event http_message_done http_reply http_request +## http_stats +## +## .. note:: This event is also raised for headers found in nested body entities. event http_header%(c: connection, is_orig: bool, name: string, value: string%) &group="http-header"; + +## Generated for HTTP headers, passing on all headers of an HTTP message at once. +## Bro supports persistent and pipelined HTTP sessions and raises corresponding +## events as it parses client/server dialogues. +## +## See `Wikipedia `__ for +## more information about the HTTP protocol. +## +## c: The connection. +## +## is_orig: True if the header was sent by the originator of the TCP connection. +## +## hlist: A *table* containing all headers extracted from the current entity. +## The table is indexed by the position of the header (1 for the first, 2 for the +## second, etc.). +## +## .. bro:see:: http_begin_entity http_content_type http_end_entity http_entity_data +## http_event http_header http_message_done http_reply http_request http_stats +## +## .. note:: This event is also raised for headers found in nested body entities. event http_all_headers%(c: connection, is_orig: bool, hlist: mime_header_list%) &group="http-header"; + +## Generated when starting to parse an HTTP body entity. This event is generated +## at least once for each non-empty (client or server) HTTP body; and potentially +## more than once if the body contains further nested MIME entities. Bro raises +## this event just before it starts parsing each entity's content. +## +## See `Wikipedia `__ for +## more information about the HTTP protocol. +## +## c: The connection. +## +## is_orig: True if the entity was was sent by the originator of the TCP +## connection. +## +## .. bro:see:: http_all_headers http_content_type http_end_entity http_entity_data +## http_event http_header http_message_done http_reply http_request http_stats +## mime_begin_entity event http_begin_entity%(c: connection, is_orig: bool%) &group="http-body"; + +## Generated when finishing parsing an HTTP body entity. This event is generated +## at least once for each non-empty (client or server) HTTP body; and potentially +## more than once if the body contains further nested MIME entities. Bro raises +## this event at the point when it has finished parsing an entity's content. +## +## See `Wikipedia `__ for +## more information about the HTTP protocol. +## +## c: The connection. +## +## is_orig: True if the entity was was sent by the originator of the TCP +## connection. +## +## .. bro:see:: http_all_headers http_begin_entity http_content_type http_entity_data +## http_event http_header http_message_done http_reply http_request +## http_stats mime_end_entity event http_end_entity%(c: connection, is_orig: bool%) &group="http-body"; -event http_content_type%(c: connection, is_orig: bool, ty: string, subty: string%) &group="http-body"; + +## Generated when parsing an HTTP body entity, passing on the data. This event +## can potentially be raised many times for each entity, each time passing a +## chunk of the data of not further defined size. +## +## A common idiom for using this event is to first *reassemble* the data +## at the scripting layer by concatening it to a successvily growing +## string; and only perform further content analysis once the corresponding +## :bro:id:`http_end_entity` event has been raised. Note, however, that doing so +## can be quite expensive for HTTP tranders. At the very least, one should +## impose an upper size limit on how much data is being buffered. +## +## See `Wikipedia `__ for +## more information about the HTTP protocol. +## +## c: The connection. +## +## is_orig: True if the entity was was sent by the originator of the TCP +## connection. +## +## length: The length of *data*. +## +## data: One chunk of raw entity data. +## +## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity +## http_event http_header http_message_done http_reply http_request http_stats +## mime_entity_data http_entity_data_delivery_size skip_http_data event http_entity_data%(c: connection, is_orig: bool, length: count, data: string%) &group="http-body"; + +## Generated for reporting an HTTP bodie's content type. This event is +## generated at the end of parsing an HTTP header, passing on the MIME +## type as specified by the ``Content-Type`` header. If that header is +## missing, this event is still raised with a default value of ``text/plain``. +## +## See `Wikipedia `__ for +## more information about the HTTP protocol. +## +## c: The connection. +## +## is_orig: True if the entity was was sent by the originator of the TCP +## connection. +## +## ty: The main type. +## +## subty: The subtype. +## +## .. bro:see:: http_all_headers http_begin_entity http_end_entity http_entity_data +## http_event http_header http_message_done http_reply http_request http_stats +## +## .. note:: This event is also raised for headers found in nested body +## entities. +event http_content_type%(c: connection, is_orig: bool, ty: string, subty: string%) &group="http-body"; + +## Generated once at the end of parsing an HTTP message. Bro supports persistent +## and pipelined HTTP sessions and raises corresponding events as it parses +## client/server dialogues. A "message" is one top-level HTTP entity, such as a +## complete request or reply. Each message can have further nested sub-entities +## inside. This event is raised once all sub-entities belonging to a top-level +## message have been processed (and their corresponding ``http_entity_*`` events +## generated). +## +## See `Wikipedia `__ for +## more information about the HTTP protocol. +## +## c: The connection. +## +## is_orig: True if the entity was was sent by the originator of the TCP +## connection. +## +## stat: Further meta information about the message. +## +## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity +## http_entity_data http_event http_header http_reply http_request http_stats event http_message_done%(c: connection, is_orig: bool, stat: http_message_stat%) &group="http-body"; + +## Generated for errors found when decoding HTTP requests or replies. +## +## See `Wikipedia `__ for +## more information about the HTTP protocol. +## +## c: The connection. +## +## event_type: A string describing the general category of the problem found (e.g., +## ``illegal format``). +## +## detail: Further more detailed description of the error. +## +## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity +## http_entity_data http_header http_message_done http_reply http_request +## http_stats mime_event event http_event%(c: connection, event_type: string, detail: string%); + +## Generated at the end of an HTTP session to report statistics about it. This +## event is raised after all of an HTTP session's requests and replies have been +## fully processed. +## +## c: The connection. +## +## stats: Statistics summarizing HTTP-level properties of the finished connection. +## +## .. bro:see:: http_all_headers http_begin_entity http_content_type http_end_entity +## http_entity_data http_event http_header http_message_done http_reply +## http_request event http_stats%(c: connection, stats: http_stats_rec%); +## Generated when seeing an SSH client's version identification. The SSH protocol +## starts with a clear-test handshake message that reports client and server +## protocol/software versions. This event provides access to what the client +## sent. +## +## +## See `Wikipedia `__ for more +## information about the SSH protocol. +## +## c: The connection. +## +## version: The version string the client sent (e.g., `SSH-2.0-libssh-0.11`). +## +## .. bro:see:: ssh_server_version +## +## .. note:: As everything after the initial version handshake proceeds encrypted, +## Bro cannot further analyze SSH sessions. event ssh_client_version%(c: connection, version: string%); + +## Generated when seeing an SSH server's version identification. The SSH protocol +## starts with a clear-test handshake message that reports client and server +## protocol/software versions. This event provides access to what the server +## sent. +## +## See `Wikipedia `__ for more +## information about the SSH protocol. +## +## c: The connection. +## +## version: The version string the server sent (e.g., +## ``SSH-1.99-OpenSSH_3.9p1``). +## +## .. bro:see:: ssh_client_version +## +## .. note:: As everything coming after the initial version handshake proceeds +## encrypted, Bro cannot further analyze SSH sessions. event ssh_server_version%(c: connection, version: string%); +## Generated for an SSL/TLS client's initial *hello* message. SSL/TLS sessions +## start with an unencrypted handshake, and Bro extracts as much information out +## that it as it can. This event provides access to the initial information sent by +## the client. +## +## See `Wikipedia `__ for +## more information about the SSL/TLS protocol. +## +## c: The connection. +## +## version: The protocol version as extracted from the client's +## message. The values are standardized as part of the SSL/TLS protocol. The +## :bro:id:`SSL::version_strings` table maps them to descriptive names. +## +## possible_ts: The current time as sent by the client. Note that SSL/TLS does not +## require clocks to be set correctly, so treat with care. +## +## session_id: The session ID sent by the client (if any). +## +## ciphers: The list of ciphers the client offered to use. The values are +## standardized as part of the SSL/TLS protocol. The :bro:id:`SSL::cipher_desc` table +## maps them to descriptive names. +## +## .. bro:see:: ssl_alert ssl_established ssl_extension ssl_server_hello +## x509_certificate x509_error x509_extension ssl_max_cipherspec_size event ssl_client_hello%(c: connection, version: count, possible_ts: time, session_id: string, ciphers: count_set%); + +## Generated for an SSL/TLS servers's initial *hello* message. SSL/TLS sessions +## start with an unencrypted handshake, and Bro extracts as much information out +## of that as it can. This event provides access to the initial information sent by +## the client. +## +## See `Wikipedia `__ for +## more information about the SSL/TLS protocol. +## +## c: The connection. +## +## version: The protocol version as extracted from the servers's message. +## The values are standardized as part of the SSL/TLS protocol. The +## :bro:id:`SSL::version_strings` table maps them to descriptive names. +## +## possible_ts: The current time as sent by the server. Note that SSL/TLS does not +## require clocks to be set correctly, so treat with care. +## +## session_id: The session ID as sent back by the server (if any). +## +## cipher: The cipher chosen by the server. The values are standardized as part +## of the SSL/TLS protocol. The :bro:id:`SSL::cipher_desc` table maps them to +## descriptive names. +## +## comp_method: The compression method chosen by the client. The values are +## standardized as part of the SSL/TLS protocol. +## +## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_extension +## x509_certificate x509_error x509_extension ssl_max_cipherspec_size event ssl_server_hello%(c: connection, version: count, possible_ts: time, session_id: string, cipher: count, comp_method: count%); + +## Generated for SSL/TLS extensions seen in an initial handshake. SSL/TLS sessions +## start with an unencrypted handshake, and Bro extracts as much information out of +## that as it can. This event provides access to any extensions either side sents +## as part of extended *hello* message. +## +## c: The connection. +## +## code: The numerical code of the extension. The values are standardized as +## part of the SSL/TLS protocol. The :bro:id:`SSL::extensions` table maps them to +## descriptive names. +## +## val: The raw extension value that was sent in the message. +## +## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_server_hello +## x509_certificate x509_error x509_extension +## +## .. todo: The event lacks a ``is_orig`` parameter. event ssl_extension%(c: connection, code: count, val: string%); + +## Generated at the end of an SSL/TLS handshake. SSL/TLS sessions start with +## an unencrypted handshake, and Bro extracts as much information out of that as +## it can. This event signals the time when an SSL/TLS has finished the handshake +## and its endpoints consider it as fully established. Typically, everything from +## now on will be encrypted. +## +## See `Wikipedia `__ for +## more information about the SSL/TLS protocol. +## +## c: The connection. +## +## .. bro:see:: ssl_alert ssl_client_hello ssl_extension ssl_server_hello +## x509_certificate x509_error x509_extension event ssl_established%(c: connection%); + +## Generated for SSL/TLS alert records. SSL/TLS sessions start with an unencrypted +## handshake, and Bro extracts as much information out of that as it can. If during +## that handshake, an endpoint encounteres a fatal error, it sends an *alert* +## record, that it turns triggers this event. After an *alert*, any endpoint +## may close the connection immediately. +## +## See `Wikipedia `__ for +## more information about the SSL/TLS protocol. +## +## c: The connection. +## +## level: The severity level, as sent in the *alert*. The values are defined as +## part of the SSL/TLS protocol. +## +## desc: A numerical value identifying the cause of the *alert*. The values are +## defined as part of the SSL/TLS protocol. +## +## .. bro:see:: ssl_client_hello ssl_established ssl_extension ssl_server_hello +## x509_certificate x509_error x509_extension +## +## .. todo: The event lacks a ``is_orig`` parameter. event ssl_alert%(c: connection, level: count, desc: count%); +## Generated for x509 certificates seen in SSL/TLS connections. During the initial +## SSL/TLS handshake, certificates are exchanged in the clear. Bro raises this +## event for each certificate seen (including both a site's primary cert, and +## further certs sent as part of the validation chain). +## +## See `Wikipedia `__ for more information about +## the X.509 format. +## +## c: The connection. +## +## cert: The parsed certificate. +## +## is_server: True if the certificate was sent by the server. +## +## chain_idx: The index in the validation chain that this cert has. Index zero +## indicates an endpoints primary cert, while higher indices +## indicate the place in the validation chain (which has length +## *chain_len*). +## +## chain_len: The total length of the validation chain that this cert is part +## of. +## +## der_cert: The complete cert encoded in `DER +## inguished_Encoding_Rules`__ format. +## +## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_extension +## ssl_server_hello x509_error x509_extension x509_verify event x509_certificate%(c: connection, cert: X509, is_server: bool, chain_idx: count, chain_len: count, der_cert: string%); + +## Generated for X.509 extensions seen in a certificate. +## +## See `Wikipedia `__ for more information about +## the X.509 format. +## +## c: The connection. +## +## data: The raw data associated with the extension. +## +## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_extension +## ssl_server_hello x509_certificate x509_error x509_verify +## +## .. todo: The event lacks a ``is_orig`` parameter. event x509_extension%(c: connection, data: string%); + +## Generated when errors occur during parsing an X.509 certificate. +## +## See `Wikipedia `__ for more information about +## the X.509 format. +## +## c: The connection. +## +## err: An error code describing what went wrong. :bro:id:`SSL::x509_errors` maps +## error codes to a textual description. +## +## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_extension +## ssl_server_hello x509_certificate x509_extension x509_err2str x509_verify +## +## .. todo: The event lacks a ``is_orig`` parameter. event x509_error%(c: connection, err: count%); -event stp_create_endp%(c: connection, e: int, is_orig: bool%); -event stp_resume_endp%(e: int%); -event stp_correlate_pair%(e1: int, e2: int%); -event stp_remove_pair%(e1: int, e2: int%); -event stp_remove_endp%(e: int%); - +## TODO. +## +## .. bro:see:: rpc_call rpc_dialogue rpc_reply dce_rpc_bind dce_rpc_request +## dce_rpc_response rpc_timeout event dce_rpc_message%(c: connection, is_orig: bool, ptype: dce_rpc_ptype, msg: string%); + +## TODO. +## +## .. bro:see:: rpc_call rpc_dialogue rpc_reply dce_rpc_message dce_rpc_request +## dce_rpc_response rpc_timeout event dce_rpc_bind%(c: connection, uuid: string%); + +## TODO. +## +## .. bro:see:: rpc_call rpc_dialogue rpc_reply dce_rpc_bind dce_rpc_message +## dce_rpc_response rpc_timeout event dce_rpc_request%(c: connection, opnum: count, stub: string%); + +## TODO. +## +## .. bro:see:: rpc_call rpc_dialogue rpc_reply dce_rpc_bind dce_rpc_message +## dce_rpc_request rpc_timeout event dce_rpc_response%(c: connection, opnum: count, stub: string%); -# DCE/RPC endpoint mapper events. +## TODO. +## +## .. bro:see:: rpc_call rpc_dialogue rpc_reply dce_rpc_bind dce_rpc_message +## dce_rpc_request dce_rpc_response rpc_timeout event epm_map_response%(c: connection, uuid: string, p: port, h: addr%); -# "length" is the length of body (not including the frame header) +## Generated for NCP requests (Netware Core Protocol). +## +## See `Wikipedia `__ for more +## information about the NCP protocol. +## +## c: The connection. +## +## frame_type: The frame type, as specified by the protocol. +## +## length: The length of the request body, excluding the frame header, +## +## func: The requested function, as specified by the protocol. +## +## .. bro:see:: ncp_reply event ncp_request%(c: connection, frame_type: count, length: count, func: count%); + +## Generated for NCP replies (Netware Core Protocol). +## +## See `Wikipedia `__ for more +## information about the NCP protocol. +## +## c: The connection. +## +## frame_type: The frame type, as specified by the protocol. +## +## length: The length of the request body, excluding the frame header, +## +## req_frame: The frame type from the corresponding request. +## +## req_frame: The function code from the corresponding request. +## +## completion_code: The replie's completion code, as specified by the protocol. +## +## .. bro:see:: ncp_request event ncp_reply%(c: connection, frame_type: count, length: count, req_frame: count, req_func: count, completion_code: count%); -event interconn_stats%(c: connection, os: interconn_endp_stats, rs: interconn_endp_stats%); -event interconn_remove_conn%(c: connection%); - -event backdoor_stats%(c: connection, os: backdoor_endp_stats, rs: backdoor_endp_stats%); -event backdoor_remove_conn%(c: connection%); -event ssh_signature_found%(c: connection, is_orig: bool%); -event telnet_signature_found%(c: connection, is_orig: bool, len: count%); -event rlogin_signature_found%(c: connection, is_orig: bool, num_null: count, len: count%); -event root_backdoor_signature_found%(c: connection%); -event ftp_signature_found%(c: connection%); -event napster_signature_found%(c: connection%); -event gnutella_signature_found%(c: connection%); -event kazaa_signature_found%(c: connection%); -event http_signature_found%(c: connection%); -event http_proxy_signature_found%(c: connection%); -event smtp_signature_found%(c: connection%); -event irc_signature_found%(c: connection%); -event gaobot_signature_found%(c: connection%); - +## Generated for client-side commands on POP3 connections. +## +## See `Wikipedia `__ for more information about +## the POP3 protocol. +## +## c: The connection. +## +## is_orig: True if the command was sent by the originator of the TCP connection. +## +## command: The command sent. +## +## arg: The argument to the command. +## +## .. bro:see:: pop3_data pop3_login_failure pop3_login_success pop3_reply +## pop3_terminate pop3_unexpected event pop3_request%(c: connection, is_orig: bool, command: string, arg: string%); + +## Generated for server-side replies to commands on POP3 connections. +## +## See `Wikipedia `__ for more information about +## the POP3 protocol. +## +## c: The connection. +## +## is_orig: True if the command was sent by the originator of the TCP connection. +## +## cmd: The success indicator sent by the server. This corresponds to the +## first token on the line sent, and should be either ``OK`` or ``ERR``. +## +## msg: The textual description the server sent along with *cmd*. +## +## arg: The argument to the command. +## +## .. bro:see:: pop3_data pop3_login_failure pop3_login_success pop3_request +## pop3_terminate pop3_unexpected +## +## .. todo: This event is receiving odd parameters, should unify. event pop3_reply%(c: connection, is_orig: bool, cmd: string, msg: string%); + +## Generated for server-side multi-lines responses on POP3 connections. POP3 +## connection use multi-line responses to send buld data, such as the actual +## mails. This event is generated once for each line that's part of such a +## response. +## +## See `Wikipedia `__ for more information about +## the POP3 protocol. +## +## c: The connection. +## +## is_orig: True if the data was sent by the originator of the TCP connection. +## +## data: The data sent. +## +## .. bro:see:: pop3_login_failure pop3_login_success pop3_reply pop3_request +## pop3_terminate pop3_unexpected event pop3_data%(c: connection, is_orig: bool, data: string%); + +## Generated for errors encountered on POP3 sessions. If the POP3 analyzers finds +## state transition that do not confirm to the protocol specification, or other +## situations it can't handle, it raises this event. +## +## See `Wikipedia `__ for more information about +## the POP3 protocol. +## +## c: The connection. +## +## is_orig: True if the data was sent by the originator of the TCP connection. +## +## msg: A textual description of the situation. +## +## detail: The input that triggered the event. +## +## .. bro:see:: pop3_data pop3_login_failure pop3_login_success pop3_reply pop3_request +## pop3_terminate event pop3_unexpected%(c: connection, is_orig: bool, msg: string, detail: string%); + +## Generated when POP3 connection go encrypted. While POP3 is by default a +## clear-text protocol, extensions exist to switch to encryption. This event is +## generated if that happens and the analyzers then stops processing the +## connection. +## +## See `Wikipedia `__ for more information about +## the POP3 protocol. +## +## c: The connection. +## +## is_orig: Always false. +## +## msg: A descriptive message why processing was stopped. +## +## .. bro:see:: pop3_data pop3_login_failure pop3_login_success pop3_reply pop3_request +## pop3_unexpected +## +## .. note:: Currently, only the ``STARTLS`` command is recognized and +## triggers this. event pop3_terminate%(c: connection, is_orig: bool, msg: string%); + +## Generated for successful authentications on POP3 connections. +## +## See `Wikipedia `__ for more information about +## the POP3 protocol. +## +## c: The connection. +## +## is_orig: Always false. +## +## user: The user name used for authentication. The event is only generated if +## a non-empty user name was used. +## +## password: The password used for authentication. +## +## .. bro:see:: pop3_data pop3_login_failure pop3_reply pop3_request pop3_terminate +## pop3_unexpected event pop3_login_success%(c: connection, is_orig: bool, user: string, password: string%); + +## Generated for unsuccessful authentications on POP3 connections. +## +## See `Wikipedia `__ for more information about +## the POP3 protocol. +## +## c: The connection. +## +## is_orig: Always false. +## +## user: The user name attempted for authentication. The event is only generated if +## a non-empty user name was used. +## +## password: The password attempted for authentication. +## +## .. bro:see:: pop3_data pop3_login_success pop3_reply pop3_request pop3_terminate +## pop3_unexpected event pop3_login_failure%(c: connection, is_orig: bool, user: string, password: string%); + +## Generated for all client-side IRC commands. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: Always true. +## +## prefix: The optional prefix coming with the command. IRC uses the prefix to +## indicate the true origin of a message. +## +## command: The command. +## +## arguments: The arguments for the command. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_message irc_oper_response +## irc_part_message irc_password_message +## +## .. note:: This event is generated only for message that originate at the +## clients-side. Commands coming in from remote trigger the ge:bro:id:`irc_message` +## event instead. event irc_request%(c: connection, is_orig: bool, prefix: string, command: string, arguments: string%); + +## Generated for all IRC replies. IRC replies are sent in response to a +## request and come with a reply code. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## prefix: The optional prefix comming with the reply. IRC uses the prefix to +## indicate the true origin of a message. +## +## code: The reply code, as specified by the protocol. +## +## params: The reply's parameters. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_message irc_oper_response +## irc_part_message irc_password_message event irc_reply%(c: connection, is_orig: bool, prefix: string, code: count, params: string%); + +## Generated for IRC commands forwarded from the server to the client. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: Always false. +## +## prefix: The optional prefix coming with the command. IRC uses the prefix to +## indicate the true origin of a message. +## +## command: The command. +## +## arguments: The arguments for the command. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_message irc_oper_response +## irc_part_message irc_password_message +## +## .. note:: +## +## This event is generated only for messages that are forwarded by the server +## to the client. Commands coming from client trigger the :bro:id:`irc_request` +## event instead. event irc_message%(c: connection, is_orig: bool, prefix: string, command: string, message: string%); + +## Generated for IRC messages of type *quit*. This event is generated for messages +## coming from both the client and the server. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## nick: The nick name coming with the message. +## +## message: The text included with the message. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_message irc_oper_response +## irc_part_message irc_password_message event irc_quit_message%(c: connection, is_orig: bool, nick: string, message: string%); + +## Generated for IRC messages of type *privmsg*. This event is generated for messages +## coming from both the client and the server. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## source: The source of the private communication. +## +## target: The target of the private communication. +## +## message: The text of communication. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_message irc_oper_response +## irc_part_message irc_password_message event irc_privmsg_message%(c: connection, is_orig: bool, source: string, target: string, message: string%); + +## Generated for IRC messages of type *notice*. This event is generated for +## messages coming from both the client and the server. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## source: The source of the private communication. +## +## target: The target of the private communication. +## +## message: The text of communication. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_oper_message irc_oper_response irc_part_message +## irc_password_message event irc_notice_message%(c: connection, is_orig: bool, source: string, target: string, message: string%); + +## Generated for IRC messages of type *squery*. This event is generated for +## messages coming from both the client and the server. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## source: The source of the private communication. +## +## target: The target of the private communication. +## +## message: The text of communication. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_message irc_oper_response +## irc_part_message irc_password_message event irc_squery_message%(c: connection, is_orig: bool, source: string, target: string, message: string%); + +## Generated for IRC messages of type *join*. This event is generated for +## messages coming from both the client and the server. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## info_list: The user information coming with the command. +## +## message: The text of communication. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_kick_message +## irc_message irc_mode_message irc_names_info irc_network_info irc_nick_message +## irc_notice_message irc_oper_message irc_oper_response irc_part_message +## irc_password_message event irc_join_message%(c: connection, is_orig: bool, info_list: irc_join_list%); + +## Generated for IRC messages of type *part*. This event is generated for messages +## coming from both the client and the server. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## nick: The nickname coming with the message. +## +## chans: The set of channels affected. +## +## message: The text coming with the message. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_message irc_oper_response +## irc_password_message event irc_part_message%(c: connection, is_orig: bool, nick: string, chans: string_set, message: string%); + +## Generated for IRC messages of type *nick*. This event is generated for messages +## coming from both the client and the server. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## who: The user changing its nickname. +## +## newnick: The new nickname. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_notice_message irc_oper_message irc_oper_response irc_part_message +## irc_password_message event irc_nick_message%(c: connection, is_orig: bool, who: string, newnick: string%); + +## Generated when a server rejects an IRC nickname. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invite_message irc_join_message irc_kick_message +## irc_message irc_mode_message irc_names_info irc_network_info irc_nick_message +## irc_notice_message irc_oper_message irc_oper_response irc_part_message +## irc_password_message event irc_invalid_nick%(c: connection, is_orig: bool%); + +## Generated for an IRC reply of type *luserclient*. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## users: The number of users as returned in the reply. +## +## services: The number of services as returned in the reply. +## +## servers: The number of servers as returned in the reply. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_nick_message +## irc_notice_message irc_oper_message irc_oper_response irc_part_message +## irc_password_message event irc_network_info%(c: connection, is_orig: bool, users: count, services: count, servers: count%); + +## Generated for an IRC reply of type *luserme*. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## users: The number of users as returned in the reply. +## +## services: The number of services as returned in the reply. +## +## servers: The number of servers as returned in the reply. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_message irc_oper_response +## irc_part_message irc_password_message event irc_server_info%(c: connection, is_orig: bool, users: count, services: count, servers: count%); + +## Generated for an IRC reply of type *luserchannels*. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## chans: The number of channels as returned in the reply. +## +## .. bro:see:: irc_channel_topic irc_dcc_message irc_error_message irc_global_users +## irc_invalid_nick irc_invite_message irc_join_message irc_kick_message +## irc_message irc_mode_message irc_names_info irc_network_info irc_nick_message +## irc_notice_message irc_oper_message irc_oper_response irc_part_message +## irc_password_message event irc_channel_info%(c: connection, is_orig: bool, chans: count%); + +## Generated for an IRC reply of type *whoreply*. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## target_nick: The target nick name. +## +## channel: The channel. +## +## user: The user. +## +## host: The host. +## +## server: The server. +## +## nick: The nick name. +## +## params: The parameters. +## +## hops: The hop count. +## +## real_name: The real name. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_message irc_oper_response +## irc_part_message irc_password_message event irc_who_line%(c: connection, is_orig: bool, target_nick: string, channel: string, user: string, host: string, server: string, nick: string, params: string, hops: count, real_name: string%); -event irc_who_message%(c: connection, is_orig: bool, mask: string, oper: bool%); -event irc_whois_message%(c: connection, is_orig: bool, server: string, users: string%); -event irc_whois_user_line%(c: connection, is_orig: bool, nick: string, - user: string, host: string, real_name: string%); -event irc_whois_operator_line%(c: connection, is_orig: bool, nick: string%); -event irc_whois_channel_line%(c: connection, is_orig: bool, nick: string, - chans: string_set%); -event irc_oper_message%(c: connection, is_orig: bool, user: string, password: string%); -event irc_oper_response%(c: connection, is_orig: bool, got_oper: bool%); -event irc_kick_message%(c: connection, is_orig: bool, prefix: string, - chans: string, users: string, comment: string%); -event irc_error_message%(c: connection, is_orig: bool, prefix: string, message: string%); -event irc_invite_message%(c: connection, is_orig: bool, prefix: string, - nickname: string, channel: string%); -event irc_mode_message%(c: connection, is_orig: bool, prefix: string, params: string%); -event irc_squit_message%(c: connection, is_orig: bool, prefix: string, - server: string, message: string%); + + +## Generated for an IRC reply of type *namereply*. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## c_type: The channel type. +## +## channel: The channel. +## +## users: The set of users. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_network_info irc_nick_message +## irc_notice_message irc_oper_message irc_oper_response irc_part_message +## irc_password_message event irc_names_info%(c: connection, is_orig: bool, c_type: string, channel: string, users: string_set%); + +## Generated for an IRC reply of type *whoisoperator*. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## nick: The nick name specified in the reply. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_message irc_oper_response +## irc_part_message irc_password_message +event irc_whois_operator_line%(c: connection, is_orig: bool, nick: string%); + +## Generated for an IRC reply of type *whoischannels*. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## nick: The nick name specified in the reply. +## +## chans: The set of channels returned. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_message irc_oper_response +## irc_part_message irc_password_message +event irc_whois_channel_line%(c: connection, is_orig: bool, nick: string, + chans: string_set%); + +## Generated for an IRC reply of type *whoisuser*. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## nick: The nick name specified in the reply. +## +## user: The user name specified in the reply. +## +## host: The host name specified in the reply. +## +## user: The user name specified in the reply. +## +## real_name: The real name specified in the reply. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_message irc_oper_response +## irc_part_message irc_password_message +event irc_whois_user_line%(c: connection, is_orig: bool, nick: string, + user: string, host: string, real_name: string%); + +## Generated for IRC replies of type *youreoper* and *nooperhost*. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## got_oper: True if the *oper* command was executed successfully +## (*youreport*) and false otherwise (*nooperhost*). +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_message irc_part_message +## irc_password_message +event irc_oper_response%(c: connection, is_orig: bool, got_oper: bool%); + +## Generated for an IRC reply of type *globalusers*. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## prefix: The optional prefix coming with the command. IRC uses the prefix to +## indicate the true origin of a message. +## +## msg: The message coming with the reply. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_invalid_nick irc_invite_message irc_join_message irc_kick_message +## irc_message irc_mode_message irc_names_info irc_network_info irc_nick_message +## irc_notice_message irc_oper_message irc_oper_response irc_part_message +## irc_password_message +event irc_global_users%(c: connection, is_orig: bool, prefix: string, msg: string%); + +## Generated for an IRC reply of type *topic*. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## channel: The channel name specified in the reply. +## +## topic: The topic specified in the reply. +## +## .. bro:see:: irc_channel_info irc_dcc_message irc_error_message irc_global_users +## irc_invalid_nick irc_invite_message irc_join_message irc_kick_message +## irc_message irc_mode_message irc_names_info irc_network_info irc_nick_message +## irc_notice_message irc_oper_message irc_oper_response irc_part_message +## irc_password_message +event irc_channel_topic%(c: connection, is_orig: bool, channel: string, topic: string%); + +## Generated for IRC messages of type *who*. This event is generated for messages +## coming from both the client and the server. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## is_orig: True if the command what sent by the originator of the TCP connection. +## +## mask: The mask specified in the message. +## +## oper: True if the operator flag was set. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_message irc_oper_response +## irc_part_message irc_password_message +event irc_who_message%(c: connection, is_orig: bool, mask: string, oper: bool%); + +## Generated for IRC messages of type *whois*. This event is generated for messages +## coming from both the client and the server. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_message irc_oper_response +## irc_part_message irc_password_message +event irc_whois_message%(c: connection, is_orig: bool, server: string, users: string%); + +## Generated for IRC messages of type *oper*. This event is generated for messages +## coming from both the client and the server. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## user: The user specified in the message. +## +## password: The password specified in the message. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_response irc_part_message +## irc_password_message +event irc_oper_message%(c: connection, is_orig: bool, user: string, password: string%); + +## Generated for IRC messages of type *kick*. This event is generated for messages +## coming from both the client and the server. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## prefix: The optional prefix coming with the command. IRC uses the prefix to +## indicate the true origin of a message. +## +## chans: The channels specified in the message. +## +## users: The users specified in the message. +## +## comment: The comment specified in the message. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_message irc_mode_message irc_names_info irc_network_info irc_nick_message +## irc_notice_message irc_oper_message irc_oper_response irc_part_message +## irc_password_message +event irc_kick_message%(c: connection, is_orig: bool, prefix: string, + chans: string, users: string, comment: string%); + +## Generated for IRC messages of type *error*. This event is generated for messages +## coming from both the client and the server. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## prefix: The optional prefix coming with the command. IRC uses the prefix to +## indicate the true origin of a message. +## +## message: The textual description specified in the message. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_global_users +## irc_invalid_nick irc_invite_message irc_join_message irc_kick_message +## irc_message irc_mode_message irc_names_info irc_network_info irc_nick_message +## irc_notice_message irc_oper_message irc_oper_response irc_part_message +## irc_password_message +event irc_error_message%(c: connection, is_orig: bool, prefix: string, message: string%); + +## Generated for IRC messages of type *invite*. This event is generated for +## messages coming from both the client and the server. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## prefix: The optional prefix coming with the command. IRC uses the prefix to +## indicate the true origin of a message. +## +## nickname: The nick name specified in the message. +## +## channel: The channel specified in the message. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_join_message irc_kick_message +## irc_message irc_mode_message irc_names_info irc_network_info irc_nick_message +## irc_notice_message irc_oper_message irc_oper_response irc_part_message +## irc_password_message +event irc_invite_message%(c: connection, is_orig: bool, prefix: string, + nickname: string, channel: string%); + +## Generated for IRC messages of type *mode*. This event is generated for messages +## coming from both the client and the server. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## prefix: The optional prefix coming with the command. IRC uses the prefix to +## indicate the true origin of a message. +## +## params: The parameters coming with the message. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_names_info irc_network_info irc_nick_message +## irc_notice_message irc_oper_message irc_oper_response irc_part_message +## irc_password_message +event irc_mode_message%(c: connection, is_orig: bool, prefix: string, params: string%); + +## Generated for IRC messages of type *squit*. This event is generated for messages +## coming from both the client and the server. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## prefix: The optional prefix coming with the command. IRC uses the prefix to +## indicate the true origin of a message. +## +## server: The server specified in the message. +## +## messate: The textual description specified in the message. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_message irc_oper_response +## irc_part_message irc_password_message +event irc_squit_message%(c: connection, is_orig: bool, prefix: string, + server: string, message: string%); + +## Generated for IRC messages of type *dcc*. This event is generated for messages +## coming from both the client and the server. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## prefix: The optional prefix coming with the command. IRC uses the prefix to +## indicate the true origin of a message. +## +## target: The target specified in the message. +## +## dcc_type: The DCC type specified in the message. +## +## argument: The argument specified in the message. +## +## address: The address specified in the message. +## +## dest_port: The destination port specified in the message. +## +## size: The size specified in the message. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_error_message irc_global_users +## irc_invalid_nick irc_invite_message irc_join_message irc_kick_message +## irc_message irc_mode_message irc_names_info irc_network_info irc_nick_message +## irc_notice_message irc_oper_message irc_oper_response irc_part_message +## irc_password_message event irc_dcc_message%(c: connection, is_orig: bool, prefix: string, target: string, dcc_type: string, argument: string, address: addr, dest_port: count, size: count%); -event irc_global_users%(c: connection, is_orig: bool, prefix: string, msg: string%); + +## Generated for IRC messages of type *user*. This event is generated for messages +## coming from both the client and the server. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## user: The user specified in the message. +## +## host: The host name specified in the message. +## +## server: The server name specified in the message. +## +## real_name: The real name specified in the message. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_message irc_oper_response +## irc_part_message irc_password_message event irc_user_message%(c: connection, is_orig: bool, user: string, host: string, server: string, real_name: string%); -event irc_channel_topic%(c: connection, is_orig: bool, channel: string, topic: string%); + +## Generated for IRC messages of type *password*. This event is generated for +## messages coming from both the client and the server. +## +## See `Wikipedia `__ for more +## information about the IRC protocol. +## +## c: The connection. +## +## password: The password specified in the message. +## +## .. bro:see:: irc_channel_info irc_channel_topic irc_dcc_message irc_error_message +## irc_global_users irc_invalid_nick irc_invite_message irc_join_message +## irc_kick_message irc_message irc_mode_message irc_names_info irc_network_info +## irc_nick_message irc_notice_message irc_oper_message irc_oper_response +## irc_part_message event irc_password_message%(c: connection, is_orig: bool, password: string%); +## TODO. +## +## .. bro:see:: event file_transferred%(c: connection, prefix: string, descr: string, mime_type: string%); -event file_virus%(c: connection, virname: string%); +## TODO. event syslog_message%(c: connection, facility: count, severity: count, msg: string%); +## TODO. event signature_match%(state: signature_state, msg: string, data: string%); -# Generated if a handler finds an identification of the software -# used on a system. +## Generated if a handler finds an identification of the software +## used on a system. TODO +## +## .. bro:see:: software_parse_error software_unparsed_version_found event software_version_found%(c: connection, host: addr, s: software, descr: string%); -# Generated if a handler finds a version but cannot parse it. +## Generated if a handler finds a version but cannot parse it. TODO +## +## .. bro:see:: software_version_found software_unparsed_version_found event software_parse_error%(c: connection, host: addr, descr: string%); -# Generated once for each raw (unparsed) software identification. +## Generated once for each raw (unparsed) software identification. TODO +## +## .. bro:see:: software_parse_error software_version_found event software_unparsed_version_found%(c: connection, host: addr, str: string%); -# Generated when an operating system has been fingerprinted. +## Generated when an operating system has been fingerprinted. TODO event OS_version_found%(c: connection, host: addr, OS: OS_version%); -# Generated when an IP address gets mapped for the first time. -event anonymization_mapping%(orig: addr, mapped: addr%); - -# Generated when a connection to a remote Bro has been established. +## Generated when a connection to a remote Bro has been established. This event +## is intended primarily for use by Bro's communication framework, but it can also +## trigger additional code if helpful. +## +## p: A record describing the peer. +## +## .. bro:see:: remote_capture_filter remote_connection_closed remote_connection_error +## remote_connection_handshake_done remote_event_registered remote_log remote_pong +## remote_state_access_performed remote_state_inconsistency print_hook event remote_connection_established%(p: event_peer%); -# Generated when a connection to a remote Bro has been closed. +## Generated when a connection to a remote Bro has been closed. This event is +## intended primarily for use by Bro's communication framework, but it can +## also trigger additional code if helpful. +## +## p: A record describing the peer. +## +## .. bro:see:: remote_capture_filter remote_connection_error +## remote_connection_established remote_connection_handshake_done +## remote_event_registered remote_log remote_pong remote_state_access_performed +## remote_state_inconsistency print_hook event remote_connection_closed%(p: event_peer%); -# Generated when a remote connection's handshake has been completed. +## Generated when a remote connection's initial handshake has been completed. This +## event is intended primarily for use by Bro's communication framework, but it can +## also trigger additional code if helpful. +## +## p: A record describing the peer. +## +## .. bro:see:: remote_capture_filter remote_connection_closed remote_connection_error +## remote_connection_established remote_event_registered remote_log remote_pong +## remote_state_access_performed remote_state_inconsistency print_hook event remote_connection_handshake_done%(p: event_peer%); -# Generated for each event registered by a remote peer. +## Generated for each event registered by a remote peer. This event is intended +## primarily for use by Bro's communication framework, but it can also trigger +## additional code if helpful. +## +## p: A record describing the peer. +## +## .. bro:see:: remote_capture_filter remote_connection_closed +## remote_connection_error remote_connection_established +## remote_connection_handshake_done remote_log remote_pong +## remote_state_access_performed remote_state_inconsistency print_hook event remote_event_registered%(p: event_peer, name: string%); -# Generated when a connection to a remote Bro causes some error. +## Generated when a connection to a remote Bro encountered an error. This event +## is intended primarily for use by Bro's communication framework, but it can also +## trigger additional code if helpful. +## +## p: A record describing the peer. +## +## reason: A textual description of the error. +## +## .. bro:see:: remote_capture_filter remote_connection_closed +## remote_connection_established remote_connection_handshake_done +## remote_event_registered remote_log remote_pong remote_state_access_performed +## remote_state_inconsistency print_hook event remote_connection_error%(p: event_peer, reason: string%); -# Generated when a remote peer sends us some capture filter. + + +## Generated when a remote peer sent us a capture filter. While this event is +## intended primarily for use by Bro's communication framework, it can also trigger +## additional code if helpful. +## +## p: A record describing the peer. +## +## filter: The filter string sent by the peer. +## +## .. bro:see:: remote_connection_closed remote_connection_error +## remote_connection_established remote_connection_handshake_done +## remote_event_registered remote_log remote_pong remote_state_access_performed +## remote_state_inconsistency print_hook event remote_capture_filter%(p: event_peer, filter: string%); -# Generated after a call to send_state() when all data has been successfully -# sent to the remote side. +## Generated after a call to :bro:id:`send_state` when all data has been +## successfully sent to the remote side. While this event is +## intended primarily for use by Bro's communication framework, it can also trigger +## additional code if helpful. +## +## p: A record describing the remote peer. +## +## .. bro:see:: remote_capture_filter remote_connection_closed +## remote_connection_error remote_connection_established +## remote_connection_handshake_done remote_event_registered remote_log remote_pong +## remote_state_access_performed remote_state_inconsistency print_hook event finished_send_state%(p: event_peer%); -# Generated if state synchronization detects an inconsistency. +## Generated if state synchronization detects an inconsistency. While this event +## is intended primarily for use by Bro's communication framework, it can also +## trigger additional code if helpful. This event is only raised if +## :bro:id:`remote_check_sync_consistency` is false. +## +## operation: The textual description of the state operation performed. +## +## id: The name of the Bro script identifier that was operated on. +## +## expected_old: A textual representation of the value of *id* that was expected to +## be found before the operation was carried out. +## +## real_old: A textual representation of the value of *id* that was actually found +## before the operation was carried out. The difference between +## *real_old* and *expected_old* is the inconsistency being reported. +## +## .. bro:see:: remote_capture_filter remote_connection_closed +## remote_connection_error remote_connection_established +## remote_connection_handshake_done remote_event_registered remote_log remote_pong +## remote_state_access_performed print_hook remote_check_sync_consistency event remote_state_inconsistency%(operation: string, id: string, expected_old: string, real_old: string%); -# Generated for communication log message. +## Generated for communication log messages. While this event is +## intended primarily for use by Bro's communication framework, it can also trigger +## additional code if helpful. +## +## level: The log level, which is either :bro:enum:`REMOTE_LOG_INFO` or +## :bro:enum:`REMOTE_LOG_ERROR`. +## +## src: The component of the comminication system that logged the message. +## Currently, this will be one of :bro:enum:`REMOTE_SRC_CHILD` (Bro's +## child process), :bro:enum:`REMOTE_SRC_PARENT` (Bro's main process), or +## :bro:enum:`REMOTE_SRC_SCRIPT` (the script level). +## +## msg: The message logged. +## +## .. bro:see:: remote_capture_filter remote_connection_closed remote_connection_error +## remote_connection_established remote_connection_handshake_done +## remote_event_registered remote_pong remote_state_access_performed +## remote_state_inconsistency print_hook event remote_log%(level: count, src: count, msg: string%); -# Generated when a remote peer has answered to our ping. +## Generated when a remote peer has answered to our ping. This event is part of +## Bro's infrastructure for measuring communication latency. One can send a ping +## by calling :bro:id:`send_ping` and when a corresponding reply is received, this +## event will be raised. +## +## p: The peer sending us the pong. +## +## seq: The sequence number passed to the original :bro:id:`send_ping` call. +## The number is sent back by the peer in its response. +## +## d1: The time interval between sending the ping and receiving the pong. This +## is the latency of the complete path. +## +## d2: The time interval between sending out the ping to the network and its +## reception at the peer. This is the network latency. +## +## d3: The time interval between when the peer's child process received the +## ping and when its parent process sent the pong. This is the +## processing latency at the the peer. +## +## .. bro:see:: remote_capture_filter remote_connection_closed remote_connection_error +## remote_connection_established remote_connection_handshake_done +## remote_event_registered remote_log remote_state_access_performed +## remote_state_inconsistency print_hook event remote_pong%(p: event_peer, seq: count, d1: interval, d2: interval, d3: interval%); -# Generated each time a remote state access has been replayed locally -# (primarily for debugging). +## Generated each time a remote state access has been replayed locally. This event +## is primarily intended for debugging. measurments. +## +## id: The name of the Bro script variable that's being operated on. +## +## v: The new value of the variable. +## +## .. bro:see:: remote_capture_filter remote_connection_closed remote_connection_error +## remote_connection_established remote_connection_handshake_done +## remote_event_registered remote_log remote_pong remote_state_inconsistency +## print_hook event remote_state_access_performed%(id: string, v: any%); -# Generated each time profiling_file is updated. "expensive" means that -# this event corresponds to heavier-weight profiling as indicated by the -# expensive_profiling_multiple variable. +## Generated each time Bro's internal profiling log is updated. The file is +## defined by :bro:id:`profiling_file`, and its update frequency by +## :bro:id:`profiling_interval` and :bro:id:`expensive_profiling_multiple`. +## +## f: The profiling file. +## +## expensive: True if this event corresponds to heavier-weight profiling as +## indicated by the :bro:enum:`expensive_profiling_multiple` variable. +## +## .. bro:see:: profiling_interval expensive_profiling_multiple event profiling_update%(f: file, expensive: bool%); +## Generated each time Bro's script interpreter opens a file. This event is +## triggered only for files opened via :bro:id:`open`, and in particular not for +## normal log files as created by a log writers. +## +## f: The opened file. event file_opened%(f: file%); -# Each print statement generates an event. -event print_hook%(f:file, s: string%); - -# Generated for &rotate_interval. -event rotate_interval%(f: file%); - -# Generated for &rotate_size. -event rotate_size%(f: file%); - +## Generated for a received NetFlow v5 header. Bro's NetFlow processor raises this +## event whenever it either receives a NetFlow header on the port it's listening +## on, or reads one from a trace file. +## +## h: The parsed NetFlow header. +## +## .. bro:see:: netflow_v5_record event netflow_v5_header%(h: nf_v5_header%); + +## Generated for a received NetFlow v5 record. Bro's NetFlow processor raises this +## event whenever it either receives a NetFlow record on the port it's listening +## on, or reads one from a trace file. +## +## h: The parsed NetFlow header. +## +## .. bro:see:: netflow_v5_record event netflow_v5_record%(r: nf_v5_record%); -# Different types of reporter messages. These won't be called -# recursively. +## Raised for informational messages reported via Bro's reporter framework. Such +## messages may be generated internally by the event engine and also by other +## scripts calling :bro:id:`Reporter::info`. +## +## t: The time the message was passed to the reporter. +## +## msg: The message itself. +## +## location: A (potentially empty) string describing a location associated with the +## message. +## +## .. bro:see:: reporter_warning reporter_error Reporter::info Reporter::warning +## Reporter::error +## +## .. note:: Bro will not call reporter events recursively. If the handler of any +## reporter event triggers a new reporter message itself, the output will go to +## ``stderr`` instead. event reporter_info%(t: time, msg: string, location: string%) &error_handler; + +## Raised for warnings reported via Bro's reporter framework. Such messages may +## be generated internally by the event engine and also by other scripts calling +## :bro:id:`Reporter::warning`. +## +## t: The time the warning was passed to the reporter. +## +## msg: The warning message. +## +## location: A (potentially empty) string describing a location associated with the +## warning. +## +## .. bro:see:: reporter_warning reporter_error Reporter::info Reporter::warning +## Reporter::error +## +## .. note:: Bro will not call reporter events recursively. If the handler of any +## reporter event triggers a new reporter message itself, the output will go to +## ``stderr`` instead. event reporter_warning%(t: time, msg: string, location: string%) &error_handler; + +## Raised for errors reported via Bro's reporter framework. Such messages may +## be generated internally by the event engine and also by other scripts calling +## :bro:id:`Reporter::error`. +## +## t: The time the error was passed to the reporter. +## +## msg: The error message. +## +## location: A (potentially empty) string describing a location associated with the +## error. +## +## .. bro:see:: reporter_error reporter_error Reporter::info Reporter::error +## Reporter::error +## +## .. note:: Bro will not call reporter events recursively. If the handler of any +## reporter event triggers a new reporter message itself, the output will go to +## ``stderr`` instead. event reporter_error%(t: time, msg: string, location: string%) &error_handler; -# Raised for each policy script loaded. +## Raised for each policy script loaded by the script interpreter. +## +## path: The full path to the script loaded. +## +## level: The "nesting level": zero for a top-level Bro script and incremented +## recursively for each ``@load``. event bro_script_loaded%(path: string, level: count%); + +# ##### Deprecated events. Proposed for removal. + +## Deprecated. Will be removed. +event stp_create_endp%(c: connection, e: int, is_orig: bool%); + +## Deprecated. Will be removed. +event stp_resume_endp%(e: int%); + +## Deprecated. Will be removed. +event stp_correlate_pair%(e1: int, e2: int%); + +## Deprecated. Will be removed. +event stp_remove_pair%(e1: int, e2: int%); + +## Deprecated. Will be removed. +event stp_remove_endp%(e: int%); + +## Deprecated. Will be removed. +event interconn_stats%(c: connection, os: interconn_endp_stats, rs: interconn_endp_stats%); + +## Deprecated. Will be removed. +event interconn_remove_conn%(c: connection%); + +## Deprecated. Will be removed. +event backdoor_stats%(c: connection, os: backdoor_endp_stats, rs: backdoor_endp_stats%); + +## Deprecated. Will be removed. +event backdoor_remove_conn%(c: connection%); + +## Deprecated. Will be removed. +event ssh_signature_found%(c: connection, is_orig: bool%); + +## Deprecated. Will be removed. +event telnet_signature_found%(c: connection, is_orig: bool, len: count%); + +## Deprecated. Will be removed. +event rlogin_signature_found%(c: connection, is_orig: bool, num_null: count, len: count%); + +## Deprecated. Will be removed. +event root_backdoor_signature_found%(c: connection%); + +## Deprecated. Will be removed. +event ftp_signature_found%(c: connection%); + +## Deprecated. Will be removed. +event napster_signature_found%(c: connection%); + +## Deprecated. Will be removed. +event gnutella_signature_found%(c: connection%); + +## Deprecated. Will be removed. +event kazaa_signature_found%(c: connection%); + +## Deprecated. Will be removed. +event http_signature_found%(c: connection%); + +## Deprecated. Will be removed. +event http_proxy_signature_found%(c: connection%); + +## Deprecated. Will be removed. +event smtp_signature_found%(c: connection%); + +## Deprecated. Will be removed. +event irc_signature_found%(c: connection%); + +## Deprecated. Will be removed. +event gaobot_signature_found%(c: connection%); + +## Deprecated. Will be removed. +## +## .. todo:: Unclear what this event is for; it's never raised. We should just +## remove it. +event dns_full_request%(%) &group="dns"; + +## Deprecated. Will be removed. +event anonymization_mapping%(orig: addr, mapped: addr%); + +## Deprecated. Will be removed. +event rotate_interval%(f: file%); + +## Deprecated. Will be removed. +event rotate_size%(f: file%); + +## Deprecated. Will be removed. +event print_hook%(f:file, s: string%); From 0523a18a2381fb67953cbce07eb5455db6a95c19 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 29 Nov 2011 10:10:46 -0800 Subject: [PATCH 104/964] Almost done with event.bif. Added comments to almost all events. The only ones I'm leaving out are Gnutella and BitTorrent, don't know enough about those protocols to document the events. If anybody does, please chime in. A round of proofreading would be good too, there are probably still a few typos in the texts. --- src/event.bif | 381 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 305 insertions(+), 76 deletions(-) diff --git a/src/event.bif b/src/event.bif index a8d858de57..d52a16442d 100644 --- a/src/event.bif +++ b/src/event.bif @@ -6,14 +6,14 @@ # - List parameters with an empty line in between. # # - Within the description, reference other parameters of the same events -# with *arg*. +# as *arg*. # # - Order: # # - Short initial sentence (which doesn't need to be a sentence), -# starting with "Generated ..." +# starting with "Generated ..." # -# - Description. +# - Description # # - Parameters # @@ -22,14 +22,6 @@ # - .. note:: # # - .. todo:: -# -# Templates: -# -# See `Wikipedia `__ for more -# information about the XXX protocol. -# -# See `RFC XXX `__ for more information about -# the XXX protocol. ## Generated at Bro initialization time. The event engine generates this ## event just before normal input processing begins. It can be used to execute @@ -61,32 +53,64 @@ event bro_init%(%); ## not generated. event bro_done%(%); -## TODO. +## Generated when an internal DNS lookup reduces the same result as last time. +## Bro keeps an internal DNS cache for host names and IP addresses it has +## already resolved. This event is generated when subsequent lookup returns +## the same result as stored in the cache. +## +## dm: A record describing the new resolver result (which matches the old one). ## ## .. bro:see:: dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified event dns_mapping_valid%(dm: dns_mapping%); -## TODO. +## Generated when an internal DNS lookup got no answer even though it had succeeded he +## past. Bro keeps an internal DNS cache for host names and IP addresses it has +## already resolved. This event is generated when a subsequent lookup does not +## produce an answer even though we have already stored a result in the cache. +## +## dm: A record describing the old resolver result. ## ## .. bro:see:: dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_valid event dns_mapping_unverified%(dm: dns_mapping%); -## TODO. -## +## Generated when an internal DNS lookup succeeed but an earlier attempt not. had +## had succeeded he past. Bro keeps an internal DNS cache for host names and IP +## addresses it has already resolved. This event is generated when a subsequent +## lookup produces an answer for a query that was marked as failed in the cache. +## +## dm: A record describing the new resolver result. +## ## .. bro:see:: dns_mapping_altered dns_mapping_lost_name dns_mapping_unverified ## dns_mapping_valid event dns_mapping_new_name%(dm: dns_mapping%); -## TODO. +## Generated when an internal DNS lookup returned zero answers even though it +## had succeeded he past. Bro keeps an internal DNS cache for host names and IP +## addresses it has already resolved. This event is generated when for a subsequent +## lookup we received answer that however was empty even though we have +## already stored a result in the cache. +## +## dm: A record describing the old resolver result. ## ## .. bro:see:: dns_mapping_altered dns_mapping_new_name dns_mapping_unverified ## dns_mapping_valid event dns_mapping_lost_name%(dm: dns_mapping%); -## TODO. -## +## Generated when an internal DNS lookup produced a different result than in +## past. Bro keeps an internal DNS cache for host names and IP addresses it has +## already resolved. This event is generated when a subsequent lookup returns +## a different answer than we have stored in the cache. +## +## dm: A record describing the new resolver result. +## +## old_addrs: Addresses that used to be part of the returned set for the query +## described by *dm*, but are not anymore. +## +## new_addrs: Addresses that did not use to be part of the returned set for the +## query described by *dm*, but now are. +## ## .. bro:see:: dns_mapping_lost_name dns_mapping_new_name dns_mapping_unverified ## dns_mapping_valid event dns_mapping_altered%(dm: dns_mapping, old_addrs: addr_set, new_addrs: addr_set%); @@ -170,7 +194,13 @@ event connection_established%(c: connection%); ## event partial_connection%(c: connection%); -## TODO. +## Generated when a previously inactive endpoint attempts to close a TCP connection +## via a normal FIN handshake or an abort RST sequence. When the endpoint sent +## one of these packets, Bro waits :bro:id:`tcp_partial_close_delay` prior +## to generating the event, to give the other endpoint a chance to close the +## connection normally. +## +## c: The connection. ## ## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt ## connection_established connection_external connection_finished @@ -193,7 +223,11 @@ event connection_partial_close%(c: connection%); ## new_connection new_connection_contents partial_connection event connection_finished%(c: connection%); -## TODO. +## Generated when one endpoint of a TCP connection attempted to gracefully close +## the connection, but the other endpoint is in the TCP_INACTIVE state. This can +## happen due to split routing, in which Bro only sees one side of a connection. +## +## c: The connection. ## ## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt ## connection_established connection_external connection_finished @@ -239,14 +273,16 @@ event connection_rejected%(c: connection%); ## partial_connection event connection_reset%(c: connection%); -## TODO +## Generated for each still-open connection when Bro terminates. +## +## c: The connection. ## ## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt ## connection_established connection_external connection_finished ## connection_first_ACK connection_half_finished connection_partial_close ## connection_rejected connection_reset connection_reused connection_state_remove ## connection_status_update connection_timeout expected_connection_seen -## new_connection new_connection_contents partial_connection +## new_connection new_connection_contents partial_connection bro_done event connection_pending%(c: connection%); ## Generated when a connection's internal state is about to be removed from @@ -264,7 +300,7 @@ event connection_pending%(c: connection%); ## connection_pending connection_rejected connection_reset connection_reused ## connection_status_update connection_timeout expected_connection_seen ## new_connection new_connection_contents partial_connection udp_inactivity_timeout -## tcp_inactivity_timeout icmp_inactivity_timeout +## tcp_inactivity_timeout icmp_inactivity_timeout conn_stats event connection_state_remove%(c: connection%); ## Generated for a SYN packet. Bro raises this event for every SYN packet seen by @@ -377,9 +413,9 @@ event connection_status_update%(c: connection%); event connection_EOF%(c: connection, is_orig: bool%); ## Generated for a new connection received from the communication subsystem. Remote -## peers can inject packets into Bro's packet loop, for example via TODO_Broccoli. -## The communication systems raises this event with the first packet of a -## connection coming in this way. +## peers can inject packets into Bro's packet loop, for example via :doc:`Broccoli +## `. The communication systems raises this event +## with the first packet of a connection coming in this way. event connection_external%(c: connection, tag: string%); ## Generated when a connected is seen that has previously marked as being expected. @@ -392,7 +428,7 @@ event connection_external%(c: connection, tag: string%); ## a: The analyzer that was scheduled for the connection with the ## :bro:id:`expect_connection` call. When the event is raised, that ## analyzer will already have been activated to process the connection. The -## ``count`` is one of the ``ANALYZER_*`` constants, see TODO. +## ``count`` is one of the ``ANALYZER_*`` constants, e.g., ``ANALYZER_HTTP``. ## ## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt ## connection_established connection_external connection_finished @@ -400,6 +436,9 @@ event connection_external%(c: connection, tag: string%); ## connection_pending connection_rejected connection_reset connection_reused ## connection_state_remove connection_status_update connection_timeout ## new_connection new_connection_contents partial_connection +## +## .. todo: We don't have a good way to document the automatically generated +## ``ANALYZER_*`` constants right now. event expected_connection_seen%(c: connection, a: count%); ## Generated for every packet Bro sees. This is a very low-level and expensive @@ -473,15 +512,14 @@ event tcp_packet%(c: connection, is_orig: bool, flags: string, seq: count, ack: ## .. note:: There is currently no way to get the actual option value, if any. event tcp_option%(c: connection, is_orig: bool, opt: count, optlen: count%); -## Generated for each chunk of reassembled TCP payload. When reassembly is enabled -## for a TCP connection (as it is automatically whenever there is TCP protocol -## analyzer parsing it), this event is raised for each chunk of in-order payload -## reconstructed from the packet stream. Note that this event is potentially -## expensive if many connections carry signficant amounts of data as then all -## that needs to be passed on to the scripting layer. -## -## TODO: add link to options (or discussionof options) for enabling TCP -## reaseembly in other ways. +## Generated for each chunk of reassembled TCP payload. When content delivery is +## enabled for a TCP connection (via :bro:id:`tcp_content_delivery_ports_orig`, +## :bro:id:`tcp_content_delivery_ports_resp`, +## :bro:id:`tcp_content_delivery_all_orig`, +## :bro:id:`tcp_content_delivery_all_resp`), this event is raised for each chunk +## of in-order payload reconstructed from the packet stream. Note that this event +## is potentially expensive if many connections carry signficant amounts of data as +## then all that needs to be passed on to the scripting layer. ## ## c: The connection the payload is part of. ## @@ -492,18 +530,18 @@ event tcp_option%(c: connection, is_orig: bool, opt: count, optlen: count%); ## ## payload: The raw payload, which will be non-empty. ## -## .. bro:see:: tcp_packet tcp_option tcp_rexmit -## +## .. bro:see:: tcp_packet tcp_option tcp_rexmit +## tcp_content_delivery_ports_orig tcp_content_delivery_ports_resp +## tcp_content_deliver_all_resp tcp_content_deliver_all_orig +## ## .. note:: ## ## The payload received by this event is the same that is also passed into ## application-layer protocol analyzers internally. Subsequent invocations of ## this event for the same connection receive non-overlapping in-order chunks ## of its TCP payload stream. It is however undefined what size each chunk -## has; while Bro passes the data on as soon as possible, specifics depend -## on network-level effects such as latency, acknowledgements, reordering, -## etc. Also note that Bro's reassembler holds on to any data until it has seen -## the other side's acknowledgement. +## has; while Bro passes the data on as soon as possible, specifics depend on +## network-level effects such as latency, acknowledgements, reordering, etc. event tcp_contents%(c: connection, is_orig: bool, seq: count, contents: string%); ## Generated @@ -589,18 +627,17 @@ event content_gap%(c: connection, is_orig: bool, seq: count, length: count%); ## ## .. note:: ## -## Bro comes with a script :doc:`/scripts/policy/misc/capture-loss.bro` that -## uses this event to estimate packet loss and report when a predefined -## threshold is exceeded. +## Bro comes with a script :doc:`/scripts/policy/misc/capture-loss` that uses +## this event to estimate packet loss and report when a predefined threshold is +## exceeded. event gap_report%(dt: interval, info: gap_info%); ## Generated when a protocol analyzer confirms that a connection is indeed -## using that protocol. Bro's TODO_dynamic protocol detection heuristically -## activates analyzers as soon as it believe a connection *could* be using a -## particular protocol. It is then left to the corresponding analyzer to -## verify whether that is indeed the case; if so, this event will be -## generated. +## using that protocol. Bro's dynamic protocol detection heuristically activates +## analyzers as soon as it believe a connection *could* be using a particular +## protocol. It is then left to the corresponding analyzer to verify whether that +## is indeed the case; if so, this event will be generated. ## ## c: The connection. ## @@ -619,15 +656,16 @@ event gap_report%(dt: interval, info: gap_info%); ## .. note:: ## ## Bro's default scripts use this event to determine the ``service`` column of -## TODO``conn.log``: once confirmed, the protocol will be listed there. +## :bro:id:`Conn::Info`: once confirmed, the protocol will be listed there (and +## thus in ``conn.log``). event protocol_confirmation%(c: connection, atype: count, aid: count%); -## Generated when a protocol analyzer determines that a connection it is parsing is -## not conforming to the protocol it expects. Bro's TODO_dynamic protocol detection +## Generated when a protocol analyzer determines that a connection it is parsing +## is not conforming to the protocol it expects. Bro's dynamic protocol detection ## heuristically activates analyzers as soon as it believe a connection *could* be ## using a particular protocol. It is then left to the corresponding analyzer to -## verify whether that is indeed the case; if not, the analyzer will -## trigger this event. +## verify whether that is indeed the case; if not, the analyzer will trigger this +## event. ## ## c: The connection. ## @@ -818,7 +856,17 @@ event icmp_time_exceeded%(c: connection, icmp: icmp_conn, code: count, context: ## icmp_time_exceeded icmp_unreachable event icmp_redirect%(c: connection, icmp: icmp_conn, a: addr%); -## TODO. +## Generated when a TCP connection terminated, passing on statistics about the +## two endpoints. This event is generated when Bro flushes the internal connection +## state, independent of how the connection gad terminated. +## +## c: The connection. +## +## os: Statistics for the originator endpoint. +## +## rs: Statistics for the responder endpoint. +## +## .. bro:see:: connection_state_remove event conn_stats%(c: connection, os: endpoint_stats, rs: endpoint_stats%); ## Generated for unexpected activity related to a specific connection. When @@ -956,6 +1004,9 @@ event arp_reply%(mac_src: string, mac_dst: string, SPA: addr, SHA: string, event bad_arp%(SPA: addr, SHA: string, TPA: addr, THA: string, explanation: string%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the BitTorrent protocol. ## ## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke ## bittorrent_peer_have bittorrent_peer_interested bittorrent_peer_keep_alive @@ -966,6 +1017,9 @@ event bittorrent_peer_handshake%(c: connection, is_orig: bool, reserved: string, info_hash: string, peer_id: string%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the BitTorrent protocol. ## ## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke ## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested @@ -975,6 +1029,9 @@ event bittorrent_peer_handshake%(c: connection, is_orig: bool, event bittorrent_peer_keep_alive%(c: connection, is_orig: bool%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the BitTorrent protocol. ## ## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel ## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested @@ -984,6 +1041,9 @@ event bittorrent_peer_keep_alive%(c: connection, is_orig: bool%); event bittorrent_peer_choke%(c: connection, is_orig: bool%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the BitTorrent protocol. ## ## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke ## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested @@ -993,6 +1053,9 @@ event bittorrent_peer_choke%(c: connection, is_orig: bool%); event bittorrent_peer_unchoke%(c: connection, is_orig: bool%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the BitTorrent protocol. ## ## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke ## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_keep_alive @@ -1002,6 +1065,9 @@ event bittorrent_peer_unchoke%(c: connection, is_orig: bool%); event bittorrent_peer_interested%(c: connection, is_orig: bool%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the BitTorrent protocol. ## ## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke ## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested @@ -1011,6 +1077,9 @@ event bittorrent_peer_interested%(c: connection, is_orig: bool%); event bittorrent_peer_not_interested%(c: connection, is_orig: bool%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the BitTorrent protocol. ## ## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke ## bittorrent_peer_handshake bittorrent_peer_interested bittorrent_peer_keep_alive @@ -1020,6 +1089,9 @@ event bittorrent_peer_not_interested%(c: connection, is_orig: bool%); event bittorrent_peer_have%(c: connection, is_orig: bool, piece_index: count%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the BitTorrent protocol. ## ## .. bro:see:: bittorrent_peer_cancel bittorrent_peer_choke bittorrent_peer_handshake ## bittorrent_peer_have bittorrent_peer_interested bittorrent_peer_keep_alive @@ -1029,6 +1101,9 @@ event bittorrent_peer_have%(c: connection, is_orig: bool, piece_index: count%); event bittorrent_peer_bitfield%(c: connection, is_orig: bool, bitfield: string%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the BitTorrent protocol. ## ## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke ## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested @@ -1039,6 +1114,9 @@ event bittorrent_peer_request%(c: connection, is_orig: bool, index: count, begin: count, length: count%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the BitTorrent protocol. ## ## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke ## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested @@ -1049,6 +1127,9 @@ event bittorrent_peer_piece%(c: connection, is_orig: bool, index: count, begin: count, piece_length: count%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the BitTorrent protocol. ## ## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_choke ## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested @@ -1059,6 +1140,9 @@ event bittorrent_peer_cancel%(c: connection, is_orig: bool, index: count, begin: count, length: count%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the BitTorrent protocol. ## ## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke ## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested @@ -1068,6 +1152,9 @@ event bittorrent_peer_cancel%(c: connection, is_orig: bool, index: count, event bittorrent_peer_port%(c: connection, is_orig: bool, listen_port: port%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the BitTorrent protocol. ## ## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke ## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested @@ -1078,6 +1165,9 @@ event bittorrent_peer_unknown%(c: connection, is_orig: bool, message_id: count, data: string%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the BitTorrent protocol. ## ## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke ## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested @@ -1087,6 +1177,9 @@ event bittorrent_peer_unknown%(c: connection, is_orig: bool, message_id: count, event bittorrent_peer_weird%(c: connection, is_orig: bool, msg: string%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the BitTorrent protocol. ## ## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke ## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested @@ -1097,6 +1190,9 @@ event bt_tracker_request%(c: connection, uri: string, headers: bt_tracker_headers%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the BitTorrent protocol. ## ## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke ## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested @@ -1109,6 +1205,9 @@ event bt_tracker_response%(c: connection, status: count, benc: bittorrent_benc_dir%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the BitTorrent protocol. ## ## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke ## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested @@ -1119,6 +1218,9 @@ event bt_tracker_response_not_ok%(c: connection, status: count, headers: bt_tracker_headers%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the BitTorrent protocol. ## ## .. bro:see:: bittorrent_peer_bitfield bittorrent_peer_cancel bittorrent_peer_choke ## bittorrent_peer_handshake bittorrent_peer_have bittorrent_peer_interested @@ -1156,9 +1258,10 @@ event finger_request%(c: connection, full: bool, username: string, hostname: str event finger_reply%(c: connection, reply_line: string%); -# Gnutella analyzer. - ## TODO. +## +## See `Wikipedia `__ for more +## information about the Gnutella protocol. ## ## .. bro:see:: gnutella_binary_msg gnutella_establish gnutella_http_notify ## gnutella_not_establish gnutella_partial_binary_msg gnutella_signature_found @@ -1166,6 +1269,9 @@ event finger_reply%(c: connection, reply_line: string%); event gnutella_text_msg%(c: connection, orig: bool, headers: string%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the Gnutella protocol. ## ## .. bro:see:: gnutella_establish gnutella_http_notify gnutella_not_establish ## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg @@ -1175,6 +1281,9 @@ event gnutella_binary_msg%(c: connection, orig: bool, msg_type: count, trunc: bool, complete: bool%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the Gnutella protocol. ## ## .. bro:see:: gnutella_binary_msg gnutella_establish gnutella_http_notify ## gnutella_not_establish gnutella_signature_found gnutella_text_msg @@ -1182,18 +1291,27 @@ event gnutella_partial_binary_msg%(c: connection, orig: bool, msg: string, len: count%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the Gnutella protocol. ## ## .. bro:see:: gnutella_binary_msg gnutella_http_notify gnutella_not_establish ## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg event gnutella_establish%(c: connection%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the Gnutella protocol. ## ## .. bro:see:: gnutella_binary_msg gnutella_establish gnutella_http_notify ## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg event gnutella_not_establish%(c: connection%); ## TODO. +## +## See `Wikipedia `__ for more +## information about the Gnutella protocol. ## ## .. bro:see:: gnutella_binary_msg gnutella_establish gnutella_not_establish ## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg @@ -1388,6 +1506,9 @@ event login_display%(c: connection, display: string%); ## includes options for negotiating authentication. When such an option is sent ## from client to server and the server replies that it accepts the authentication, ## then the event engine generates this event. +## +## See `Wikipedia `__ for more information +## about the Telnet protocol. ## ## name: The authenticated name. ## @@ -1403,6 +1524,9 @@ event authentication_accepted%(name: string, c: connection%); ## protocol includes options for negotiating authentication. When such an option ## is sent from client to server and the server replies that it did not accept the ## authentication, then the event engine generates this event. +## +## See `Wikipedia `__ for more information +## about the Telnet protocol. ## ## name: The attempted authentication name. ## @@ -1417,6 +1541,9 @@ event authentication_rejected%(name: string, c: connection%); ## Generated when for Telnet/Rlogin sessions when a pattern match indicates ## that no authentication is performed. +## +## See `Wikipedia `__ for more information +## about the Telnet protocol. ## ## c: The connection. ## @@ -1432,6 +1559,9 @@ event authentication_skipped%(c: connection%); ## Generated for clients transmitting a terminal prompt in a Telnet session. This ## information is extracted out of environment variables sent as Telnet options. +## +## See `Wikipedia `__ for more information +## about the Telnet protocol. ## ## c: The connection. ## @@ -1444,6 +1574,9 @@ event login_prompt%(c: connection, prompt: string%); ## Generated for Telnet sessions when encryption is activated. The Telnet protoco; ## includes options for negotiating encryption. When such a series of options is ## successfully negotiated, the event engine generates this event. +## +## See `Wikipedia `__ for more information +## about the Telnet protocol. ## ## c: The connection. ## @@ -1452,22 +1585,47 @@ event login_prompt%(c: connection, prompt: string%); ## login_output_line login_prompt login_success login_terminal event activating_encryption%(c: connection%); -## TODO. +## Generated for inconsistent Telnet options observed. Telnet options are specified +## by the client and server stating which options they are willing to support +## vs. which they are not, and then instructing one another which in fact they +## should or should not use for the current connection. If the event engine sees +## a peer violate either what the other peer has instructed it to do, or what it +## itself offered in terms of options in the past, then the engine generates an +## inconsistent_option event. +## +## See `Wikipedia `__ for more information +## about the Telnet protocol. +## +## c: The connection. ## -## .. bro:see:: login_confused login_confused_text login_display login_failure -## login_input_line login_output_line login_prompt login_success login_terminal +## .. bro:see:: bad_option bad_option_termination authentication_accepted +## authentication_rejected authentication_skipped login_confused +## login_confused_text login_display login_failure login_input_line +## login_output_line login_prompt login_success login_terminal event inconsistent_option%(c: connection%); -## TODO. +## Generated for an ill-formed or unrecognized Telnet option. +## +## See `Wikipedia `__ for more information +## about the Telnet protocol. +## +## c: The connection. ## -## .. bro:see:: login_confused login_confused_text login_display login_failure -## login_input_line login_output_line login_prompt login_success login_terminal +## .. bro:see:: inconsistent_option bad_option_termination authentication_accepted +## authentication_rejected authentication_skipped login_confused +## login_confused_text login_display login_failure login_input_line +## login_output_line login_prompt login_success login_terminal event bad_option%(c: connection%); -## TODO. +## Generated for a Telnet option that's incorrectly terminated. +## +## See `Wikipedia `__ for more information +## about the Telnet protocol. ## -## .. bro:see:: login_confused login_confused_text login_display login_failure -## login_input_line login_output_line login_prompt login_success login_terminal +## .. bro:see:: inconsistent_option bad_option authentication_accepted +## authentication_rejected authentication_skipped login_confused +## login_confused_text login_display login_failure login_input_line +## login_output_line login_prompt login_success login_terminal event bad_option_termination%(c: connection%); ## Generated for client side commands on an RSH connection. @@ -5058,30 +5216,101 @@ event irc_password_message%(c: connection, is_orig: bool, password: string%); ## .. bro:see:: event file_transferred%(c: connection, prefix: string, descr: string, mime_type: string%); -## TODO. +## Generated for monitored Syslog messages. +## +## See `Wikipedia `__ for more +## information about the Syslog protocol. +## +## c: The connection record for the underlying transport-layer session/flow. +## +## facility: The "facility" included in the message. +## +## severity: The "severity" included in the message. +## +## msg: The message logged. +## +## .. note:: Bro currently parses only UDP syslog traffic. Support for TCP syslog +## will be added soon. event syslog_message%(c: connection, facility: count, severity: count, msg: string%); -## TODO. +## Generated when a signature matches. Bro's signature engine provide +## high-performance pattern matching separately from the normal script processing. +## If a signature with an ``event`` action matches, this event is raised. +## +## See the :doc:`user manual ` for more information about Bro's +## signature engine. +## +## state: Context about the match, including which signatures triggered the +## event and the connection for which the match was found. +## +## msg: The message passed to the ``event`` signature action. +## +## data; The last chunk of input that triggered the match. Note that the specifics +## here are no well-defined as Bro does not buffer any input. If a match is split +## across packet boundaries, only the last chunk triggering the will be passed on +## to the event. event signature_match%(state: signature_state, msg: string, data: string%); -## Generated if a handler finds an identification of the software -## used on a system. TODO +## Generated when a protocol analyzer finds an identification of a software +## used on a system. This is a protocol-independent event that is fed by +## different analyzers. For example, the HTTP analyzer reports user-agent and +## server software by raising this event, assuming it can parse it (if not, +## :bro:id:`software_parse_error` will be generated instead). +## +## c: The connection. +## +## host: The host running the reported software. +## +## s: A description of the software found. +## +## descr: The raw (unparsed) software identification string as extracted from the +## protocol. ## -## .. bro:see:: software_parse_error software_unparsed_version_found +## .. bro:see:: software_parse_error software_unparsed_version_found OS_version_found event software_version_found%(c: connection, host: addr, s: software, descr: string%); -## Generated if a handler finds a version but cannot parse it. TODO +## Generated when a protocol analyzer finds an identification of a software used on +## a system but cannot parse it. This is a protocol-independent event that is fed +## by different analyzers. For example, the HTTP analyzer reports user-agent and +## server software by raising this event if it cannot parse them directly (if canit +## :bro:id:`software_version_found` will be generated instead). +## +## c: The connection. +## +## host: The host running the reported software. +## +## descr: The raw (unparsed) software identification string as extracted from the +## protocol. ## ## .. bro:see:: software_version_found software_unparsed_version_found +## OS_version_found event software_parse_error%(c: connection, host: addr, descr: string%); -## Generated once for each raw (unparsed) software identification. TODO +## Generated when a protocol analyzer finds an identification of a software +## used on a system. This is a protocol-independent event that is fed by +## different analyzers. For example, the HTTP analyzer reports user-agent and +## server software by raising this event. Different from +## :bro:id:`software_version_found` and :bro:id:`software_parse_error`, this +## event is always raised, independent of whether Bro can parse the version +## string. +## +## c: The connection. +## +## host: The host running the reported software. +## +## descr: The software identification string as extracted from the protocol. ## -## .. bro:see:: software_parse_error software_version_found +## .. bro:see:: software_parse_error software_version_found OS_version_found event software_unparsed_version_found%(c: connection, host: addr, str: string%); -## Generated when an operating system has been fingerprinted. TODO +## Generated when an operating system has been fingerprinted. Bro uses `p0f +## `__ to fingerprint endpoints passively, +## and it raises this event for each system identified. The p0f fingerprints are +## defined by :bro:id:`passive_fingerprint_file`. +## +## .. bro:see:: passive_fingerprint_file software_parse_error +## software_version_found software_unparsed_version_found event OS_version_found%(c: connection, host: addr, OS: OS_version%); ## Generated when a connection to a remote Bro has been established. This event From 01693672bcf07477dfe8e102905599c9a477816e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 29 Nov 2011 12:18:34 -0600 Subject: [PATCH 105/964] Submodule README conformity changes Broccoli API reference docs now also get copied into sphinx output directory by the broxygen target if they exist. --- aux/binpac | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- aux/btest | 2 +- doc/CMakeLists.txt | 2 ++ doc/components/binpac/README.rst | 1 + doc/components/bro-aux/README.rst | 1 + doc/components/broccoli-ruby/README.rst | 1 + doc/components/broccoli/broccoli-manual.rst | 1 + doc/index.rst | 21 ++++++++++++++------- 10 files changed, 24 insertions(+), 11 deletions(-) create mode 120000 doc/components/binpac/README.rst create mode 120000 doc/components/bro-aux/README.rst create mode 120000 doc/components/broccoli-ruby/README.rst create mode 120000 doc/components/broccoli/broccoli-manual.rst diff --git a/aux/binpac b/aux/binpac index 34d9043740..82bd9613fb 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 34d90437403e4129468f89acce0bd1a99813a2f4 +Subproject commit 82bd9613fb869e44f5f0d7929bdd9a88cde84077 diff --git a/aux/broccoli b/aux/broccoli index d281350dbc..d8f9d4698e 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit d281350dbcc19c24aa6b6d89a4edc08a5c74a790 +Subproject commit d8f9d4698e8e02f493a669c5adcf897506671b5d diff --git a/aux/broctl b/aux/broctl index ed4d4ce1ad..6771d28af2 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit ed4d4ce1add51f0e08e6e8d2f5f247c2cbb422da +Subproject commit 6771d28af299f025a701e67f51311513af1cbc22 diff --git a/aux/btest b/aux/btest index 7230a09a8c..38890e8514 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 7230a09a8c220d2117e491fdf293bf5c19819b65 +Subproject commit 38890e851416fa9fc827a1d36f06c4cb9f7d4e69 diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 7685230980..2f5bd93cdf 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -51,6 +51,8 @@ add_custom_target(broxygen COMMAND "${CMAKE_COMMAND}" -E create_symlink ${DOC_OUTPUT_DIR}/html ${CMAKE_BINARY_DIR}/html + # copy Broccoli API reference into output dir if it exists + COMMAND test -d ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html && ( rm -rf ${CMAKE_BINARY_DIR}/html/broccoli-api && cp -r ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html ${CMAKE_BINARY_DIR}/html/broccoli-api ) || true WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "[Sphinx] Generating HTML policy script docs" # SOURCES just adds stuff to IDE projects as a convenience diff --git a/doc/components/binpac/README.rst b/doc/components/binpac/README.rst new file mode 120000 index 0000000000..4eb90ef658 --- /dev/null +++ b/doc/components/binpac/README.rst @@ -0,0 +1 @@ +../../../aux/binpac/README \ No newline at end of file diff --git a/doc/components/bro-aux/README.rst b/doc/components/bro-aux/README.rst new file mode 120000 index 0000000000..628879525d --- /dev/null +++ b/doc/components/bro-aux/README.rst @@ -0,0 +1 @@ +../../../aux/bro-aux/README \ No newline at end of file diff --git a/doc/components/broccoli-ruby/README.rst b/doc/components/broccoli-ruby/README.rst new file mode 120000 index 0000000000..da71663099 --- /dev/null +++ b/doc/components/broccoli-ruby/README.rst @@ -0,0 +1 @@ +../../../aux/broccoli/bindings/broccoli-ruby/README \ No newline at end of file diff --git a/doc/components/broccoli/broccoli-manual.rst b/doc/components/broccoli/broccoli-manual.rst new file mode 120000 index 0000000000..bd5e8d711f --- /dev/null +++ b/doc/components/broccoli/broccoli-manual.rst @@ -0,0 +1 @@ +../../../aux/broccoli/doc/broccoli-manual.rst \ No newline at end of file diff --git a/doc/index.rst b/doc/index.rst index 9e9aabc625..ad503af50b 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -57,13 +57,20 @@ current, independent component releases. .. toctree:: :maxdepth: 1 - components/btest/README - components/broccoli/README - components/broccoli-python/README - components/broctl/README - components/capstats/README - components/pysubnettree/README - components/trace-summary/README + BinPAC - A protocol parser generator + Broccoli - The Bro Client Communication Library (README) + Broccoli - User Manual + Broccoli Python Bindings + Broccoli Ruby Bindings + BroControl - Interactive Bro management shell + Bro-Aux - Small auxiliary tools for Bro + BTest - A unit testing framework + Capstats - Command-line packet statistic tool + PySubnetTree - Python module for CIDR lookups + trace-summary - Script for generating break-downs of network traffic + +The `Broccoli API Reference `_ may also be of +interest. Other Indices and References ---------------------------- From 2a6387129ceb68941beccb866822958c661ab2c4 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 29 Nov 2011 11:25:11 -0800 Subject: [PATCH 106/964] documentation --- doc/scripts/DocSourcesList.cmake | 3 + scripts/base/frameworks/input/main.bro | 96 ++++++++++++++++++++------ 2 files changed, 78 insertions(+), 21 deletions(-) diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index 9d99effc02..bd41d301c0 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -19,6 +19,7 @@ rest_target(${psd} base/init-bare.bro internal) rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro) +rest_target(${CMAKE_BINARY_DIR}/src base/input.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro) @@ -31,6 +32,8 @@ rest_target(${psd} base/frameworks/cluster/setup-connections.bro) rest_target(${psd} base/frameworks/communication/main.bro) rest_target(${psd} base/frameworks/control/main.bro) rest_target(${psd} base/frameworks/dpd/main.bro) +rest_target(${psd} base/frameworks/input/main.bro) +rest_target(${psd} base/frameworks/input/readers/ascii.bro) rest_target(${psd} base/frameworks/intel/main.bro) rest_target(${psd} base/frameworks/logging/main.bro) rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 1a9e8b885c..66b13743b8 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -1,52 +1,106 @@ +##! The input framework provides a way to read previously stored data either +##! as an event stream or into a bro table. module Input; export { + ## The default input reader used. Defaults to `READER_ASCII`. const default_reader = READER_ASCII &redef; + ## Stream decription type used for the `create_stream` method type StreamDescription: record { + ## String that allows the reader to find the source. + ## For `READER_ASCII`, this is the filename. source: string; + + ## Reader to use for this steam reader: Reader &default=default_reader; }; + ## TableFilter description type used for the `add_tablefilter` method. type TableFilter: record { - ## descriptive name. for later removal + ## Descriptive name. Used to remove a filter at a later time name: string; - ## for tables - idx: any; - val: any; + ## Table which will contain the data read by the input framework destination: any; + ## Record that defines the values used as the index of the table + idx: any; + ## Record that defines the values used as the values of the table + val: any; + ## Defines if the value of the table is a record (default), or a single value. + ## Val can only contain one element when this is set to false. want_record: bool &default=T; + + ## The event that is raised each time a value is added to, changed in or removed from the table. + ## The event will receive an Input::Event enum as the first argument, the idx record as the second argument + ## and the value (record) as the third argument. ev: any &optional; # event containing idx, val as values. - ## decision function, that decides if an insertion, update or removal should really be executed. - ## or events should be thought + ## Predicate function, that can decide if an insertion, update or removal should really be executed. + ## Parameters are the same as for the event. If true is returned, the update is performed. If false + ## is returned, it is skipped pred: function(typ: Input::Event, left: any, right: any): bool &optional; }; + ## EventFilter description type used for the `add_eventfilter` method. type EventFilter: record { - ## descriptive name. for later removal + ## Descriptive name. Used to remove a filter at a later time name: string; - # the event - ev: any; - # record describing the fields + ## Record describing the fields to be retrieved from the source input. fields: any; - - # does the event want the field unrolled (default) or as a simple record value? + ## If want_record if false (default), the event receives each value in fields as a seperate argument. + ## If it is set to true, the event receives all fields in a signle record value. want_record: bool &default=F; + + ## The event that is rised each time a new line is received from the reader. + ## The event will receive an Input::Event enum as the first element, and the fields as the following arguments. + ev: any; + }; #const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. - global create_stream: function(id: Log::ID, description: Input::StreamDescription) : bool; - global remove_stream: function(id: Log::ID) : bool; - global force_update: function(id: Log::ID) : bool; - global add_tablefilter: function(id: Log::ID, filter: Input::TableFilter) : bool; - global remove_tablefilter: function(id: Log::ID, name: string) : bool; - global add_eventfilter: function(id: Log::ID, filter: Input::EventFilter) : bool; - global remove_eventfilter: function(id: Log::ID, name: string) : bool; + ## Create a new input stream from a given source. Returns true on success. + ## + ## id: `Input::ID` enum value identifying this stream + ## description: `StreamDescription` record describing the source. + global create_stream: function(id: Input::ID, description: Input::StreamDescription) : bool; + + ## Remove a current input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream to be removed + global remove_stream: function(id: Input::ID) : bool; + + ## Forces the current input to be checked for changes. + ## + ## id: `Input::ID` enum value identifying the stream + global force_update: function(id: Input::ID) : bool; + + ## Adds a table filter to a specific input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream + ## filter: the `TableFilter` record describing the filter. + global add_tablefilter: function(id: Input::ID, filter: Input::TableFilter) : bool; + + ## Removes a named table filter to a specific input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream + ## name: the name of the filter to be removed. + global remove_tablefilter: function(id: Input::ID, name: string) : bool; + + ## Adds an event filter to a specific input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream + ## filter: the `EventFilter` record describing the filter. + global add_eventfilter: function(id: Input::ID, filter: Input::EventFilter) : bool; + + ## Removes a named event filter to a specific input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream + ## name: the name of the filter to be removed. + global remove_eventfilter: function(id: Input::ID, name: string) : bool; #global get_filter: function(id: ID, name: string) : Filter; } @@ -85,13 +139,13 @@ function remove_tablefilter(id: Input::ID, name: string) : bool return __remove_tablefilter(id, name); } -function add_eventfilter(id: Log::ID, filter: Input::EventFilter) : bool +function add_eventfilter(id: Input::ID, filter: Input::EventFilter) : bool { # filters[id, filter$name] = filter; return __add_eventfilter(id, filter); } -function remove_eventfilter(id: Log::ID, name: string) : bool +function remove_eventfilter(id: Input::ID, name: string) : bool { # delete filters[id, name]; return __remove_eventfilter(id, name); From 4fc5dea3d75cead3d36b61bf063abd4ffb9b980e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 29 Nov 2011 13:31:27 -0600 Subject: [PATCH 107/964] Add Broxygen links to notice frameworks docs. --- doc/notice.rst | 157 ++++++++++++++++++++++++++----------------------- 1 file changed, 83 insertions(+), 74 deletions(-) diff --git a/doc/notice.rst b/doc/notice.rst index b1ffdacb75..bab44ab9e4 100644 --- a/doc/notice.rst +++ b/doc/notice.rst @@ -29,17 +29,18 @@ definitions of what constitutes an attack or even a compromise differ quite a bit between environments, and activity deemed malicious at one site might be fully acceptable at another. -Whenever one of Bro's analysis scripts sees something potentially interesting -it flags the situation by calling the ``NOTICE`` function and giving it a -single ``Notice::Info`` record. A Notice has a ``Notice::Type``, which -reflects the kind of activity that has been seen, and it is usually also -augmented with further context about the situation. +Whenever one of Bro's analysis scripts sees something potentially +interesting it flags the situation by calling the :bro:see:`NOTICE` +function and giving it a single :bro:see:`Notice::Info` record. A Notice +has a :bro:see:`Notice::Type`, which reflects the kind of activity that +has been seen, and it is usually also augmented with further context +about the situation. More information about raising notices can be found in the `Raising Notices`_ section. Once a notice is raised, it can have any number of actions applied to it by -the ``Notice::policy`` set which is described in the `Notice Policy`_ +the :bro:see:`Notice::policy` set which is described in the `Notice Policy`_ section below. Such actions can be to send a mail to the configured address(es) or to simply ignore the notice. Currently, the following actions are defined: @@ -52,20 +53,20 @@ are defined: - Description * - Notice::ACTION_LOG - - Write the notice to the ``Notice::LOG`` logging stream. + - Write the notice to the :bro:see:`Notice::LOG` logging stream. * - Notice::ACTION_ALARM - - Log into the ``Notice::ALARM_LOG`` stream which will rotate + - Log into the :bro:see:`Notice::ALARM_LOG` stream which will rotate hourly and email the contents to the email address or addresses - defined in the ``Notice::mail_dest`` variable. + defined in the :bro:see:`Notice::mail_dest` variable. * - Notice::ACTION_EMAIL - Send the notice in an email to the email address or addresses given in - the ``Notice::mail_dest`` variable. + the :bro:see:`Notice::mail_dest` variable. * - Notice::ACTION_PAGE - Send an email to the email address or addresses given in the - ``Notice::mail_page_dest`` variable. + :bro:see:`Notice::mail_page_dest` variable. * - Notice::ACTION_NO_SUPPRESS - This action will disable the built in notice suppression for the @@ -82,15 +83,17 @@ Processing Notices Notice Policy ************* -The predefined set ``Notice::policy`` provides the mechanism for applying -actions and other behavior modifications to notices. Each entry of -``Notice::policy`` is a record of the type ``Notice::PolicyItem`` which -defines a condition to be matched against all raised notices and one or more -of a variety of behavior modifiers. The notice policy is defined by adding any -number of ``Notice::PolicyItem`` records to the ``Notice::policy`` set. +The predefined set :bro:see:`Notice::policy` provides the mechanism for +applying actions and other behavior modifications to notices. Each entry +of :bro:see:`Notice::policy` is a record of the type +:bro:see:`Notice::PolicyItem` which defines a condition to be matched +against all raised notices and one or more of a variety of behavior +modifiers. The notice policy is defined by adding any number of +:bro:see:`Notice::PolicyItem` records to the :bro:see:`Notice::policy` +set. Here's a simple example which tells Bro to send an email for all notices of -type ``SSH::Login`` if the server is 10.0.0.1: +type :bro:see:`SSH::Login` if the server is 10.0.0.1: .. code:: bro @@ -113,11 +116,11 @@ flexibility due to having access to Bro's full programming language. Predicate Field ^^^^^^^^^^^^^^^ -The ``Notice::PolicyItem`` record type has a field name ``$pred`` which -defines the entry's condition in the form of a predicate written as a Bro -function. The function is passed the notice as a ``Notice::Info`` record and -it returns a boolean value indicating if the entry is applicable to that -particular notice. +The :bro:see:`Notice::PolicyItem` record type has a field name ``$pred`` +which defines the entry's condition in the form of a predicate written +as a Bro function. The function is passed the notice as a +:bro:see:`Notice::Info` record and it returns a boolean value indicating +if the entry is applicable to that particular notice. .. note:: @@ -125,14 +128,14 @@ particular notice. (``T``) since an implicit false (``F``) value would never be used. Bro evaluates the predicates of each entry in the order defined by the -``$priority`` field in ``Notice::PolicyItem`` records. The valid values are -0-10 with 10 being earliest evaluated. If ``$priority`` is omitted, the -default priority is 5. +``$priority`` field in :bro:see:`Notice::PolicyItem` records. The valid +values are 0-10 with 10 being earliest evaluated. If ``$priority`` is +omitted, the default priority is 5. Behavior Modification Fields ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -There are a set of fields in the ``Notice::PolicyItem`` record type that +There are a set of fields in the :bro:see:`Notice::PolicyItem` record type that indicate ways that either the notice or notice processing should be modified if the predicate field (``$pred``) evaluated to true (``T``). Those fields are explained in more detail in the following table. @@ -146,8 +149,8 @@ explained in more detail in the following table. - Example * - ``$action=`` - - Each Notice::PolicyItem can have a single action applied to the notice - with this field. + - Each :bro:see:`Notice::PolicyItem` can have a single action + applied to the notice with this field. - ``$action = Notice::ACTION_EMAIL`` * - ``$suppress_for=`` @@ -162,9 +165,9 @@ explained in more detail in the following table. - This field can be used for modification of the notice policy evaluation. To stop processing of notice policy items before evaluating all of them, set this field to ``T`` and make the ``$pred`` - field return ``T``. ``Notice::PolicyItem`` records defined at a higher - priority as defined by the ``$priority`` field will still be evaluated - but those at a lower priority won't. + field return ``T``. :bro:see:`Notice::PolicyItem` records defined at + a higher priority as defined by the ``$priority`` field will still be + evaluated but those at a lower priority won't. - ``$halt = T`` @@ -186,11 +189,11 @@ Notice Policy Shortcuts Although the notice framework provides a great deal of flexibility and configurability there are many times that the full expressiveness isn't needed and actually becomes a hindrance to achieving results. The framework provides -a default ``Notice::policy`` suite as a way of giving users the +a default :bro:see:`Notice::policy` suite as a way of giving users the shortcuts to easily apply many common actions to notices. These are implemented as sets and tables indexed with a -``Notice::Type`` enum value. The following table shows and describes +:bro:see:`Notice::Type` enum value. The following table shows and describes all of the variables available for shortcut configuration of the notice framework. @@ -201,40 +204,44 @@ framework. * - Variable name - Description - * - Notice::ignored_types - - Adding a ``Notice::Type`` to this set results in the notice + * - :bro:see:`Notice::ignored_types` + - Adding a :bro:see:`Notice::Type` to this set results in the notice being ignored. It won't have any other action applied to it, not even - ``Notice::ACTION_LOG``. + :bro:see:`Notice::ACTION_LOG`. - * - Notice::emailed_types - - Adding a ``Notice::Type`` to this set results in - ``Notice::ACTION_EMAIL`` being applied to the notices of that type. + * - :bro:see:`Notice::emailed_types` + - Adding a :bro:see:`Notice::Type` to this set results in + :bro:see:`Notice::ACTION_EMAIL` being applied to the notices of + that type. - * - Notice::alarmed_types - - Adding a Notice::Type to this set results in - ``Notice::ACTION_ALARM`` being applied to the notices of that type. + * - :bro:see:`Notice::alarmed_types` + - Adding a :bro:see:`Notice::Type` to this set results in + :bro:see:`Notice::ACTION_ALARM` being applied to the notices of + that type. - * - Notice::not_suppressed_types - - Adding a ``Notice::Type`` to this set results in that notice no longer - undergoing the normal notice suppression that would take place. Be - careful when using this in production it could result in a dramatic - increase in the number of notices being processed. + * - :bro:see:`Notice::not_suppressed_types` + - Adding a :bro:see:`Notice::Type` to this set results in that notice + no longer undergoes the normal notice suppression that would + take place. Be careful when using this in production it could + result in a dramatic increase in the number of notices being + processed. - * - Notice::type_suppression_intervals - - This is a table indexed on ``Notice::Type`` and yielding an interval. - It can be used as an easy way to extend the default suppression - interval for an entire ``Notice::Type`` without having to create a - whole ``Notice::policy`` entry and setting the ``$suppress_for`` - field. + * - :bro:see:`Notice::type_suppression_intervals` + - This is a table indexed on :bro:see:`Notice::Type` and yielding an + interval. It can be used as an easy way to extend the default + suppression interval for an entire :bro:see:`Notice::Type` + without having to create a whole :bro:see:`Notice::policy` entry + and setting the ``$suppress_for`` field. Raising Notices --------------- -A script should raise a notice for any occurrence that a user may want to be -notified about or take action on. For example, whenever the base SSH analysis -scripts sees an SSH session where it is heuristically guessed to be a -successful login, it raises a Notice of the type ``SSH::Login``. The code in -the base SSH analysis script looks like this: +A script should raise a notice for any occurrence that a user may want +to be notified about or take action on. For example, whenever the base +SSH analysis scripts sees an SSH session where it is heuristically +guessed to be a successful login, it raises a Notice of the type +:bro:see:`SSH::Login`. The code in the base SSH analysis script looks +like this: .. code:: bro @@ -242,10 +249,10 @@ the base SSH analysis script looks like this: $msg="Heuristically detected successful SSH login.", $conn=c]); -``NOTICE`` is a normal function in the global namespace which wraps a function -within the ``Notice`` namespace. It takes a single argument of the -``Notice::Info`` record type. The most common fields used when raising notices -are described in the following table: +:bro:see:`NOTICE` is a normal function in the global namespace which +wraps a function within the ``Notice`` namespace. It takes a single +argument of the :bro:see:`Notice::Info` record type. The most common +fields used when raising notices are described in the following table: .. list-table:: :widths: 32 40 @@ -295,9 +302,10 @@ are described in the following table: * - ``$suppress_for`` - This field can be set if there is a natural suppression interval for - the notice that may be different than the default value. The value set - to this field can also be modified by a user's ``Notice::policy`` so - the value is not set permanently and unchangeably. + the notice that may be different than the default value. The + value set to this field can also be modified by a user's + :bro:see:`Notice::policy` so the value is not set permanently + and unchangeably. When writing Bro scripts which raise notices, some thought should be given to what the notice represents and what data should be provided to give a consumer @@ -325,7 +333,7 @@ The notice framework supports suppression for notices if the author of the script that is generating the notice has indicated to the notice framework how to identify notices that are intrinsically the same. Identification of these "intrinsically duplicate" notices is implemented with an optional field in -``Notice::Info`` records named ``$identifier`` which is a simple string. +:bro:see:`Notice::Info` records named ``$identifier`` which is a simple string. If the ``$identifier`` and ``$type`` fields are the same for two notices, the notice framework actually considers them to be the same thing and can use that information to suppress duplicates for a configurable period of time. @@ -337,12 +345,13 @@ information to suppress duplicates for a configurable period of time. could be completely legitimate usage if no notices could ever be considered to be duplicates. -The ``$identifier`` field is typically comprised of several pieces of data -related to the notice that when combined represent a unique instance of that -notice. Here is an example of the script -``policy/protocols/ssl/validate-certs.bro`` raising a notice for session -negotiations where the certificate or certificate chain did not validate -successfully against the available certificate authority certificates. +The ``$identifier`` field is typically comprised of several pieces of +data related to the notice that when combined represent a unique +instance of that notice. Here is an example of the script +:doc:`scripts/policy/protocols/ssl/validate-certs` raising a notice +for session negotiations where the certificate or certificate chain did +not validate successfully against the available certificate authority +certificates. .. code:: bro @@ -369,7 +378,7 @@ it's assumed that the script author who is raising the notice understands the full problem set and edge cases of the notice which may not be readily apparent to users. If users don't want the suppression to take place or simply want a different interval, they can always modify it with the -``Notice::policy``. +:bro:see:`Notice::policy`. Extending Notice Framework From 76ca1f532fd0cbaf19d928d5c254a7cd4d47c693 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Tue, 29 Nov 2011 12:17:51 -0800 Subject: [PATCH 108/964] Begin with documenting strings.bif. --- src/strings.bif | 87 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 19 deletions(-) diff --git a/src/strings.bif b/src/strings.bif index 7e9885e296..88d4f82791 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -10,6 +10,12 @@ using namespace std; %%} +## Concates all arguments into a single string. The function takes a variable +## number of arguments of type string and stiches them together. +## +## Returns: The concatenation of all (string) arguments. +## +## see:: cat cat_string_array cat_string_array_n fmt join_string_vec join_string_array function string_cat%(...%): string %{ int n = 0; @@ -73,18 +79,44 @@ BroString* cat_string_array_n(TableVal* tbl, int start, int end) } %%} +## Concatenates all elements in an array of strings. +## +## a: The :bro:id:`string_array` (``table[count] of string``). +## +## Returns: The concatenation of all elements in *a*. +## +## see:: cat string_cat cat_string_array_n fmt join_string_vec join_string_array function cat_string_array%(a: string_array%): string %{ TableVal* tbl = a->AsTableVal(); return new StringVal(cat_string_array_n(tbl, 1, a->AsTable()->Length())); %} +## Concatenates a specific range of elements in an array of strings. +## +## a: The :bro:id:`string_array` (``table[count] of string``). +## start: The array index of the first element of the range. +## end: The array index of the last element of the range. +## +## Returns: The concatenation of the range *[start, end]* in *a*. +## +## see:: cat string_cat cat_string_array_n fmt join_string_vec join_string_array function cat_string_array_n%(a: string_array, start: count, end: count%): string %{ TableVal* tbl = a->AsTableVal(); return new StringVal(cat_string_array_n(tbl, start, end)); %} +## Joins all values in the given array of strings with a separator placed +## between each element. +## +## sep: The separator to place between each element. +## a: The :bro:id:`string_array` (``table[count] of string``). +## +## Returns: The concatenation of all elements in *a*, with *sep* placed +## between each element. +## +## see:: cat string_cat cat_string_array_n fmt join_string_vec join_string_array function join_string_array%(sep: string, a: string_array%): string %{ vector vs; @@ -108,6 +140,42 @@ function join_string_array%(sep: string, a: string_array%): string return new StringVal(concatenate(vs)); %} +## Joins all values in the given vector of strings with a separator placed +## between each element. +## +## sep: The separator to place between each element. +## a: The :bro:id:`string_vec` (``vector of string``). +## +## Returns: The concatenation of all elements in *a*, with *sep* placed +## between each element. +## +## see:: cat string_cat cat_string_array_n fmt join_string_vec join_string_array +function join_string_vec%(vec: string_vec, sep: string%): string + %{ + ODesc d; + VectorVal *v = vec->AsVectorVal(); + + for ( unsigned i = 0; i < v->Size(); ++i ) + { + if ( i > 0 ) + d.Add(sep->CheckString(), 0); + + v->Lookup(i+1)->Describe(&d); + } + + BroString* s = new BroString(1, d.TakeBytes(), d.Len()); + s->SetUseFreeToDelete(true); + + return new StringVal(s); + %} + +## Sorts an array of strings. +## +## a: The :bro:id:`string_array` (``table[count] of string``). +## +## Returns: A sorted copy of *a*. +## +## see:: sort function sort_string_array%(a: string_array%): string_array %{ TableVal* tbl = a->AsTableVal(); @@ -133,25 +201,6 @@ function sort_string_array%(a: string_array%): string_array vs_to_string_array(vs, b, 1, n); return b; %} - -function join_string_vec%(vec: string_vec, sep: string%): string - %{ - ODesc d; - VectorVal *v = vec->AsVectorVal(); - - for ( unsigned i = 0; i < v->Size(); ++i ) - { - if ( i > 0 ) - d.Add(sep->CheckString(), 0); - - v->Lookup(i+1)->Describe(&d); - } - - BroString* s = new BroString(1, d.TakeBytes(), d.Len()); - s->SetUseFreeToDelete(true); - - return new StringVal(s); - %} function edit%(arg_s: string, arg_edit_char: string%): string From a68e6b9fa49211a7c51c45f83ec0c610f03a956d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 29 Nov 2011 14:32:53 -0800 Subject: [PATCH 109/964] allow sets to be read from files, convenience function for reading a file once, bug in destructor that could lead to a segfault. --- scripts/base/frameworks/input/main.bro | 34 +++++++++++- src/InputMgr.cc | 54 ++++++++++++++----- .../scripts/base/frameworks/input/basic.bro | 2 + 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 66b13743b8..c76eba80b9 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -4,6 +4,8 @@ module Input; export { + redef enum Input::ID += { TABLE_READ }; + ## The default input reader used. Defaults to `READER_ASCII`. const default_reader = READER_ASCII &redef; @@ -27,7 +29,8 @@ export { ## Record that defines the values used as the index of the table idx: any; ## Record that defines the values used as the values of the table - val: any; + ## If val is undefined, destination has to be a set. + val: any &optional; ## Defines if the value of the table is a record (default), or a single value. ## Val can only contain one element when this is set to false. want_record: bool &default=T; @@ -102,6 +105,14 @@ export { ## name: the name of the filter to be removed. global remove_eventfilter: function(id: Input::ID, name: string) : bool; #global get_filter: function(id: ID, name: string) : Filter; + + ## Convenience function for reading a specific input source exactly once using + ## exactly one tablefilter + ## + ## id: `Input::ID` enum value identifying the stream + ## description: `StreamDescription` record describing the source. + ## filter: the `TableFilter` record describing the filter. + global read_table: function(description: Input::StreamDescription, filter: Input::TableFilter) : bool; } @@ -151,6 +162,27 @@ function remove_eventfilter(id: Input::ID, name: string) : bool return __remove_eventfilter(id, name); } +function read_table(description: Input::StreamDescription, filter: Input::TableFilter) : bool { + local ok: bool = T; + # since we create and delete it ourselves this should be ok... at least for singlethreaded operation + local id: Input::ID = Input::TABLE_READ; + + ok = create_stream(id, description); + if ( ok ) { + ok = add_tablefilter(id, filter); + } + if ( ok ) { + ok = force_update(id); + } + if ( ok ) { + ok = remove_stream(id); + } else { + remove_stream(id); + } + + return ok; +} + #function get_filter(id: ID, name: string) : Filter # { # if ( [id, name] in filters ) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 0df11ea359..7ec5974199 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -86,7 +86,8 @@ InputMgr::Filter::~Filter() { InputMgr::TableFilter::~TableFilter() { Unref(tab); Unref(itype); - Unref(rtype); + if ( rtype ) // can be 0 for sets + Unref(rtype); delete currDict; delete lastDict; @@ -110,6 +111,7 @@ InputMgr::ReaderInfo::~ReaderInfo() { while ( it != filters.end() ) { delete (*it).second; + ++it; } Unref(type); @@ -354,7 +356,10 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { Val* pred = fval->Lookup(rtype->FieldOffset("pred")); RecordType *idx = fval->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); - RecordType *val = fval->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + RecordType *val = 0; + if ( fval->Lookup(rtype->FieldOffset("val")) != 0 ) { + val = fval->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + } TableVal *dst = fval->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); @@ -408,9 +413,14 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { int idxfields = fieldsV.size(); - status = status || !UnrollRecordType(&fieldsV, val, ""); + if ( val ) // if we are not a set + status = status || !UnrollRecordType(&fieldsV, val, ""); + int valfields = fieldsV.size() - idxfields; + if ( !val ) + assert(valfields == 0); + if ( status ) { reporter->Error("Problem unrolling"); return false; @@ -429,7 +439,7 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { filter->num_idx_fields = idxfields; filter->num_val_fields = valfields; filter->tab = dst->Ref()->AsTableVal(); - filter->rtype = val->Ref()->AsRecordType(); + filter->rtype = val ? val->Ref()->AsRecordType() : 0; filter->itype = idx->Ref()->AsRecordType(); filter->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; filter->currDict = new PDict(InputHash); @@ -681,7 +691,10 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c HashKey* idxhash = HashLogVals(filter->num_idx_fields, vals); //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); //reporter->Error("Hashing %d val fields", i->num_val_fields); - HashKey* valhash = HashLogVals(filter->num_val_fields, vals+filter->num_idx_fields); + HashKey* valhash = 0; + if ( filter->num_val_fields > 0 ) + HashLogVals(filter->num_val_fields, vals+filter->num_idx_fields); + //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); @@ -689,12 +702,13 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before - if ( h->valhash->Hash() == valhash->Hash() ) { - // ok, double. + if ( filter->num_val_fields == 0 || h->valhash->Hash() == valhash->Hash() ) { + // ok, exact duplicate filter->lastDict->Remove(idxhash); filter->currDict->Insert(idxhash, h); return; } else { + assert( filter->num_val_fields > 0 ); // updated filter->lastDict->Remove(idxhash); delete(h); @@ -708,7 +722,9 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c Val* valval; int position = filter->num_idx_fields; - if ( filter->num_val_fields == 1 && !filter->want_record ) { + if ( filter->num_val_fields == 0 ) { + valval = 0; + } else if ( filter->num_val_fields == 1 && !filter->want_record ) { valval = LogValToVal(vals[position], filter->rtype->FieldType(0)); } else { RecordVal * r = new RecordVal(filter->rtype); @@ -732,8 +748,9 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c Val* oldval = 0; if ( updated == true ) { - // in that case, we need the old value to send the event (if we send an event). - oldval = filter->tab->Lookup(idxval); + assert(filter->num_val_fields > 0); + // in that case, we need the old value to send the event (if we send an event). + oldval = filter->tab->Lookup(idxval); } @@ -749,10 +766,12 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); } - val_list vl(3); + val_list vl( 2 + (filter->num_val_fields > 0) ); // 2 if we don't have values, 3 otherwise. vl.append(ev); vl.append(idxval); - vl.append(valval); + if ( filter->num_val_fields > 0 ) + vl.append(valval); + Val* v = filter->pred->Call(&vl); bool result = v->AsBool(); Unref(v); @@ -794,6 +813,7 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c Ref(idxval); if ( updated ) { // in case of update send back the old value. + assert ( filter->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); Ref(oldval); @@ -801,7 +821,11 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); Ref(valval); - SendEvent(filter->event, 3, ev, idxval, valval); + if ( filter->num_val_fields == 0 ) { + SendEvent(filter->event, 3, ev, idxval); + } else { + SendEvent(filter->event, 3, ev, idxval, valval); + } } } } @@ -963,7 +987,9 @@ void InputMgr::PutTable(const InputReader* reader, int id, const LogVal* const * Val* valval; int position = filter->num_idx_fields; - if ( filter->num_val_fields == 1 && !filter->want_record ) { + if ( filter->num_val_fields == 0 ) { + valval = 0; + } else if ( filter->num_val_fields == 1 && !filter->want_record ) { valval = LogValToVal(vals[filter->num_idx_fields], filter->rtype->FieldType(filter->num_idx_fields)); } else { RecordVal * r = new RecordVal(filter->rtype); diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 10cc7376a8..d1b6659eb6 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -49,4 +49,6 @@ event bro_init() Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::INPUT); print servers; + Input::remove_tablefilter(A::INPUT, "ssh"); + Input::remove_stream(A::INPUT); } From ebd15cf12e5af1741eb553369b6a2139ffdba4aa Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 29 Nov 2011 16:55:31 -0800 Subject: [PATCH 110/964] Fixing ASCII logger to escape the unset-field place-holder if written out literally. --- CHANGES | 5 ++++ VERSION | 2 +- src/LogWriterAscii.cc | 29 +++++++++++++++++-- .../test.log | 5 ++++ .../logging/ascii-escape-notset-str.bro | 23 +++++++++++++++ 5 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log create mode 100644 testing/btest/scripts/base/frameworks/logging/ascii-escape-notset-str.bro diff --git a/CHANGES b/CHANGES index 85793fbc18..9474645544 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.0-beta-69 | 2011-11-29 16:55:31 -0800 + + * Fixing ASCII logger to escape the unset-field place holder if + written out literally. (Robin Sommer) + 2.0-beta-68 | 2011-11-29 15:23:12 -0800 * Lots of documentation polishing. (Jon Siwek) diff --git a/VERSION b/VERSION index 1d512c0f18..0ce06b2179 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0-beta-68 +2.0-beta-69 diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 9fc71789d8..5bd476c936 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -200,10 +200,33 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) case TYPE_FUNC: { int size = val->val.string_val->size(); - if ( size ) - desc->AddN(val->val.string_val->data(), val->val.string_val->size()); - else + const char* data = val->val.string_val->data(); + + if ( ! size ) + { desc->AddN(empty_field, empty_field_len); + break; + } + + if ( size == unset_field_len && memcmp(data, unset_field, size) == 0 ) + { + // The value we'd write out would match exactly the + // place-holder we use for unset optional fields. We + // escape the first character so that the output + // won't be ambigious. + static const char hex_chars[] = "0123456789abcdef"; + char hex[6] = "\\x00"; + hex[2] = hex_chars[((*data) & 0xf0) >> 4]; + hex[3] = hex_chars[(*data) & 0x0f]; + desc->AddRaw(hex, 4); + + ++data; + --size; + } + + if ( size ) + desc->AddN(data, size); + break; } diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log new file mode 100644 index 0000000000..683fed60f2 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log @@ -0,0 +1,5 @@ +#separator \x09 +#path test +#fields x y z +#types string string string +\x2d - - diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-escape-notset-str.bro b/testing/btest/scripts/base/frameworks/logging/ascii-escape-notset-str.bro new file mode 100644 index 0000000000..8c1401b179 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/ascii-escape-notset-str.bro @@ -0,0 +1,23 @@ +# +# @TEST-EXEC: bro -b %INPUT +# @TEST-EXEC: btest-diff test.log + +module Test; + +export { + redef enum Log::ID += { LOG }; + + type Log: record { + x: string &optional; + y: string &optional; + z: string &optional; + } &log; +} + +event bro_init() +{ + Log::create_stream(Test::LOG, [$columns=Log]); + Log::write(Test::LOG, [$x=LogAscii::unset_field, $z=""]); +} + + From 98028dba8967d0ab2ea3295e19346f8a19c118b2 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Tue, 29 Nov 2011 14:55:26 -0800 Subject: [PATCH 111/964] Finish Broxygen documentation of string.bif. --- src/strings.bif | 361 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 318 insertions(+), 43 deletions(-) diff --git a/src/strings.bif b/src/strings.bif index 88d4f82791..6eb04b1bcd 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -15,7 +15,9 @@ using namespace std; ## ## Returns: The concatenation of all (string) arguments. ## -## see:: cat cat_string_array cat_string_array_n fmt join_string_vec join_string_array +## .. bro:see:: cat cat_string_array cat_string_array_n +## fmt +## join_string_vec join_string_array function string_cat%(...%): string %{ int n = 0; @@ -85,7 +87,9 @@ BroString* cat_string_array_n(TableVal* tbl, int start, int end) ## ## Returns: The concatenation of all elements in *a*. ## -## see:: cat string_cat cat_string_array_n fmt join_string_vec join_string_array +## .. bro:see:: cat string_cat cat_string_array_n +## fmt +## join_string_vec join_string_array function cat_string_array%(a: string_array%): string %{ TableVal* tbl = a->AsTableVal(); @@ -95,12 +99,16 @@ function cat_string_array%(a: string_array%): string ## Concatenates a specific range of elements in an array of strings. ## ## a: The :bro:id:`string_array` (``table[count] of string``). +## ## start: The array index of the first element of the range. +## ## end: The array index of the last element of the range. ## ## Returns: The concatenation of the range *[start, end]* in *a*. ## -## see:: cat string_cat cat_string_array_n fmt join_string_vec join_string_array +## .. bro:see:: cat string_cat cat_string_array +## fmt +## join_string_vec join_string_array function cat_string_array_n%(a: string_array, start: count, end: count%): string %{ TableVal* tbl = a->AsTableVal(); @@ -111,12 +119,15 @@ function cat_string_array_n%(a: string_array, start: count, end: count%): string ## between each element. ## ## sep: The separator to place between each element. +## ## a: The :bro:id:`string_array` (``table[count] of string``). ## ## Returns: The concatenation of all elements in *a*, with *sep* placed ## between each element. ## -## see:: cat string_cat cat_string_array_n fmt join_string_vec join_string_array +## .. bro:see:: cat string_cat cat_string_array cat_string_array_n +## fmt +## join_string_vec function join_string_array%(sep: string, a: string_array%): string %{ vector vs; @@ -144,12 +155,15 @@ function join_string_array%(sep: string, a: string_array%): string ## between each element. ## ## sep: The separator to place between each element. +## ## a: The :bro:id:`string_vec` (``vector of string``). ## ## Returns: The concatenation of all elements in *a*, with *sep* placed ## between each element. ## -## see:: cat string_cat cat_string_array_n fmt join_string_vec join_string_array +## .. bro:see:: cat string_cat cat_string_array cat_string_array_n +## fmt +## join_string_array function join_string_vec%(vec: string_vec, sep: string%): string %{ ODesc d; @@ -175,7 +189,7 @@ function join_string_vec%(vec: string_vec, sep: string%): string ## ## Returns: A sorted copy of *a*. ## -## see:: sort +## .. bro:see:: sort function sort_string_array%(a: string_array%): string_array %{ TableVal* tbl = a->AsTableVal(); @@ -203,6 +217,24 @@ function sort_string_array%(a: string_array%): string_array %} +## Returns an edited version of a string that applies a special +## "backspace character" (usually ``\x08`` for backspace or ``\x7f`` for DEL). +## For ## example, ``edit("hello there", "e")`` returns ``"llo t"``. +## +## arg_s: The string to edit. +## +## arg_edit_char: A string of exactly one character that represents the +## "backspace character". If it is longer than one character Bro +## generates a run-time error and uses the first character in +## the string. +## +## Returns: An edited version of *arg_s* where *arg_edit_char* triggers the +## deletetion of the last character. +## +## .. bro:see:: clean +## to_string_literal +## escape_string +## strip function edit%(arg_s: string, arg_edit_char: string%): string %{ if ( arg_edit_char->Len() != 1 ) @@ -233,11 +265,28 @@ function edit%(arg_s: string, arg_edit_char: string%): string return new StringVal(new BroString(1, byte_vec(new_s), ind)); %} +## Returns the number of characters (i.e., bytes) in the given string. The +## length computation includes any embedded NULs, and also a trailing NUL, +## if any (which is why the function isn't called ``strlen``; to remind +## the user that Bro strings can include NULs). +## +## s: The string to compute the length for. +## +## Returns: The number of characters in *s*. function byte_len%(s: string%): count %{ return new Val(s->Len(), TYPE_COUNT); %} +## Get a substring of from a string, given a starting position length. +## +## s: The string to obtain a substring from. +## +## start: The starting position of the substring in *s* +## +## n: The number of characters to extract, beginning at *start*. +## +## Returns: A substring of *s* of length *n* from position *start*. function sub_bytes%(s: string, start: count, n: int%): string %{ if ( start > 0 ) @@ -417,42 +466,94 @@ Val* do_sub(StringVal* str_val, RE_Matcher* re, StringVal* repl, int do_all) } %%} -# Similar to split in awk. - +## Splits a string into an array of strings according to a pattern. +## +## str: The string to split. +## +## re: The pattern describing the element separator in *str*. +## +## Returns: An array of strings where each element corresponds to a substring +## in *str* separated by *re*. +## +## .. bro:see:: split1 split_all split_n str_split +## +## .. note:: The returned table starts at index 1. Note that conceptually the +## return value is meant to be a vector and this might change in the +## future. +## function split%(str: string, re: pattern%): string_array %{ return do_split(str, re, 0, 0, 0); %} -# split1(str, pattern, include_separator): table[count] of string -# -# Same as split, except that str is only split (if possible) at the -# earliest position and an array of two strings is returned. -# An array of one string is returned when str cannot be splitted. - +## Splits a string *once* into a a two-element array of strings according to a +## pattern. This function is the same as :bro:id:`split`, but * is only split +## once (if possible) at the earliest position and an array of two strings is +## returned. +## +## str: The string to split. +## +## re: The pattern describing the separator to split *str* in two pieces. +## +## Returns: An array of strings with two elements in which the first represents +## the substring in *str* up to the first occurence of *re*, and the +## second everything after *re*. An array of one string is returned +## when *s* cannot be split. +## +## .. bro:see:: split split_all split_n str_split function split1%(str: string, re: pattern%): string_array %{ return do_split(str, re, 0, 0, 1); %} -# Same as split, except that the array returned by split_all also -# includes parts of string that match the pattern in the array. - -# For example, split_all("a-b--cd", /(\-)+/) returns {"a", "-", "b", -# "--", "cd"}: odd-indexed elements do not match the pattern -# and even-indexed ones do. - +## Splits a string into an array of strings according to a pattern. This +## function is the same as :bro:id:`split`, except that the separators are +## returned as well. For example, ``split_all("a-b--cd", /(\-)+/)`` returns +## ``{"a", "-", "b", "--", "cd"}``: odd-indexed elements do not match the +## pattern and even-indexed ones do. +## +## str: The string to split. +## +## re: The pattern describing the element separator in *str*. +## +## Returns: An array of strings where each two successive elements correspond +## to a substring in *str* of the part not matching *re* (odd-indexed) and the +## part that matches *re* (even-indexed). +## +## .. bro:see:: split split1 split_n str_split function split_all%(str: string, re: pattern%): string_array %{ return do_split(str, re, 0, 1, 0); %} +## Splits a string a given number of times into an array of strings according +## to a pattern. This function is similar to :bro:id:`split1` and +## :bro:id:`split_all`, but with customizable behavior with respect to +## including separators in the result and the number of times to split. +## +## str: The string to split. +## +## re: The pattern describing the element separator in *str*. +## +## incl_sep: A flag indicating whether to include the separator matches in the +## result (as in :bro:id:`split_all`). +## +## max_num_sep: The number of times to split *str*. +## +## Returns: An array of strings where, if *incl_sep* is true, each two +## successive elements correspond to a substring in *str* of the part +## not matching *re* (odd-indexed) and the part that matches *re* +## (even-indexed). +## +## .. bro:see:: split split1 split_all str_split function split_n%(str: string, re: pattern, incl_sep: bool, max_num_sep: count%): string_array %{ return do_split(str, re, 0, incl_sep, max_num_sep); %} +## Deprecated. Will be removed. +# Reason: the parameter ``other`` does nothing. function split_complete%(str: string, re: pattern, other: string_set, incl_sep: bool, max_num_sep: count%): string_array @@ -460,22 +561,65 @@ function split_complete%(str: string, return do_split(str, re, other->AsTableVal(), incl_sep, max_num_sep); %} +## Substitutes a given replacement string for the first occurrence of a pattern +## in a given string. +## +## str: The string to perform the substitution in. +## +## re: The pattern being replaced with *repl*. +## +## repl: The string that replacs *re*. +## +## Returns: A copy of *str* with the first occurence of *re* replaced with +## *repl*. +## +## .. bro:see:: gsub subst_string function sub%(str: string, re: pattern, repl: string%): string %{ return do_sub(str, re, repl, 0); %} +## Substitutes a given replacement string for the all occurrences of a pattern +## in a given string. +## +## str: The string to perform the substitution in. +## +## re: The pattern being replaced with *repl*. +## +## repl: The string that replacs *re*. +## +## Returns: A copy of *str* with all occurences of *re* replaced with *repl*. +## +## .. bro:see:: sub subst_string function gsub%(str: string, re: pattern, repl: string%): string %{ return do_sub(str, re, repl, 1); %} + +## Lexicographically compares two string. +## +## s1: The first string. +## +## s2: The second string. +## +## Returns: An integer greater than, equal to, or less than 0 according as +## *s1* is greater than, equal to, or less than *s2*. function strcmp%(s1: string, s2: string%): int %{ return new Val(Bstr_cmp(s1->AsString(), s2->AsString()), TYPE_INT); %} -# Returns 0 if $little is not found in $big. +## Locates the first occurrence of one string in another. +## +## big: The string to look in. +## +## little: The (smaller) string to find inside *big*. +## +## Returns: The location of *little* in *big* or 0 if *little* is not found in +## *big*. +## +## .. bro:see:: find_all find_last function strstr%(big: string, little: string%): count %{ return new Val( @@ -483,8 +627,17 @@ function strstr%(big: string, little: string%): count TYPE_COUNT); %} -# Substitute each (non-overlapping) appearance of $from in $s to $to, -# and return the resulting string. +## Substitutes each (non-overlapping) appearance of a string in another. +## +## s: The string in which to perform the substitution. +## +## from: The string to look for which is replaced with *to*. +## +## to: The string that replaces all occurrences of *from* in *s*. +## +## Returns: A copy of *s* where each occurrence of *from* is replaced with *to*. +## +## .. bro:see:: sub gsub function subst_string%(s: string, from: string, to: string%): string %{ const int little_len = from->Len(); @@ -527,6 +680,14 @@ function subst_string%(s: string, from: string, to: string%): string return new StringVal(concatenate(vs)); %} +## Replaces all uppercase letters in a string with their lowercase counterpart. +## +## str: The string to convert to lowercase letters. +## +## Returns: A copy of the given string with the uppercase letters (as indicated +## by ``isascii`` and \verb|isupper|``) folded to lowercase (via ``tolower``). +## +## .. bro:see:: to_upper is_ascii function to_lower%(str: string%): string %{ const u_char* s = str->Bytes(); @@ -547,6 +708,14 @@ function to_lower%(str: string%): string return new StringVal(new BroString(1, lower_s, n)); %} +## Replaces all lowercase letters in a string with their uppercase counterpart. +## +## str: The string to convert to uppercase letters. +## +## Returns: A copy of the given string with the lowercase letters (as indicated +## by ``isascii`` and \verb|islower|``) folded to uppercase (via ``toupper``). +## +## .. bro:see:: to_lower is_ascii function to_upper%(str: string%): string %{ const u_char* s = str->Bytes(); @@ -567,18 +736,54 @@ function to_upper%(str: string%): string return new StringVal(new BroString(1, upper_s, n)); %} +## Replaces non-printable characters in a string with escaped sequences. The +## mappings are: +## +## - ``NUL`` to ``\0`` +## - ``DEL`` to ``^?`` +## - values <= 26 to ``^[A-Z]`` +## - values not in *[32, 126]** to ``%XX`` +## +## If the string does not yet have a trailing NUL, one is added. +## +## str: The string to escape. +## +## Returns: The escaped string. +## +## .. bro:see:: to_string_literal escape_string function clean%(str: string%): string %{ char* s = str->AsString()->Render(); return new StringVal(new BroString(1, byte_vec(s), strlen(s))); %} +## Replaces non-printable characters in a string with escaped sequences. The +## mappings are: +## +## - ``NUL`` to ``\0`` +## - ``DEL`` to ``^?`` +## - values <= 26 to ``^[A-Z]`` +## - values not in *[32, 126]** to ``%XX`` +## +## str: The string to escape. +## +## Returns: The escaped string. +## +## .. bro:see:: clean escape_string function to_string_literal%(str: string%): string %{ char* s = str->AsString()->Render(BroString::BRO_STRING_LITERAL); return new StringVal(new BroString(1, byte_vec(s), strlen(s))); %} +## Determines whether a given string contains only ASCII characters. +## +## str: The string to examine. +## +## Returns: False if any byte value of *str* is greater than 127, and true +## otherwise. +## +## .. bro:see:: to_upper to_lower function is_ascii%(str: string%): bool %{ int n = str->Len(); @@ -591,7 +796,14 @@ function is_ascii%(str: string%): bool return new Val(1, TYPE_BOOL); %} -# Make printable version of string. +## Creates a printable version of a string. This function is the same as +## :bro:id:`clean` except that non-printable characters are removed. +## +## s: The string to escape. +## +## Returns: The escaped string. +## +## .. bro:see:: clean to_string_literal function escape_string%(s: string%): string %{ char* escstr = s->AsString()->Render(); @@ -600,7 +812,12 @@ function escape_string%(s: string%): string return val; %} -# Returns an ASCII hexadecimal representation of a string. +## Returns an ASCII hexadecimal representation of a string. +## +## s: The string to convert to hex. +## +## Returns: A copy of *s* where each byte is replaced with the corresponding +## hex nibble. function string_to_ascii_hex%(s: string%): string %{ char* x = new char[s->Len() * 2 + 1]; @@ -612,8 +829,16 @@ function string_to_ascii_hex%(s: string%): string return new StringVal(new BroString(1, (u_char*) x, s->Len() * 2)); %} -function str_smith_waterman%(s1: string, s2: string, params: sw_params%) -: sw_substring_vec +## Uses the `Smith Waterman algorithm +## `_ to find +## similar/overlapping substrings. +## +## s1: The first string. +## +## s2: The second string. +## +## Returns: The result of the Smit Waterman algorithm calculation. +function str_smith_waterman%(s1: string, s2: string, params: sw_params%) : sw_substring_vec %{ SWParams sw_params(params->AsRecordVal()->Lookup(0)->AsCount(), SWVariant(params->AsRecordVal()->Lookup(1)->AsCount())); @@ -627,6 +852,16 @@ function str_smith_waterman%(s1: string, s2: string, params: sw_params%) return result; %} +## Splits a string into substrings with the help of an index vector of cutting +## points. +## +## s: The string to split. +## +## idx: The index vector (``vector of count``) with the cutting points. +## +## Returns: A vector of strings. +## +## .. bro:see:: split split1 split_all split_n function str_split%(s: string, idx: index_vec%): string_vec %{ vector* idx_v = idx->AsVector(); @@ -655,6 +890,13 @@ function str_split%(s: string, idx: index_vec%): string_vec return result_v; %} +## Strips whitespace at both ends of a string. +## +## str: The string to strip the whitespace from. +## +## Returns: A copy of *str* with leading and trailing whitespace removed. +## +## .. bro:see:: sub gsub function strip%(str: string%): string %{ const u_char* s = str->Bytes(); @@ -678,6 +920,14 @@ function strip%(str: string%): string return new StringVal(new BroString(sp, (e - sp + 1), 1)); %} +## Generates a string of a given size and fills it with repetitions of a source +## string. +## +## len: The length of the output string. +## +## source: The string to concatenate repeatedly until *len* has been reached. +## +## Returns: A string of length *len* filled with *source*. function string_fill%(len: int, source: string%): string %{ const u_char* src = source->Bytes(); @@ -692,10 +942,15 @@ function string_fill%(len: int, source: string%): string return new StringVal(new BroString(1, byte_vec(dst), len)); %} -# Takes a string and escapes characters that would allow execution of commands -# at the shell level. Must be used before including strings in system() or -# similar calls. -# +## Takes a string and escapes characters that would allow execution of +## commands at the shell level. Must be used before including strings in +## :bro:id:`system` or similar calls. +## +## source: The string to escape. +## +## Returns: A shell-escaped version of *source*. +## +## .. bro:see:: system function str_shell_escape%(source: string%): string %{ unsigned j = 0; @@ -724,8 +979,15 @@ function str_shell_escape%(source: string%): string return new StringVal(new BroString(1, dst, j)); %} -# Returns all occurrences of the given pattern in the given string (an empty -# empty set if none). +## Finds all occurrences of a pattern in a string. +## +## str: The string to inspect. +## +## re: The pattern to look for in *str*. +## +## Returns: The set of strings in *str* that match *re*, or the empty set. +## +## .. bro:see: find_last strstr function find_all%(str: string, re: pattern%) : string_set %{ TableVal* a = new TableVal(internal_type("string_set")->AsTableType()); @@ -746,11 +1008,18 @@ function find_all%(str: string, re: pattern%) : string_set return a; %} -# Returns the last occurrence of the given pattern in the given string. -# If not found, returns an empty string. Note that this function returns -# the match that starts at the largest index in the string, which is -# not necessarily the longest match. For example, a pattern of /.*/ -# will return the final character in the string. +## Finds the last occurrence of a pattern in a string. This function returns +## the match that starts at the largest index in the string, which is not +## necessarily the longest match. For example, a pattern of ``/.*/`` will +## return the final character in the string. +## +## str: The string to inspect. +## +## re: The pattern to look for in *str*. +## +## Returns: The last string in *str* that matches *re*, or the empty string. +## +## .. bro:see: find_all strstr function find_last%(str: string, re: pattern%) : string %{ const u_char* s = str->Bytes(); @@ -766,10 +1035,16 @@ function find_last%(str: string, re: pattern%) : string return new StringVal(""); %} -# Returns a hex dump for given input data. The hex dump renders -# 16 bytes per line, with hex on the left and ASCII (where printable) -# on the right. Based on Netdude's hex editor code. -# +## Returns a hex dump for given input data. The hex dump renders 16 bytes per +## line, with hex on the left and ASCII (where printable) +## on the right. +## +## data_str: The string to dump in hex format. +## +## .. bro:see:: string_to_ascii_hex bytestring_to_hexstr +## +## .. note:: Based on Netdude's hex editor code. +## function hexdump%(data_str: string%) : string %{ From bb47289bfaa426d2b5624cd9fee2d411cabf86ea Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 30 Nov 2011 10:19:41 -0500 Subject: [PATCH 112/964] Some updates to the base DNS script. - Answers and TTLs are now vectors. - The warning that was being generated (dns_reply_seen_after_done) from transaction ID reuse is fixed. - Updated the single failing btest baseline. --- scripts/base/protocols/dns/main.bro | 56 +++++++++++-------- .../dns.log | 6 +- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro index 2580b003dd..d738e4943c 100644 --- a/scripts/base/protocols/dns/main.bro +++ b/scripts/base/protocols/dns/main.bro @@ -6,26 +6,26 @@ export { redef enum Log::ID += { LOG }; type Info: record { - ts: time &log; - uid: string &log; - id: conn_id &log; - proto: transport_proto &log; - trans_id: count &log &optional; - query: string &log &optional; - qclass: count &log &optional; - qclass_name: string &log &optional; - qtype: count &log &optional; - qtype_name: string &log &optional; - rcode: count &log &optional; - rcode_name: string &log &optional; - QR: bool &log &default=F; - AA: bool &log &default=F; - TC: bool &log &default=F; - RD: bool &log &default=F; - RA: bool &log &default=F; - Z: count &log &default=0; - TTL: interval &log &optional; - answers: set[string] &log &optional; + ts: time &log; + uid: string &log; + id: conn_id &log; + proto: transport_proto &log; + trans_id: count &log &optional; + query: string &log &optional; + qclass: count &log &optional; + qclass_name: string &log &optional; + qtype: count &log &optional; + qtype_name: string &log &optional; + rcode: count &log &optional; + rcode_name: string &log &optional; + QR: bool &log &default=F; + AA: bool &log &default=F; + TC: bool &log &default=F; + RD: bool &log &default=F; + RA: bool &log &default=F; + Z: count &log &default=0; + answers: vector of string &log &optional; + TTLs: vector of interval &log &optional; ## This value indicates if this request/response pair is ready to be logged. ready: bool &default=F; @@ -102,7 +102,13 @@ function new_session(c: connection, trans_id: count): Info function set_session(c: connection, msg: dns_msg, is_query: bool) { if ( ! c?$dns_state || msg$id !in c$dns_state$pending ) + { c$dns_state$pending[msg$id] = new_session(c, msg$id); + # Try deleting this transaction id from the set of finished answers. + # Sometimes hosts will reuse ports and transaction ids and this should + # be considered to be a legit scenario (although bad practice). + delete c$dns_state$finished_answers[msg$id]; + } c$dns = c$dns_state$pending[msg$id]; @@ -136,7 +142,10 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) c$dns$AA = msg$AA; c$dns$RA = msg$RA; - c$dns$TTL = ans$TTL; + + if ( ! c$dns?$TTLs ) + c$dns$TTLs = vector(); + c$dns$TTLs[|c$dns$TTLs|] = ans$TTL; if ( ans$answer_type == DNS_ANS ) { @@ -146,8 +155,8 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) if ( reply != "" ) { if ( ! c$dns?$answers ) - c$dns$answers = set(); - add c$dns$answers[reply]; + c$dns$answers = vector(); + c$dns$answers[|c$dns$answers|] = reply; } if ( c$dns?$answers && |c$dns$answers| == c$dns$total_answers ) @@ -164,7 +173,6 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) if ( c$dns$ready ) { Log::write(DNS::LOG, c$dns); - add c$dns_state$finished_answers[c$dns$trans_id]; # This record is logged and no longer pending. delete c$dns_state$pending[c$dns$trans_id]; } diff --git a/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log b/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log index 945960e03e..78e61070d7 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log +++ b/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log @@ -1,5 +1,5 @@ #separator \x09 #path dns -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id query qclass qclass_name qtype qtype_name rcode rcode_name QR AA TC RD RA Z TTL answers auth addl -#types time string addr port addr port enum count string count string count string count string bool bool bool bool bool count interval table table table -930613226.529070 UWkUyAuUGXf 212.180.42.100 25000 131.243.64.3 53 tcp 34798 - - - - - 0 NOERROR F F F F T 0 31337.000000 4.3.2.1 - - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id query qclass qclass_name qtype qtype_name rcode rcode_name QR AA TC RD RA Z answers TTLs auth addl +#types time string addr port addr port enum count string count string count string count string bool bool bool bool bool count vector vector table table +930613226.529070 UWkUyAuUGXf 212.180.42.100 25000 131.243.64.3 53 tcp 34798 - - - - - 0 NOERROR F F F F T 0 4.3.2.1 31337.000000 - - From 95ca102546e9a1872b1b506ff928c1e32e23d3b5 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 30 Nov 2011 09:44:59 -0600 Subject: [PATCH 113/964] Sphinx-built docs now adopt 960.css style from website --- doc/_templates/layout.html | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html index 0c58f34e17..8edeeb07b6 100644 --- a/doc/_templates/layout.html +++ b/doc/_templates/layout.html @@ -2,6 +2,7 @@ {% block extrahead %} + @@ -15,6 +16,14 @@ {% block relbar2 %}{% endblock %} {% block relbar1 %}{% endblock %} +{% block content %} +
+
+ {{ super() }} +
+
+{% endblock %} + {% block footer %} {{ super() }} {% endblock %} @@ -17,15 +17,82 @@ {% block relbar1 %}{% endblock %} {% block content %} -
-
- {{ super() }} -
-
+ +
+
+ +
+ +
+ {{ relbar() }} +
+ +
+ {% block body %} + {% endblock %} +
+
+ + +
+ +
+ +
+
+ + +
+

+ Table of Contents +

+

+

    {{toc}}
+

+
+ + {% if next %} +
+

+ Next Page +

+

+ {{ next.title }} +

+
+ {% endif %} + + {% if prev %} +
+

+ Previous Page +

+

+ {{ prev.title }} +

+
+ {% endif %} + +
+
+ +
+
+
+ + Copyright {{ copyright }}. + Last updated on {{ last_updated }}. + Created using Sphinx {{ sphinx_version }}. + +
+
+
+
+ + {% endblock %} {% block footer %} -{{ super() }} {% endblock %} diff --git a/doc/conf.py.in b/doc/conf.py.in index 8844370a07..8959c0b2c9 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -90,44 +90,20 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = 'basic' html_last_updated_fmt = '%B %d, %Y' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -html_theme_options = { -"rightsidebar": "true", -"stickysidebar": "false", -"externalrefs": "false", -"footerbgcolor": "#333", -"footertextcolor": "#ddd", -"sidebarbgcolor": "#ffffff", -#"sidebarbtncolor": "", -"sidebartextcolor": "#333", -"sidebarlinkcolor": "#2a85a7", -"relbarbgcolor": "#ffffff", -"relbartextcolor": "#333", -"relbarlinkcolor": "#2a85a7", -"bgcolor": "#ffffff", -"textcolor": "#333", -"linkcolor": "#2a85a7", -"visitedlinkcolor": "#2a85a7", -"headbgcolor": "#f0f0f0", -"headtextcolor": "#000", -"headlinkcolor": "#2a85a7", -"codebgcolor": "#FFFAE2", -#"codetextcolor": "", -"bodyfont": "Arial, Helvetica, sans-serif", -"headfont": "Palatino,'Palatino Linotype',Georgia,serif", -} +html_theme_options = { } # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". +# " v Documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. diff --git a/doc/index.rst b/doc/index.rst index ad503af50b..a5af2a4f8d 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,9 +1,11 @@ .. Bro documentation master file +================= Bro Documentation ================= -Documentation for version: |version| +Guides +------ .. toctree:: :maxdepth: 1 From 1fb58eaeb70970b73118114858c5535cf66c6af7 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Wed, 30 Nov 2011 22:40:50 -0800 Subject: [PATCH 120/964] Start documenting bro.bif. --- src/bro.bif | 106 ++++++++++++++++++++++++++++++++++++++++++++++-- src/strings.bif | 8 ++-- 2 files changed, 107 insertions(+), 7 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index a2f97356a7..bc9d4bac07 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -317,6 +317,12 @@ static int next_fmt(const char*& fmt, val_list* args, ODesc* d, int& n) } %%} +## Returns the number of elements in a container. This function works with all +## container types, i.e., sets, tables, and vectors. +## +## v: The container whose elements are counted. +## +## Returns: The number of elements in *v*. function length%(v: any%): count %{ TableVal* tv = v->Type()->Tag() == TYPE_TABLE ? v->AsTableVal() : 0; @@ -334,11 +340,25 @@ function length%(v: any%): count } %} +## Checks whether two objects reference the same internal object. This function +## uses equality comparison of C++ raw pointer values to determine if the two +## objects are the same. +## +## o1: The first object. +## +## o2: The second object. +## +## Returns: True if *o1* and *o2* are equal. function same_object%(o1: any, o2: any%): bool %{ return new Val(o1 == o2, TYPE_BOOL); %} +## Removes all elements from a set or table. +## +## v: The set or table +## +## Returns: The cleared set/table or 0 if *v* is not a set/table type. function clear_table%(v: any%): any %{ if ( v->Type()->Tag() == TYPE_TABLE ) @@ -349,6 +369,11 @@ function clear_table%(v: any%): any return 0; %} +## Returns the concatenation of the string representation of its arguments. The +## arguments can be of any type. For example, ``cat("foo", 3, T)`` returns +## ``"foo3T"``. +## +## Returns: A string concatentation of all arguments. function cat%(...%): string %{ ODesc d; @@ -361,6 +386,12 @@ function cat%(...%): string return new StringVal(s); %} +## Converts a record type name into a vector of strings, where each element is +## the name of a record field. Nested records are flattened. +## +## rt: The name of the record type. +## +## Returns: A string vector with the field names of *rt*. function record_type_to_vector%(rt: string%): string_vec %{ VectorVal* result = @@ -380,7 +411,19 @@ function record_type_to_vector%(rt: string%): string_vec return result; %} - +## Concatenates all arguments, with a separator placed between each one. This +## function is similar to :bro:id:`cat`, but places a separator between each +## given argument. If any of the variable arguments is an empty string it is +## replaced by a given default string instead. +## +## sep: The separator to place betwen each argument. +## +## def: The default string to use when an argument is the empty string. +## +## Returns: A concatenation of all arguments with *sep* between each one and +## empty strings replaced with *def*. +## +## .. bro:see:: cat string_cat cat_string_array cat_string_array_n function cat_sep%(sep: string, def: string, ...%): string %{ ODesc d; @@ -408,6 +451,46 @@ function cat_sep%(sep: string, def: string, ...%): string return new StringVal(s); %} +## Produces a formatted string à la ``printf``. The first argument is the +## *format string* and specifies how subsequent arguments are converted for +## output. It is composed of zero or more directives: ordinary characters (not +## ``%``), which are copied unchanged to the output, and conversion +## specifications, each of which fetches zero or more subsequent arguments. +## Conversion specifications begin with ``%`` and the arguments must properly +## correspond to the specifier. After the ``%``, the following characters +## may appear in sequence: +## +## - ``%``: Literal ``%`` +## +## - ``-``: Left-align field +## +## - ``[0-9]+``: The field width (< 128) +## +## - ``.``: Precision of floating point specifiers ``[efg]`` (< 128) +## +## - ``A``: Escape NUL bytes, i.e., replace ``0`` with ``\0`` +## +## - ``[DTdxsefg]``: Format specifier +## +## - ``[DT]``: ISO timestamp with microsecond precision +## +## - ``d``: Signed/Unsigned integer (using C-style ``%lld|``/``%llu`` +## for ``int``/``count``) +## +## - ``x``: Unsigned hexadecimal (using C-style ``%llx``); +## addresses/ports are converted to host-byte order +## +## - ``s``: Escaped string +## +## - ``[efg]``: Double +## +## Returns: Given no arguments, :bro:id:`fmt` returns an empty string. Given a +## non-string first argument, :bro:id:`fmt` returns the concatenation +## of all its arguments, per :bro:id:`cat`. Finally, given the wrong +## number of additional arguments for the given format specifier, +## :bro:id:`fmt` generates a run-time error. +## +## .. bro:see:: cat cat_sep string_cat cat_string_array cat_string_array_n function fmt%(...%): string %{ if ( @ARGC@ == 0 ) @@ -436,6 +519,12 @@ function fmt%(...%): string return new StringVal(s); %} + +## Returns the type name of an arbitrary Bro variable. +## +## t: An arbitrary object. +## +## Returns: The type name of *t*. function type_name%(t: any%): string %{ ODesc d; @@ -447,6 +536,11 @@ function type_name%(t: any%): string return new StringVal(s); %} +## Converts a string into a (signed) integer. +## +## str: The string to convert. +## +## Returns: The string *str* as ``int``. function to_int%(str: string%): int %{ const char* s = str->CheckString(); @@ -465,6 +559,12 @@ function to_int%(str: string%): int return new Val(i, TYPE_INT); %} + +## Converts a positive integer into a ``count``. +## +## n: The integer to convert. +## +## Returns: The int *n* as unsigned integer or 0 if *n* < 0. function int_to_count%(n: int%): count %{ if ( n < 0 ) @@ -791,9 +891,9 @@ function getenv%(var: string%): string function setenv%(var: string, val: string%): bool %{ - int result = setenv(var->AsString()->CheckString(), + int result = setenv(var->AsString()->CheckString(), val->AsString()->CheckString(), 1); - + if ( result < 0 ) return new Val(0, TYPE_BOOL); return new Val(1, TYPE_BOOL); diff --git a/src/strings.bif b/src/strings.bif index 18252bd4b4..3fec92cd7a 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -15,7 +15,7 @@ using namespace std; ## ## Returns: The concatenation of all (string) arguments. ## -## .. bro:see:: cat cat_string_array cat_string_array_n +## .. bro:see:: cat cat_sep cat_string_array cat_string_array_n ## fmt ## join_string_vec join_string_array function string_cat%(...%): string @@ -87,7 +87,7 @@ BroString* cat_string_array_n(TableVal* tbl, int start, int end) ## ## Returns: The concatenation of all elements in *a*. ## -## .. bro:see:: cat string_cat cat_string_array_n +## .. bro:see:: cat cat_sep string_cat cat_string_array_n ## fmt ## join_string_vec join_string_array function cat_string_array%(a: string_array%): string @@ -125,7 +125,7 @@ function cat_string_array_n%(a: string_array, start: count, end: count%): string ## Returns: The concatenation of all elements in *a*, with *sep* placed ## between each element. ## -## .. bro:see:: cat string_cat cat_string_array cat_string_array_n +## .. bro:see:: cat cat_sep string_cat cat_string_array cat_string_array_n ## fmt ## join_string_vec function join_string_array%(sep: string, a: string_array%): string @@ -161,7 +161,7 @@ function join_string_array%(sep: string, a: string_array%): string ## Returns: The concatenation of all elements in *a*, with *sep* placed ## between each element. ## -## .. bro:see:: cat string_cat cat_string_array cat_string_array_n +## .. bro:see:: cat cat_sep string_cat cat_string_array cat_string_array_n ## fmt ## join_string_array function join_string_vec%(vec: string_vec, sep: string%): string From f6494a09c95063f717539a19b44454c1abba3ab5 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 1 Dec 2011 09:16:38 -0600 Subject: [PATCH 121/964] Add missing doc targets to top Makefile; remove old doc/Makefile. (fixes #705) --- Makefile | 12 +++++++++ doc/CMakeLists.txt | 6 ++--- doc/Makefile | 7 ----- doc/README | 5 ++-- doc/bin/rst2html.py | 62 --------------------------------------------- doc/scripts/README | 2 +- 6 files changed, 19 insertions(+), 75 deletions(-) delete mode 100644 doc/Makefile delete mode 100755 doc/bin/rst2html.py diff --git a/Makefile b/Makefile index e0c2860873..83a149ca58 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,18 @@ doc: configured docclean: configured $(MAKE) -C $(BUILD) $@ +restdoc: configured + $(MAKE) -C $(BUILD) $@ + +restclean: configured + $(MAKE) -C $(BUILD) $@ + +broxygen: configured + $(MAKE) -C $(BUILD) $@ + +broxygenclean: configured + $(MAKE) -C $(BUILD) $@ + dist: @rm -rf $(VERSION_FULL) $(VERSION_FULL).tgz @rm -rf $(VERSION_MIN) $(VERSION_MIN).tgz diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 2f5bd93cdf..bdbb0e7b69 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -60,16 +60,16 @@ add_custom_target(broxygen # The "sphinxclean" target removes just the Sphinx input/output directories # from the build directory. -add_custom_target(broxygen-clean +add_custom_target(broxygenclean COMMAND "${CMAKE_COMMAND}" -E remove_directory ${DOC_SOURCE_WORKDIR} COMMAND "${CMAKE_COMMAND}" -E remove_directory ${DOC_OUTPUT_DIR} VERBATIM) -add_dependencies(broxygen broxygen-clean restdoc) +add_dependencies(broxygen broxygenclean restdoc) add_custom_target(doc) add_custom_target(docclean) add_dependencies(doc broxygen) -add_dependencies(docclean broxygen-clean restclean) +add_dependencies(docclean broxygenclean restclean) diff --git a/doc/Makefile b/doc/Makefile deleted file mode 100644 index 2756093a27..0000000000 --- a/doc/Makefile +++ /dev/null @@ -1,7 +0,0 @@ - -all: - test -d html || mkdir html - for i in *.rst; do echo "$$i ..."; ./bin/rst2html.py $$i >html/`echo $$i | sed 's/rst$$/html/g'`; done - -clean: - rm -rf html diff --git a/doc/README b/doc/README index a9f2cdbc4e..57d569db84 100644 --- a/doc/README +++ b/doc/README @@ -15,8 +15,9 @@ which adds some reST directives and roles that aid in generating useful index entries and cross-references. Other extensions can be added in a similar fashion. -Either the ``make doc`` or ``make broxygen`` can be used to locally -render the reST files into HTML. Those targets depend on: +Either the ``make doc`` or ``make broxygen`` targets in the top-level +Makefile can be used to locally render the reST files into HTML. +Those targets depend on: * Python interpreter >= 2.5 * `Sphinx `_ >= 1.0.1 diff --git a/doc/bin/rst2html.py b/doc/bin/rst2html.py deleted file mode 100755 index 79c835d6c4..0000000000 --- a/doc/bin/rst2html.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python -# -# Derived from docutils standard rst2html.py. -# -# $Id: rst2html.py 4564 2006-05-21 20:44:42Z wiemann $ -# Author: David Goodger -# Copyright: This module has been placed in the public domain. -# -# -# Extension: we add to dummy directorives "code" and "console" to be -# compatible with Bro's web site setup. - -try: - import locale - locale.setlocale(locale.LC_ALL, '') -except: - pass - -import textwrap - -from docutils.core import publish_cmdline, default_description - -from docutils import nodes -from docutils.parsers.rst import directives, Directive -from docutils.parsers.rst.directives.body import LineBlock - -class Literal(Directive): - #max_line_length = 68 - max_line_length = 0 - - required_arguments = 0 - optional_arguments = 1 - final_argument_whitespace = True - has_content = True - - def wrapped_content(self): - content = [] - - if Literal.max_line_length: - for line in self.content: - content += textwrap.wrap(line, Literal.max_line_length, subsequent_indent=" ") - else: - content = self.content - - return u'\n'.join(content) - - def run(self): - self.assert_has_content() - content = self.wrapped_content() - literal = nodes.literal_block(content, content) - return [literal] - -directives.register_directive('code', Literal) -directives.register_directive('console', Literal) - -description = ('Generates (X)HTML documents from standalone reStructuredText ' - 'sources. ' + default_description) - -publish_cmdline(writer_name='html', description=description) - - - diff --git a/doc/scripts/README b/doc/scripts/README index b3e44914f4..a15812609c 100644 --- a/doc/scripts/README +++ b/doc/scripts/README @@ -1,6 +1,6 @@ This directory contains scripts and templates that can be used to automate the generation of Bro script documentation. Several build targets are defined -by CMake: +by CMake and available in the top-level Makefile: ``restdoc`` From 14c1d2ae1fcb1b8e9692a0b79816d95e0bcb5325 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 1 Dec 2011 09:31:38 -0600 Subject: [PATCH 122/964] Remove example redef of SMTP::entity_excerpt_len from local.bro. --- scripts/site/local.bro | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scripts/site/local.bro b/scripts/site/local.bro index 7546a52b4b..8e1f4d802b 100644 --- a/scripts/site/local.bro +++ b/scripts/site/local.bro @@ -62,12 +62,3 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig"; @load protocols/http/detect-MHR # Detect SQL injection attacks @load protocols/http/detect-sqli - -# Uncomment this redef if you want to extract SMTP MIME entities for -# some file types. The numbers given indicate how many bytes to extract for -# the various mime types. -@load base/protocols/smtp/entities-excerpt -redef SMTP::entity_excerpt_len += { -# ["text/plain"] = 1024, -# ["text/html"] = 1024, -}; From 18d968adcd300425c2c540677d6e5a8c621467a3 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 1 Dec 2011 11:52:01 -0800 Subject: [PATCH 123/964] Adapting attribute serialization when talking to Broccoli. Broccoli doesn't support expressions, and we now no longer send them when serializing attributes. This is the Bro change mentioned in #606. It's needs a correspondinly modified Broccoli identifying itself as such, and it isn't tested yet ... Addresses #606. --- src/Attr.cc | 6 +++++- src/RemoteSerializer.cc | 11 ++++++++++- src/RemoteSerializer.h | 1 + src/SerialInfo.h | 14 ++++++++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/Attr.cc b/src/Attr.cc index a5a350f452..12e1c888af 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -481,7 +481,11 @@ bool Attributes::DoSerialize(SerialInfo* info) const loop_over_list((*attrs), i) { Attr* a = (*attrs)[i]; - SERIALIZE_OPTIONAL(a->AttrExpr()) + + // Broccoli doesn't support expressions. + Expr* e = (! info->broccoli_peer) ? a->AttrExpr() : 0; + SERIALIZE_OPTIONAL(e); + if ( ! SERIALIZE(char(a->Tag())) ) return false; } diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index a21a7abc60..600cdd5bca 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -385,6 +385,9 @@ inline void RemoteSerializer::SetupSerialInfo(SerialInfo* info, Peer* peer) peer->phase == Peer::RUNNING ) info->new_cache_strategy = true; + if ( (peer->caps & Peer::BROCCOLI_PEER) ) + info->broccoli_peer = true; + info->include_locations = false; } @@ -1457,7 +1460,7 @@ void RemoteSerializer::Finish() Poll(true); while ( io->CanWrite() ); - loop_over_list(peers, i) + loop_over_list(peers, i) { CloseConnection(peers[i]); } @@ -2113,6 +2116,9 @@ bool RemoteSerializer::HandshakeDone(Peer* peer) if ( (peer->caps & Peer::NEW_CACHE_STRATEGY) ) Log(LogInfo, "peer supports keep-in-cache; using that", peer); + if ( (peer->caps & Peer::BROCCOLI_PEER) ) + Log(LogInfo, "peer is a Broccoli", peer); + if ( peer->logs_requested ) log_mgr->SendAllWritersTo(peer->id); @@ -2365,6 +2371,9 @@ bool RemoteSerializer::ProcessSerialization() current_peer->phase == Peer::RUNNING ) info.new_cache_strategy = true; + if ( current_peer->caps & Peer::BROCCOLI_PEER ) + info.broccoli_peer = true; + if ( ! forward_remote_state_changes ) ignore_accesses = true; diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index f849a6a2b5..b64fdcbe66 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -198,6 +198,7 @@ protected: static const int NO_CACHING = 2; static const int PID_64BIT = 4; static const int NEW_CACHE_STRATEGY = 8; + static const int BROCCOLI_PEER = 16; // Constants to remember to who did something. static const int NONE = 0; diff --git a/src/SerialInfo.h b/src/SerialInfo.h index d322aa4b37..aa4c382349 100644 --- a/src/SerialInfo.h +++ b/src/SerialInfo.h @@ -15,6 +15,7 @@ public: pid_32bit = false; include_locations = true; new_cache_strategy = false; + broccoli_peer = false; } SerialInfo(const SerialInfo& info) @@ -28,6 +29,7 @@ public: pid_32bit = info.pid_32bit; include_locations = info.include_locations; new_cache_strategy = info.new_cache_strategy; + broccoli_peer = info.broccoli_peer; } // Parameters that control serialization. @@ -46,6 +48,11 @@ public: // If true, we support keeping objs in cache permanently. bool new_cache_strategy; + // If true, we're connecting to a Broccoli. If so, serialization + // specifics may be adapted for functionality Broccoli does not + // support. + bool broccoli_peer; + ChunkedIO::Chunk* chunk; // chunk written right before the serialization // Attributes set during serialization. @@ -70,6 +77,7 @@ public: print = 0; pid_32bit = false; new_cache_strategy = false; + broccoli_peer = false; } UnserialInfo(const UnserialInfo& info) @@ -86,6 +94,7 @@ public: print = info.print; pid_32bit = info.pid_32bit; new_cache_strategy = info.new_cache_strategy; + broccoli_peer = info.broccoli_peer; } // Parameters that control unserialization. @@ -106,6 +115,11 @@ public: // If true, we support keeping objs in cache permanently. bool new_cache_strategy; + // If true, we're connecting to a Broccoli. If so, serialization + // specifics may be adapted for functionality Broccoli does not + // support. + bool broccoli_peer; + // If a global ID already exits, of these policies is used. enum { Keep, // keep the old ID and ignore the new From 0c8b5a712d1d3117e19b8fb13dbd505938dba75d Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 1 Dec 2011 14:07:08 -0600 Subject: [PATCH 124/964] Add a remote_log_peer event which contains an event_peer record param. Addresses #493. --- .../base/frameworks/communication/main.bro | 7 ++++ src/RemoteSerializer.cc | 30 ++++++++++----- src/event.bif | 23 ++++++++++++ .../send.log | 17 +++++++++ .../communication_log_baseline.bro | 37 +++++++++++++++++++ 5 files changed, 104 insertions(+), 10 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log create mode 100644 testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro diff --git a/scripts/base/frameworks/communication/main.bro b/scripts/base/frameworks/communication/main.bro index 569ba140a9..01c608c8db 100644 --- a/scripts/base/frameworks/communication/main.bro +++ b/scripts/base/frameworks/communication/main.bro @@ -130,6 +130,13 @@ event remote_log(level: count, src: count, msg: string) do_script_log_common(level, src, msg); } +# This is a core generated event. +event remote_log_peer(p: event_peer, level: count, src: count, msg: string) + { + local rmsg = fmt("[#%d/%s:%d] %s", p$id, p$host, p$p, msg); + do_script_log_common(level, src, rmsg); + } + function do_script_log(p: event_peer, msg: string) { do_script_log_common(REMOTE_LOG_INFO, REMOTE_SRC_SCRIPT, msg); diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index a21a7abc60..3a24b8792a 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -2923,24 +2923,34 @@ void RemoteSerializer::Log(LogLevel level, const char* msg) void RemoteSerializer::Log(LogLevel level, const char* msg, Peer* peer, LogSrc src) { + if ( peer ) + { + val_list* vl = new val_list(); + vl->append(peer->val->Ref()); + vl->append(new Val(level, TYPE_COUNT)); + vl->append(new Val(src, TYPE_COUNT)); + vl->append(new StringVal(msg)); + mgr.QueueEvent(remote_log_peer, vl); + } + else + { + val_list* vl = new val_list(); + vl->append(new Val(level, TYPE_COUNT)); + vl->append(new Val(src, TYPE_COUNT)); + vl->append(new StringVal(msg)); + mgr.QueueEvent(remote_log, vl); + } + const int BUFSIZE = 1024; char buffer[BUFSIZE]; - int len = 0; if ( peer ) - len += snprintf(buffer + len, sizeof(buffer) - len, - "[#%d/%s:%d] ", int(peer->id), ip2a(peer->ip), - peer->port); + len += snprintf(buffer + len, sizeof(buffer) - len, "[#%d/%s:%d] ", + int(peer->id), ip2a(peer->ip), peer->port); len += safe_snprintf(buffer + len, sizeof(buffer) - len, "%s", msg); - val_list* vl = new val_list(); - vl->append(new Val(level, TYPE_COUNT)); - vl->append(new Val(src, TYPE_COUNT)); - vl->append(new StringVal(buffer)); - mgr.QueueEvent(remote_log, vl); - DEBUG_COMM(fmt("parent: %.6f %s", current_time(), buffer)); } diff --git a/src/event.bif b/src/event.bif index d953ac78fe..0c2f7eb780 100644 --- a/src/event.bif +++ b/src/event.bif @@ -444,6 +444,29 @@ event remote_state_inconsistency%(operation: string, id: string, # Generated for communication log message. event remote_log%(level: count, src: count, msg: string%); +## Generated for communication log messages. While this event is +## intended primarily for use by Bro's communication framework, it can also trigger +## additional code if helpful. This event is equivalent to +## :bro:see:`remote_log` except the message is with respect to a certain peer. +## +## p: A record describing the remote peer. +## +## level: The log level, which is either :bro:enum:`REMOTE_LOG_INFO` or +## :bro:enum:`REMOTE_LOG_ERROR`. +## +## src: The component of the comminication system that logged the message. +## Currently, this will be one of :bro:enum:`REMOTE_SRC_CHILD` (Bro's +## child process), :bro:enum:`REMOTE_SRC_PARENT` (Bro's main process), or +## :bro:enum:`REMOTE_SRC_SCRIPT` (the script level). +## +## msg: The message logged. +## +## .. bro:see:: remote_capture_filter remote_connection_closed remote_connection_error +## remote_connection_established remote_connection_handshake_done +## remote_event_registered remote_pong remote_state_access_performed +## remote_state_inconsistency print_hook remote_log +event remote_log_peer%(p: event_peer, level: count, src: count, msg: string%); + # Generated when a remote peer has answered to our ping. event remote_pong%(p: event_peer, seq: count, d1: interval, d2: interval, d3: interval%); diff --git a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log new file mode 100644 index 0000000000..9cf441d61a --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log @@ -0,0 +1,17 @@ +#separator \x09 +#path communication +#fields ts peer src_name connected_peer_desc connected_peer_addr connected_peer_port level message +#types time string string string addr port string string +1322759704.176437 bro parent - - - info raised pipe's socket buffer size from 8K to 1024K +1322759704.176437 bro parent - - - info [#1/127.0.0.1:47757] added peer +1322759704.183341 bro child - - - info [#1/127.0.0.1:47757] connected +1322759704.183738 bro parent - - - info [#1/127.0.0.1:47757] peer connected +1322759704.183738 bro parent - - - info [#1/127.0.0.1:47757] phase: version +1322759704.184034 bro script - - - info connection established +1322759704.184034 bro script - - - info requesting events matching /^?(NOTHING)$?/ +1322759704.184034 bro script - - - info accepting state +1322759704.185120 bro parent - - - info [#1/127.0.0.1:47757] phase: handshake +1322759704.185120 bro parent - - - info warning: no events to request +1322759704.185120 bro parent - - - info terminating... +1322759704.185120 bro parent - - - info [#1/127.0.0.1:47757] peer_description is bro +1322759704.185120 bro parent - - - info [#1/127.0.0.1:47757] closing connection diff --git a/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro b/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro new file mode 100644 index 0000000000..74ff297010 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro @@ -0,0 +1,37 @@ +# +# @TEST-EXEC: btest-bg-run receiver bro -b ../receiver.bro +# @TEST-EXEC: btest-bg-run sender bro -b ../sender.bro +# @TEST-EXEC: btest-bg-wait -k 2 +# +# Don't diff the receiver log just because port is always going to change +# @TEST-EXEC: grep -v pid sender/communication.log >send.log +# @TEST-EXEC: btest-diff send.log + +@TEST-START-FILE sender.bro + +@load base/frameworks/communication/main + +redef Communication::nodes += { + ["foo"] = [$host = 127.0.0.1, $events = /NOTHING/, $connect=T] +}; + +event remote_connection_established(p: event_peer) + { + terminate_communication(); + terminate(); + } + +@TEST-END-FILE + +############# + +@TEST-START-FILE receiver.bro + +@load frameworks/communication/listen + +event remote_connection_closed(p: event_peer) + { + terminate(); + } + +@TEST-END-FILE From edc0a451f8d72c2f14990498ba105047aecca0a5 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 1 Dec 2011 16:18:56 -0600 Subject: [PATCH 125/964] Teach LogWriterAscii to use BRO_LOG_SUFFIX env. var. (addresses #704) --- src/LogWriterAscii.cc | 11 ++++++++--- src/LogWriterAscii.h | 1 + src/main.cc | 2 ++ .../scripts/base/frameworks/logging/env-ext.test | 2 ++ 4 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 testing/btest/scripts/base/frameworks/logging/env-ext.test diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 5bd476c936..9b1fda3b62 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -88,7 +88,7 @@ bool LogWriterAscii::DoInit(string path, int num_fields, if ( output_to_stdout ) path = "/dev/stdout"; - fname = IsSpecial(path) ? path : path + ".log"; + fname = IsSpecial(path) ? path : path + "." + LogExt(); if ( ! (file = fopen(fname.c_str(), "w")) ) { @@ -320,7 +320,7 @@ bool LogWriterAscii::DoRotate(string rotated_path, double open, fclose(file); file = 0; - string nname = rotated_path + ".log"; + string nname = rotated_path + "." + LogExt(); rename(fname.c_str(), nname.c_str()); if ( ! FinishedRotation(nname, fname, open, close, terminating) ) @@ -338,4 +338,9 @@ bool LogWriterAscii::DoSetBuf(bool enabled) return true; } - +string LogWriterAscii::LogExt() + { + const char* ext = getenv("BRO_LOG_SUFFIX"); + if ( ! ext ) ext = "log"; + return ext; + } diff --git a/src/LogWriterAscii.h b/src/LogWriterAscii.h index 7755f71d06..72127c8b1f 100644 --- a/src/LogWriterAscii.h +++ b/src/LogWriterAscii.h @@ -13,6 +13,7 @@ public: ~LogWriterAscii(); static LogWriter* Instantiate() { return new LogWriterAscii; } + static string LogExt(); protected: virtual bool DoInit(string path, int num_fields, diff --git a/src/main.cc b/src/main.cc index dfa46c3050..b4a27862c9 100644 --- a/src/main.cc +++ b/src/main.cc @@ -47,6 +47,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "ConnCompressor.h" #include "DPM.h" #include "BroDoc.h" +#include "LogWriterAscii.h" #include "binpac_bro.h" @@ -194,6 +195,7 @@ void usage() fprintf(stderr, " $BRO_PREFIXES | prefix list (%s)\n", bro_prefixes()); fprintf(stderr, " $BRO_DNS_FAKE | disable DNS lookups (%s)\n", bro_dns_fake()); fprintf(stderr, " $BRO_SEED_FILE | file to load seeds from (not set)\n"); + fprintf(stderr, " $BRO_LOG_SUFFIX | ASCII log file extension (.%s)\n", LogWriterAscii::LogExt().c_str()); exit(1); } diff --git a/testing/btest/scripts/base/frameworks/logging/env-ext.test b/testing/btest/scripts/base/frameworks/logging/env-ext.test new file mode 100644 index 0000000000..e9f690caa4 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/env-ext.test @@ -0,0 +1,2 @@ +# @TEST-EXEC: BRO_LOG_SUFFIX=txt bro -r $TRACES/wikipedia.trace +# @TEST-EXEC: test -f conn.txt From e8a25ee68fbdca4950ec737258da1d7e0ee06a82 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 1 Dec 2011 15:49:10 -0800 Subject: [PATCH 126/964] Updating CHANGES and VERSION. --- CHANGES | 5 +++++ VERSION | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 9474645544..6b0a0d5a70 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.0-beta-72 | 2011-11-30 20:16:09 -0800 + + * Fine-tuning the Sphinx layout to better match www. (Jon Siwek and + Robin Sommer) + 2.0-beta-69 | 2011-11-29 16:55:31 -0800 * Fixing ASCII logger to escape the unset-field place holder if diff --git a/VERSION b/VERSION index 0ce06b2179..b46b52a627 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0-beta-69 +2.0-beta-72 From 30399d86e9bd43892b31cdadd08606a51f02c4d6 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Thu, 1 Dec 2011 20:42:05 -0800 Subject: [PATCH 127/964] Document some attributes. I am not going to edit the attributes any further, please go ahead and text to the missing TODOs. --- doc/scripts/builtins.rst | 50 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/doc/scripts/builtins.rst b/doc/scripts/builtins.rst index e6a9e9b829..6abbd9110b 100644 --- a/doc/scripts/builtins.rst +++ b/doc/scripts/builtins.rst @@ -67,52 +67,98 @@ The Bro scripting language supports the following built-in types. Attributes ---------- +Attributes occur at the end of type/event declarations and change their +behavior. The syntax is ``&key`` or ``&key=val``, e.g., +``type T: set[count] &read_expire=5min`` or ``event foo() &priority=-3``. The Bro scripting language supports the following built-in attributes. -.. TODO: add documentation - +## Allows record field to be missing. For example the type +## ``record { a: int, b: port &optional }`` could be instantiated both as +## singleton ``[$a=127.0.0.1]`` or pair ``[$a=127.0.0.1, $b=80/tcp]``. .. bro:attr:: &optional +## Uses a default value for a record field or container elements. For example, +## ``table[int] of string &default="foo" }`` would create table that returns +## The :bro:type:`string` ``"foo"`` for any non-existing index. .. bro:attr:: &default +## Allows for redefinition of initial object values. This is typically used +## with constants, for example, ``const clever = T &redef;`` would allow the +## constant to be redifined at some later point during script execution. .. bro:attr:: &redef +## Rotates a file after a specified interval. .. bro:attr:: &rotate_interval +## Rotates af file after it has reached a given size in bytes. .. bro:attr:: &rotate_size +## ..TODO: needs to be documented. .. bro:attr:: &add_func +## ..TODO: needs to be documented. .. bro:attr:: &delete_func +## Called right before a container element expires. .. bro:attr:: &expire_func +## Specifies a read expiration timeout for container elements. That is, the +## element expires after the given amount of time since the last time it has +## been read. Note that a write also counts as a read. .. bro:attr:: &read_expire +## Specifies a write expiration timeout for container elements. That is, the +## element expires after the given amount of time since the last time it has +## been written. .. bro:attr:: &write_expire +## Specifies a creation expiration timeout for container elements. That is, the +## element expires after the given amount of time since it has been inserted +## into the container, regardless of any reads or writes. .. bro:attr:: &create_expire +## Makes a variable persistent, i.e., its value is writen to disk (per default +## at shutdown time). .. bro:attr:: &persistent +## Synchronizes variable accesses across nodes. The value of a +## ``&synchronized`` variable is automatically propagated to all peers when it +## changes. .. bro:attr:: &synchronized +## ..TODO: needs to be documented. .. bro:attr:: &postprocessor +## Encryptes files right before writing them to disk. +## ..TODO: needs to be documented in more detail. .. bro:attr:: &encrypt +## ..TODO: needs to be documented. .. bro:attr:: &match +## Deprecated. Will be removed. .. bro:attr:: &disable_print_hook +## Opens a file in raw mode, i.e., non-ASCII characters are not escaped. .. bro:attr:: &raw_output +## Prefers set union to assignment for synchronized state. This attribute is +## used in conjunction with :bro:attr:`synchronized` container types: when the +## same container is updated at two peers with different value, the propagation +## of the state causes a race condition, where the last update succeeds. This +## can cause inconsistencies and can be avoided by unifying the two sets, +## rather than merely overwriting the old value. .. bro:attr:: &mergeable +## Specifies the execution priority of an event handler. Higher values are +## executed before lower ones. The default value is 0. .. bro:attr:: &priority +## Groups event handlers such that those in the same group can be jointly +## activated or deactivated. .. bro:attr:: &group +## Writes a record field to the associated log stream. .. bro:attr:: &log .. bro:attr:: (&tracked) From 25bb69c3af04299176c3bef563de5d539dd93428 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Thu, 1 Dec 2011 20:43:11 -0800 Subject: [PATCH 128/964] More BiF documentation. --- src/bro.bif | 261 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 248 insertions(+), 13 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index bc9d4bac07..3390f322a8 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -386,7 +386,7 @@ function cat%(...%): string return new StringVal(s); %} -## Converts a record type name into a vector of strings, where each element is +## Converts a record type name to a vector of strings, where each element is ## the name of a record field. Nested records are flattened. ## ## rt: The name of the record type. @@ -536,11 +536,13 @@ function type_name%(t: any%): string return new StringVal(s); %} -## Converts a string into a (signed) integer. +## Converts a :bro:type:`string` to a :bro:type:`int`. ## -## str: The string to convert. +## str: The :bro:type:`string` to convert. ## -## Returns: The string *str* as ``int``. +## Returns: The :bro:type:`string` *str* as :bro:type:`int`. +## +## .. bro:see:: to_addr to_port function to_int%(str: string%): int %{ const char* s = str->CheckString(); @@ -560,11 +562,11 @@ function to_int%(str: string%): int %} -## Converts a positive integer into a ``count``. +## Converts a (positive) :bro:type:`int` to a :bro:type:`count`. ## -## n: The integer to convert. +## n: The :bro:type:`int` to convert. ## -## Returns: The int *n* as unsigned integer or 0 if *n* < 0. +## Returns: The :bro:type:`int` *n* as unsigned integer or 0 if *n* < 0. function int_to_count%(n: int%): count %{ if ( n < 0 ) @@ -575,6 +577,13 @@ function int_to_count%(n: int%): count return new Val(n, TYPE_COUNT); %} +## Converts a :bro:type:`double` to a :bro:type:`count`. +## +## d: The :bro:type:`double` to convert. +## +## Returns: The :bro:type:`double` *d* as unsigned integer or 0 if *d* < 0.0. +## +## .. bro:see:: double_to_time function double_to_count%(d: double%): count %{ if ( d < 0.0 ) @@ -583,6 +592,14 @@ function double_to_count%(d: double%): count return new Val(bro_uint_t(rint(d)), TYPE_COUNT); %} +## Converts a :bro:type:`string` to a :bro:type:`count`. +## +## str: The :bro:type:`string` to convert. +## +## Returns: The :bro:type:`string` *str* as unsigned integer or if in invalid +## format. +## +## .. bro:see:: to_addr to_int to_port function to_count%(str: string%): count %{ const char* s = str->CheckString(); @@ -599,26 +616,61 @@ function to_count%(str: string%): count return new Val(u, TYPE_COUNT); %} +## Converts an :bro:type:`interval` to a :bro:type:`double`. +## +## i: The :bro:type:`interval` to convert. +## +## Returns: The :bro:type:`interval` *i* as :bro:type:`double`. +## +## .. bro:see:: double_to_interval function interval_to_double%(i: interval%): double %{ return new Val(i, TYPE_DOUBLE); %} +## Converts a :bro:type:`time` value to a :bro:type:`double`. +## +## t: The :bro:type:`interval` to convert. +## +## Returns: The :bro:type:`time` value *t* as :bro:type:`double`. +## +## .. bro:see:: double_to_time function time_to_double%(t: time%): double %{ return new Val(t, TYPE_DOUBLE); %} +## Converts a :bro:type:`time` value to a :bro:type:`double`. +## +## t: The :bro:type:`interval` to convert. +## +## Returns: The :bro:type:`time` value *t* as :bro:type:`double`. +## +## .. bro:see:: time_to_double double_to_count function double_to_time%(d: double%): time %{ return new Val(d, TYPE_TIME); %} +## Converts a :bro:type:`double` to an :bro:type:`interval`. +## +## d: The :bro:type:`double` to convert. +## +## Returns: The :bro:type:`double` *d* as :bro:type:`interval`. +## +## .. bro:see:: interval_to_double function double_to_interval%(d: double%): interval %{ return new Val(d, TYPE_INTERVAL); %} +## Converts a :bro:type:`addr` to a :bro:type:`count`. +## +## a: The :bro:type:`addr` to convert. +## +## Returns: The :bro:type:`addr` *a* as :bro:type:`count`. +## +## .. bro:see:: addr_to_ptr_name function addr_to_count%(a: addr%): count %{ #ifdef BROv6 @@ -635,21 +687,50 @@ function addr_to_count%(a: addr%): count return new Val(ntohl(addr), TYPE_COUNT); %} +## Converts a :bro:type:`addr` to a :bro:type:`count`. +## +## a: The :bro:type:`addr` to convert. +## +## Returns: The :bro:type:`addr` *a* as :bro:type:`count`. +## +## .. bro:see:: count_to_port function port_to_count%(p: port%): count %{ return new Val(p->Port(), TYPE_COUNT); %} +## Converts a :bro:type:`count` and ``transport_proto`` to a :bro:type:`port`. +## +## a: The :bro:type:`addr` to convert. +## +## Returns: The :bro:type:`addr` *a* as :bro:type:`count`. +## +## .. bro:see:: port_to_count function count_to_port%(c: count, t: transport_proto%): port %{ return new PortVal(c, (TransportProto)(t->InternalInt())); %} +## Chops off any decimal digits of the given double, i.e., computes the +## "floor" of it. For example, ``floor(3.14)`` returns ``3.0``. +## +## d: The :bro:type:`double` to manipulate. +## +## Returns: The next lowest integer of *d* as :bro:type:`double`. +## +## .. bro:see:: sqrt exp ln log10 function floor%(d: double%): double %{ return new Val(floor(d), TYPE_DOUBLE); %} +## Converts a :bro:type:`string` to an :bro:type:`addr`. +## +## ip: The :bro:type:`string` to convert. +## +## Returns: The :bro:type:`string` *ip* as :bro:type:`addr`. +## +## .. bro:see:: to_count to_int to_port count_to_v4_addr raw_bytes_to_v4_addr function to_addr%(ip: string%): addr %{ char* s = ip->AsString()->Render(); @@ -658,6 +739,13 @@ function to_addr%(ip: string%): addr return ret; %} +## Converts a :bro:type:`count` to an :bro:type:`addr`. +## +## ip: The :bro:type:`count` to convert. +## +## Returns: The :bro:type:`count` *ip* as :bro:type:`addr`. +## +## .. bro:see:: raw_bytes_to_v4_addr to_addr function count_to_v4_addr%(ip: count%): addr %{ if ( ip > 4294967295LU ) @@ -669,7 +757,15 @@ function count_to_v4_addr%(ip: count%): addr return new AddrVal(htonl(uint32(ip))); %} -# Interprets the first 4 bytes of 'b' as an IPv4 address in network order. +## Converts a :bro:type:`string` of bytes into an IP address. In particular, +## this function interprets the first 4 bytes of the string as an IPv4 address +## in network order. +## +## b: The raw bytes (:bro:type:`string`) to convert. +## +## Returns: The byte :bro:type:`string` *ip* as :bro:type:`addr`. +## +## .. bro:see:: raw_bytes_to_v4_addr to_addr function raw_bytes_to_v4_addr%(b: string%): addr %{ uint32 a = 0; @@ -686,20 +782,52 @@ function raw_bytes_to_v4_addr%(b: string%): addr return new AddrVal(htonl(a)); %} +## Creates a :bro:type:`port` from a given number and transport protocol. +## +## num: The port number. +## +## proto: THe transport protocol of the port. +## +## Returns: A :bro:type:`port` with number *num* and transport protocol +## *proto*. +## +## .. bro:see:: to_addr to_count to_int function to_port%(num: count, proto: transport_proto%): port %{ return new PortVal(num, (TransportProto)proto->AsEnum()); %} +## Masks an address down to the number of given upper bits. For example, +## ``mask_addr(1.2.3.4, 18)`` returns ``1.2.0.0``. +## +## a: The address to mask. +## +## top_bits_to_keep: The number of top bits to keep in *a*; must be greater +## than 0 and less than 33. +## +## Returns: The address *a* masked down to *top_bits_to_keep* bits. +## +## .. bro:see:: remask_addr function mask_addr%(a: addr, top_bits_to_keep: count%): subnet %{ return new SubNetVal(mask_addr(a, top_bits_to_keep), top_bits_to_keep); %} -# Take some top bits (e.g. subnet address) from a1 and the other -# bits (intra-subnet part) from a2 and merge them to get a new address. -# This is useful for anonymizing at subnet level while preserving -# serial scans. +## Takes some top bits (e.g., subnet address) from one address and the other +## bits (intra-subnet part) from a second address and merges them to get a new +## address. This is useful for anonymizing at subnet level while preserving +## serial scans. +## +## a1: The address to mask with *top_bits_from_a1*. +## +## a2: The address to take the remaining bits from. +## +## top_bits_from_a1: The number of top bits to keep in *a1*; must be greater +## than 0 and less than 33. +## +## Returns: The address *a* masked down to *top_bits_to_keep* bits. +## +## .. bro:see:: mask_addr function remask_addr%(a1: addr, a2: addr, top_bits_from_a1: count%): addr %{ #ifdef BROv6 @@ -720,31 +848,75 @@ function remask_addr%(a1: addr, a2: addr, top_bits_from_a1: count%): addr (x2 ^ mask_addr(x2, top_bits_from_a1)) ); %} +## Checks whether a given :bro:type:`port` has TCP as transport protocol. +## +## p: The :bro:type:`port` to check. +## +## Returns: True iff *p* is a TCP port. +## +## .. bro:see:: is_udp_port is_icmp_port function is_tcp_port%(p: port%): bool %{ return new Val(p->IsTCP(), TYPE_BOOL); %} +## Checks whether a given :bro:type:`port` has UDP as transport protocol. +## +## p: The :bro:type:`port` to check. +## +## Returns: True iff *p* is a UDP port. +## +## .. bro:see:: is_icmp_port is_tcp_port function is_udp_port%(p: port%): bool %{ return new Val(p->IsUDP(), TYPE_BOOL); %} +## Checks whether a given :bro:type:`port` has ICMP as transport protocol. +## +## p: The :bro:type:`port` to check. +## +## Returns: True iff *p* is a ICMP port. +## +## .. bro:see:: is_tcp_port is_udp_port function is_icmp_port%(p: port%): bool %{ return new Val(p->IsICMP(), TYPE_BOOL); %} +## Checks whether Bro reads traffic from one or more network interfaces (as +## opposed to from a network trace in a file). Note that this function returns +## true even after Bro has stopped reading network traffic, for example due to +## receiving a termination signal. +## +## Returns: True if reading traffic from a network interface. +## +## .. bro:see:: reading_traces function reading_live_traffic%(%): bool %{ return new Val(reading_live, TYPE_BOOL); %} +## Checks whether Bro reads traffic from a trace file (as opposed to from a +## network interface). +## +## Returns: True if reading traffic from a network trace. +## +## .. bro:see:: reading_live_traffic function reading_traces%(%): bool %{ return new Val(reading_traces, TYPE_BOOL); %} +## Opens a file for writing. If a file with the same name already exists, this +## function overwrites it (as opposed to :bro:id:`open_for_append`). +## +## f: The path to the file. +## +## Returns: A :bro:type:`file` handle for subsequent operations. +## +## .. bro:see;: active_file open_for_append close write_file +## get_file_name set_buf flush_all mkdir enable_raw_output function open%(f: string%): file %{ const char* file = f->CheckString(); @@ -755,16 +927,44 @@ function open%(f: string%): file return new Val(new BroFile(file, "w")); %} +## Opens a file for writing or appending. If a file with the same name already +## exists, this function appends to it (as opposed to :bro:id:`open`). +## +## f: The path to the file. +## +## Returns: A :bro:type:`file` handle for subsequent operations. +## +## .. bro:see;: active_file open close write_file +## get_file_name set_buf flush_all mkdir enable_raw_output function open_for_append%(f: string%): file %{ return new Val(new BroFile(f->CheckString(), "a")); %} +## Closes an open file and flushes any buffered content. +## exists, this function appends to it (as opposed to :bro:id:`open`). +## +## f: A :bro:type:`file` handle to an open file. +## +## Returns: True on success. +## +## .. bro:see;: active_file open open_for_append write_file +## get_file_name set_buf flush_all mkdir enable_raw_output function close%(f: file%): bool %{ return new Val(f->Close(), TYPE_BOOL); %} +## Writes data to an open file. +## +## f: A :bro:type:`file` handle to an open file. +## +## data: The data to write to *f*. +## +## Returns: True on success. +## +## .. bro:see;: active_file open open_for_append close +## get_file_name set_buf flush_all mkdir enable_raw_output function write_file%(f: file, data: string%): bool %{ if ( ! f ) @@ -774,17 +974,43 @@ function write_file%(f: file, data: string%): bool TYPE_BOOL); %} +## Alters the buffering behavior of a file. +## +## f: A :bro:type:`file` handle to an open file. +## +## buffered: When true, *f* is fully buffered, i.e., bytes are saved in a +## buffered until the block size has been reached. When +## false, *f* is line buffered, i.e., bytes are saved up until a +## newline occurs. +## +## .. bro:see;: active_file open open_for_append close +## get_file_name write_file flush_all mkdir enable_raw_output function set_buf%(f: file, buffered: bool%): any %{ f->SetBuf(buffered); return new Val(0, TYPE_VOID); %} +## Flushes all open files to disk. +## +## Returns: True on success. +## +## .. bro:see;: active_file open open_for_append close +## get_file_name write_file set_buf mkdir enable_raw_output function flush_all%(%): bool %{ return new Val(fflush(0) == 0, TYPE_BOOL); %} +## Creates a new directory. +## +## f: The directory name. +## +## Returns: Returns true if the operation succeeds and false if the +## creation fails or if *f* exists already. +## +## .. bro:see;: active_file open_for_append close write_file +## get_file_name set_buf flush_all enable_raw_output function mkdir%(f: string%): bool %{ const char* filename = f->CheckString(); @@ -797,6 +1023,13 @@ function mkdir%(f: string%): bool return new Val(1, TYPE_BOOL); %} +## Checks whether a given file is open. +## +## f: The file to check. +## +## Returns: True if *f* is an open :bro:type:`file`. +## +## .. TODO:: Rename to ``is_open``. function active_file%(f: file%): bool %{ return new Val(f->IsOpen(), TYPE_BOOL); @@ -1323,6 +1556,8 @@ function ptr_name_to_addr%(s: string%): addr return new AddrVal(htonl(addr)); %} +## +## .. bro:see:: addr_to_count function addr_to_ptr_name%(a: addr%): string %{ // ## Question: @@ -3094,7 +3329,7 @@ function disable_analyzer%(cid: conn_id, aid: count%) : bool return new Val(1, TYPE_BOOL); %} -# Translate analyzer type into an ASCII string. +# Translate analyzer type to an ASCII string. function analyzer_name%(aid: count%) : string %{ return new StringVal(Analyzer::GetTagName((AnalyzerTag::Tag) aid)); From a43ed633a493e5666d15ec6ca74b14ab65d642ae Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 2 Dec 2011 13:59:33 -0600 Subject: [PATCH 129/964] Changes to Broxygen master script package index - Now only lists packages as those directories in the script hierarchy that contain an __load__.bro file. - Script packages (dirs with a __load__.bro file), can now include a README (in reST format) that will automatically be appended under the link to a specific package in the master package index. --- doc/bin/group_index_generator.py | 1 + doc/scripts/CMakeLists.txt | 24 +++++++++++++++--------- doc/scripts/packages.rst | 5 ----- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/doc/bin/group_index_generator.py b/doc/bin/group_index_generator.py index e9e17f3ca9..720081e8b5 100755 --- a/doc/bin/group_index_generator.py +++ b/doc/bin/group_index_generator.py @@ -49,6 +49,7 @@ with open(group_list, 'r') as f_group_list: if not os.path.exists(os.path.dirname(group_file)): os.makedirs(os.path.dirname(group_file)) with open(group_file, 'w') as f_group_file: + f_group_file.write(":orphan:\n\n") title = "Package Index: %s\n" % os.path.dirname(group) f_group_file.write(title); for n in range(len(title)): diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt index c14eab6d25..33d473b005 100644 --- a/doc/scripts/CMakeLists.txt +++ b/doc/scripts/CMakeLists.txt @@ -73,12 +73,14 @@ macro(REST_TARGET srcDir broInput) elseif (${extension} STREQUAL ".bif.bro") set(group bifs) elseif (relDstDir) - set(pkgIndex ${relDstDir}/index) - set(group ${pkgIndex}) + set(group ${relDstDir}/index) # add package index to master package list if not already in it - list(FIND MASTER_PKG_LIST ${pkgIndex} _found) + # and if a __load__.bro exists in the original script directory + list(FIND MASTER_PKG_LIST ${relDstDir} _found) if (_found EQUAL -1) - list(APPEND MASTER_PKG_LIST ${pkgIndex}) + if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${relDstDir}/__load__.bro) + list(APPEND MASTER_PKG_LIST ${relDstDir}) + endif () endif () else () set(group "") @@ -137,11 +139,15 @@ file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}") # policy/packages.rst file set(MASTER_PKG_INDEX_TEXT "") foreach (pkg ${MASTER_PKG_LIST}) - # strip of the trailing /index for the link name - get_filename_component(lnktxt ${pkg} PATH) - # pretty-up the link name by removing common scripts/ prefix - string(REPLACE "scripts/" "" lnktxt "${lnktxt}") - set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n ${lnktxt} <${pkg}>") + set(MASTER_PKG_INDEX_TEXT + "${MASTER_PKG_INDEX_TEXT}\n:doc:`${pkg} <${pkg}/index>`\n") + if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README) + file(STRINGS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README pkgreadme) + foreach (line ${pkgreadme}) + set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n ${line}") + endforeach () + set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n") + endif () endforeach () file(WRITE ${MASTER_PACKAGE_INDEX} "${MASTER_PKG_INDEX_TEXT}") diff --git a/doc/scripts/packages.rst b/doc/scripts/packages.rst index 47e974b0c8..56909ffbc6 100644 --- a/doc/scripts/packages.rst +++ b/doc/scripts/packages.rst @@ -10,8 +10,3 @@ script, it supports being loaded in mass as a whole directory for convenience. Packages/scripts in the ``base/`` directory are all loaded by default, while ones in ``policy/`` provide functionality and customization options that are more appropriate for users to decide whether they'd like to load it or not. - -.. toctree:: - :maxdepth: 1 - - From f59c76685848c18ccc76a7570bd3a63060fad396 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 2 Dec 2011 17:00:08 -0800 Subject: [PATCH 130/964] Portability fix for new patch. --- .../send.log | 25 +++++++++---------- .../communication_log_baseline.bro | 2 +- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log index 9cf441d61a..7f71757ca0 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log +++ b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log @@ -2,16 +2,15 @@ #path communication #fields ts peer src_name connected_peer_desc connected_peer_addr connected_peer_port level message #types time string string string addr port string string -1322759704.176437 bro parent - - - info raised pipe's socket buffer size from 8K to 1024K -1322759704.176437 bro parent - - - info [#1/127.0.0.1:47757] added peer -1322759704.183341 bro child - - - info [#1/127.0.0.1:47757] connected -1322759704.183738 bro parent - - - info [#1/127.0.0.1:47757] peer connected -1322759704.183738 bro parent - - - info [#1/127.0.0.1:47757] phase: version -1322759704.184034 bro script - - - info connection established -1322759704.184034 bro script - - - info requesting events matching /^?(NOTHING)$?/ -1322759704.184034 bro script - - - info accepting state -1322759704.185120 bro parent - - - info [#1/127.0.0.1:47757] phase: handshake -1322759704.185120 bro parent - - - info warning: no events to request -1322759704.185120 bro parent - - - info terminating... -1322759704.185120 bro parent - - - info [#1/127.0.0.1:47757] peer_description is bro -1322759704.185120 bro parent - - - info [#1/127.0.0.1:47757] closing connection +1322788789.351248 bro parent - - - info [#1/127.0.0.1:47757] added peer +1322788789.354851 bro child - - - info [#1/127.0.0.1:47757] connected +1322788789.354956 bro parent - - - info [#1/127.0.0.1:47757] peer connected +1322788789.354956 bro parent - - - info [#1/127.0.0.1:47757] phase: version +1322788789.355429 bro script - - - info connection established +1322788789.355429 bro script - - - info requesting events matching /^?(NOTHING)$?/ +1322788789.355429 bro script - - - info accepting state +1322788789.355967 bro parent - - - info [#1/127.0.0.1:47757] phase: handshake +1322788789.355967 bro parent - - - info warning: no events to request +1322788789.355967 bro parent - - - info terminating... +1322788789.355967 bro parent - - - info [#1/127.0.0.1:47757] peer_description is bro +1322788789.355967 bro parent - - - info [#1/127.0.0.1:47757] closing connection diff --git a/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro b/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro index 74ff297010..c3078684af 100644 --- a/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro +++ b/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro @@ -4,7 +4,7 @@ # @TEST-EXEC: btest-bg-wait -k 2 # # Don't diff the receiver log just because port is always going to change -# @TEST-EXEC: grep -v pid sender/communication.log >send.log +# @TEST-EXEC: egrep -v 'pid|socket buffer size' sender/communication.log >send.log # @TEST-EXEC: btest-diff send.log @TEST-START-FILE sender.bro From 89f4e44f6ab5b2e98f80aeab46418ff5e4f09b01 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 2 Dec 2011 17:00:58 -0800 Subject: [PATCH 131/964] Updating submodule(s). [nomail] --- CHANGES | 21 +++++++++++++++++++++ VERSION | 2 +- aux/bro-aux | 2 +- aux/broctl | 2 +- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 6b0a0d5a70..4f34ec88b6 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,25 @@ +2.0-beta-88 | 2011-12-02 17:00:58 -0800 + + * Teach LogWriterAscii to use BRO_LOG_SUFFIX environemt variable. + Addresses #704. (Jon Siwek) + + * Fix double-free of DNS_Mgr_Request object. Addresses #661. + + * Add a remote_log_peer event which comes with an event_peer record + parameter. Addresses #493. (Jon Siwek) + + * Remove example redef of SMTP::entity_excerpt_len from local.bro. + Fixes error emitted when loading local.bro in bare mode. (Jon + Siwek) + + * Add missing doc targets to top Makefile; remove old doc/Makefile. + Fixes #705. (Jon Siwek) + + * Turn some globals into constants. Addresses #633. (Seth Hall) + + * Rearrange packet filter and DPD documentation. (Jon Siwek) + 2.0-beta-72 | 2011-11-30 20:16:09 -0800 * Fine-tuning the Sphinx layout to better match www. (Jon Siwek and diff --git a/VERSION b/VERSION index b46b52a627..b436bdb2cd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0-beta-72 +2.0-beta-88 diff --git a/aux/bro-aux b/aux/bro-aux index 7ea5837b4b..4a8551ae52 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 7ea5837b4ba8403731ca4a9875616c0ab501342f +Subproject commit 4a8551ae52d52c395b366a4eb68e63356e01999e diff --git a/aux/broctl b/aux/broctl index 6771d28af2..919eda0c1c 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 6771d28af299f025a701e67f51311513af1cbc22 +Subproject commit 919eda0c1c6ce2681de6d1b975b73313b834353d From 5a58053ef10a36d003b1e7cd1e26dd7fd42eb4e4 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 2 Dec 2011 17:19:47 -0800 Subject: [PATCH 132/964] Updating submodule(s). [nomail] --- aux/broctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broctl b/aux/broctl index 919eda0c1c..be772bbada 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 919eda0c1c6ce2681de6d1b975b73313b834353d +Subproject commit be772bbada79b106db33fb9de5f56fa71226adc5 From af9b072ddfdfa54621a02df06816bac7a87e0e85 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Fri, 2 Dec 2011 21:28:08 -0800 Subject: [PATCH 133/964] Phew, half way through bro.bif documentation. --- src/bro.bif | 437 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 412 insertions(+), 25 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index 3390f322a8..db17186427 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1035,14 +1035,14 @@ function active_file%(f: file%): bool return new Val(f->IsOpen(), TYPE_BOOL); %} +## Deprecated. Will be removed. function active_connection%(id: conn_id%): bool %{ Connection* c = sessions->FindConnection(id); return new Val(c ? 1 : 0, TYPE_BOOL); %} -# Note, you *must* first make sure that the connection is active -# (e.g., by calling active_connection()) before invoking this. +## Deprecated. Will be removed. function connection_record%(cid: conn_id%): connection %{ Connection* c = sessions->FindConnection(cid); @@ -1087,6 +1087,14 @@ EnumVal* map_conn_type(TransportProto tp) } %%} +## Extracts the transport protocol from a connection. +## +## cid: The connection identifier. +## +## Returns: The transport protocol of the connection identified by *id*. +## +## .. bro:see:: get_port_transport_proto +## get_orig_seq get_resp_seq function get_conn_transport_proto%(cid: conn_id%): transport_proto %{ Connection* c = sessions->FindConnection(cid); @@ -1099,21 +1107,55 @@ function get_conn_transport_proto%(cid: conn_id%): transport_proto return map_conn_type(c->ConnTransport()); %} +## Extracts the transport protocol from a :bro:type:`port`. +## +## p: The port. +## +## Returns: The transport protocol of the port *p*. +## +## .. bro:see:: get_conn_transport_proto +## get_orig_seq get_resp_seq function get_port_transport_proto%(p: port%): transport_proto %{ return map_conn_type(p->PortType()); %} +## Returns the current wall-clock time. +## +## In general, you should use :bro:id:`network_time` instead +## unless you are using Bro for non-networking uses (such as general +## scripting; not particularly recommended), because otherwise your script +## may behave very differently on live traffic versus played-back traffic +## from a save file. +## +## Returns: The wall-clock time. +## +## .. bro:see:: network_time function current_time%(%): time %{ return new Val(current_time(), TYPE_TIME); %} +## Returns the timestamp of the last packet processed. This function returns +## the timestamp of the most recently read packet, whether read from a +## live network interface or from a save file. +## +## Returns: The timestamp of the packet processed. +## +## .. bro:see:: current_time function network_time%(%): time %{ return new Val(network_time, TYPE_TIME); %} +## Returns a system environment variable. +## +## var: The name of the variable whose value to request. +## +## Returns: The system environment variable identified by *var*, or an empty +## string if it is not defined. +## +## .. bro:see:: setenv function getenv%(var: string%): string %{ const char* env_val = getenv(var->CheckString()); @@ -1122,6 +1164,15 @@ function getenv%(var: string%): string return new StringVal(env_val); %} +## Sets a system environment variable. +## +## var: The name of the variable. +## +## val: The (new) value of the variable *var*. +## +## Returns: True on success. +## +## .. bro:see:: getenv function setenv%(var: string, val: string%): bool %{ int result = setenv(var->AsString()->CheckString(), @@ -1132,6 +1183,14 @@ function setenv%(var: string, val: string%): bool return new Val(1, TYPE_BOOL); %} + +## Computes the square root of a :bro:type:`double`. +## +## x: The number to compute the square root of. +## +## Returns: The square root of *x*. +## +## .. bro:see:: floor exp ln log10 function sqrt%(x: double%): double %{ if ( x < 0 ) @@ -1143,29 +1202,55 @@ function sqrt%(x: double%): double return new Val(sqrt(x), TYPE_DOUBLE); %} +## Computes the exponential function. +## +## d: The argument to the exponential function. +## +## Returns: *e* to the power of *d*. +## +## .. bro:see:: floor sqrt ln log10 function exp%(d: double%): double %{ return new Val(exp(d), TYPE_DOUBLE); %} -# Natural log. +## Computes the natural logarithm of a number. +## +## d: The argument to the logarithm. +## +## Returns: The natural logarithm of *d*. +## +## .. bro:see:: exp floor sqrt log10 function ln%(d: double%): double %{ return new Val(log(d), TYPE_DOUBLE); %} -# Common log. +## Computes the common logarithm of a number. +## +## d: The argument to the logarithm. +## +## Returns: The common logarithm of *d*. +## +## .. bro:see:: exp floor sqrt ln function log10%(d: double%): double %{ return new Val(log10(d), TYPE_DOUBLE); %} +## Shuts down the Bro process immediately. +## +## .. todo: Change function signature to ``exit(code: int): any``. function exit%(%): int %{ exit(0); return 0; %} +## Gracefully shut down Bro by terminating outstanding processing. +## +## Returns: True after successful termination and false when Bro is still in +## the process of shutting down. function terminate%(%): bool %{ if ( terminating ) @@ -1222,12 +1307,39 @@ static int do_system(const char* s) } %%} +## Invokes a command via the ``system`` function of the OS. +## The command runs in the background with ``stdout`` redirecting to +## ``stderr``. Here is a usage example: +## ``system(fmt("rm \"%s\"", str_shell_escape(sniffed_data)));`` +## +## str: The command to execute. +## +## Returns: The return value from the OS ``system`` function. +## +## .. bro:see:: system_env str_shell_escape piped_exec +## +## .. note:: Note that this corresponds to the status of backgrounding the +## given command, not to the exit status of the command itself. A +## value of 127 corresponds to a failure to execute ``sh``, and -1 +## to an internal system failure. function system%(str: string%): int %{ int result = do_system(str->CheckString()); return new Val(result, TYPE_INT); %} +## Invokes a command via the ``system`` function of the OS with a prepared +## environment. The function is essentially the same as :bro:id:`system`, +## but changes the environment before invoking the command. +## +## str: The command to execute. +## +## env: A :bro:type:`set` or :bro:type:`table` with the environment variables +## in the form of key-value pairs (where the value is optional). +## +## Returns: The return value from the OS ``system`` function. +## +## .. bro:see:: system str_shell_escape piped_exec function system_env%(str: string, env: any%): int %{ if ( env->Type()->Tag() != TYPE_TABLE ) @@ -1341,7 +1453,13 @@ static Val* parse_port(const char* line) } %%} -# Returns true if the given connection exists, false otherwise. +## Checks whether a connection is (still) active. +## +## c: The connection id to check. +## +## Returns: True if the connection identified by *c* exists. +## +## .. bro:see:: lookup_connection function connection_exists%(c: conn_id%): bool %{ if ( sessions->FindConnection(c) ) @@ -1350,9 +1468,14 @@ function connection_exists%(c: conn_id%): bool return new Val(0, TYPE_BOOL); %} -# For a given connection ID, returns the corresponding "connection" record. -# Generates a run-time error and returns a dummy value if the connection -# doesn't exist. +## Returns the :bro:id:`connection` record for a given connection identifier. +## +## cid: The connection ID. +## +## Returns: The :bro:id:`connection` record for *cid*. If *cid* does not point +## to an existing connection, the function generates a run-time error +## and returns a dummy value. +## .. bro:see:: connection_exists function lookup_connection%(cid: conn_id%): connection %{ Connection* conn = sessions->FindConnection(cid); @@ -1392,6 +1515,18 @@ function lookup_connection%(cid: conn_id%): connection return c; %} +## Informs Bro that it should skip any further processing of the contents of +## a given connection. In particular, Bro will refrain from reassembling the +## TCP byte stream and from generating events relating to any analyzers that +## have been processing the connection. +## +## cid: The connection ID. +## +## Returns: False if *id* does not point to an active connection and true +## otherwise. +## +## .. note:: Bro will still generate connection-oriented events such as +## :bro:id:`connection_finished`. function skip_further_processing%(cid: conn_id%): bool %{ Connection* c = sessions->FindConnection(cid); @@ -1402,6 +1537,24 @@ function skip_further_processing%(cid: conn_id%): bool return new Val(1, TYPE_BOOL); %} +## Controls whether packet contents belonging to a connection should be +## recorded (when ``-w out.pcap`` is provided on the command line). +## +## cid: The connection identifier. +## +## do_record: True to enable packet contens and false to disable for the +## connection identified by *cid*. +## +## Returns: False if *id* does not point to an active connection and true +## otherwise. +## +## .. see:: skip_further_processing +## +## .. note:: This is independent of whether Bro processes the packets of this +## connection, which is controlled separately by +## :bro:id:`skip_further_processing`. +## +## .. bro:see: get_contents_file set_contents_file function set_record_packets%(cid: conn_id, do_record: bool%): bool %{ Connection* c = sessions->FindConnection(cid); @@ -1412,6 +1565,35 @@ function set_record_packets%(cid: conn_id, do_record: bool%): bool return new Val(1, TYPE_BOOL); %} +## Associates a file handle with a connection for writing TCP byte stream +## contents. +## +## cid: The connection ID. +## +## direction: Controls what sides of the connection to record. The argument can +## take one the four values: +## +## - ``CONTENTS_NONE``: Stop recording the connection's content. +## - ``CONTENTS_ORIG``: Record the data sent by the connection +## originator (often the client). +## - ``CONTENTS_RESP``: Record the data sent by the connection +## responder (often the server). +## - ``CONTENTS_BOTH``: Record the data sent in both directions. Results in +## the two directions being intermixed in the file, in +## the order the data was seen by Bro. +## +## f: The file handle of the file to write the contents to. +## +## Returns: Returns false if *id* does not point to an active connection and +## true otherwise. +## +## .. note:: The data recorded to the file reflects the byte stream, not the +## contents of individual packets. Reordering and duplicates are +## removed. If any data is missing, the recording stops at the +## missing data; this can happen, e.g., due to an +## :bro:id:`ack_above_hole` event. +## +## .. bro:see: get_contents_file set_record_packets function set_contents_file%(cid: conn_id, direction: count, f: file%): bool %{ Connection* c = sessions->FindConnection(cid); @@ -1422,6 +1604,19 @@ function set_contents_file%(cid: conn_id, direction: count, f: file%): bool return new Val(1, TYPE_BOOL); %} +## Returns the file handle of the contents file of a connection. +## +## cid: The connection ID. +## +## direction: Controls what sides of the connection to record. SEe +## :bro:id:`set_contents_file` for possible values. +## +## Returns: The :bro:type:`file` handle for the contentents file of the +## connection identified by *cid*. If the connection exists +## but no contents file for *direction*, the function generates a +## error and returns a file handle to ``stderr``. +## +## .. bro:see: set_contents_file set_record_packets function get_contents_file%(cid: conn_id, direction: count%): file %{ Connection* c = sessions->FindConnection(cid); @@ -1442,6 +1637,13 @@ function get_contents_file%(cid: conn_id, direction: count%): file return new Val(new BroFile(stderr, "-", "w")); %} +## Gets the filename associated with a file handle. +## +## f: The file handle to inquire the name for. +## +## Returns: The filename associated with *f*. +## +## .. bro:see:: open function get_file_name%(f: file%): string %{ if ( ! f ) @@ -1450,9 +1652,14 @@ function get_file_name%(f: file%): string return new StringVal(f->Name()); %} -# Set an individual inactivity timeout for this connection -# (overrides the global inactivity_timeout). Returns previous -# timeout interval. +## Sets an individual inactivity timeout for a connection and thus +## overrides the global inactivity timeout. +## +## cid: The connection ID. +## +## t: The new inactivity timeout for the connection identified by *cid*. +## +## Returns: The previous timeout interval. function set_inactivity_timeout%(cid: conn_id, t: interval%): interval %{ Connection* c = sessions->FindConnection(cid); @@ -1465,6 +1672,25 @@ function set_inactivity_timeout%(cid: conn_id, t: interval%): interval return new Val(old_timeout, TYPE_INTERVAL); %} +## Returns the state of the given login (Telnet or Rlogin) connection. +## +## cid: The connection ID. +## +## Returns: False if the connection is not active or is not tagged as a +## login analyzer. Otherwise the function returns the state, which can +## be one of: +## +## - ``LOGIN_STATE_AUTHENTICATE``: The connection is in its +## initial authentication dialog. +## - ``OGIN_STATE_LOGGED_IN``: The analyzer believes the user has +## successfully authenticated. +## - ``LOGIN_STATE_SKIP``: The analyzer has skipped any further +## processing of the connection. +## - ``LOGIN_STATE_CONFUSED``: The analyzer has concluded that it +## does not correctly know the state of the connection, and/or +## the username associated with it. +## +## .. bro:see: set_login_state function get_login_state%(cid: conn_id%): count %{ Connection* c = sessions->FindConnection(cid); @@ -1479,6 +1705,17 @@ function get_login_state%(cid: conn_id%): count TYPE_COUNT); %} +## Sets the login state of a connection with a login analyzer. +## +## cid: The connection ID. +## +## new_state: The new state of the login analyzer. See +## :bro:id:`get_login_state` for possible values. +## +## Returns: Returns false if *cid* is not an active connection +## or does not tagged as login analyzer, and true otherwise. +## +## .. bro:see: get_login_state function set_login_state%(cid: conn_id, new_state: count%): bool %{ Connection* c = sessions->FindConnection(cid); @@ -1497,6 +1734,16 @@ function set_login_state%(cid: conn_id, new_state: count%): bool #include "TCP.h" %%} +## Get the originator sequence number of a TCP connection. Sequence numbers +## are absolute (i.e., they reflect the values seen directly in packet headers; +## they are not relative to the beginning of the connection). +## +## cid: The connection ID. +## +## Returns: The highest sequence number sent by a connection's originator, or 0 +## if *cid* does not point to an active TCP connection. +## +## .. bro:see:: get_resp_seq function get_orig_seq%(cid: conn_id%): count %{ Connection* c = sessions->FindConnection(cid); @@ -1517,6 +1764,16 @@ function get_orig_seq%(cid: conn_id%): count } %} +## Get the responder sequence number of a TCP connection. Sequence numbers +## are absolute (i.e., they reflect the values seen directly in packet headers; +## they are not relative to the beginning of the connection). +## +## cid: The connection ID. +## +## Returns: The highest sequence number sent by a connection's responder, or 0 +## if *cid* does not point to an active TCP connection. +## +## .. bro:see:: get_orig_seq function get_resp_seq%(cid: conn_id%): count %{ Connection* c = sessions->FindConnection(cid); @@ -1537,7 +1794,14 @@ function get_resp_seq%(cid: conn_id%): count } %} -# These convert addresses <-> n3.n2.n1.n0.in-addr.arpa +## Converts a reverse pointer name to an address. For example, +## ``1.0.168.192.in-addr.arpa`` to ``192.168.0.1``. +## +## s: The string with the reverse pointer name. +## +## Returns: The IP address corresponding to *s*. +## +## .. bro:see:: addr_to_ptr_name parse_dotted_addr function ptr_name_to_addr%(s: string%): addr %{ int a[4]; @@ -1556,8 +1820,14 @@ function ptr_name_to_addr%(s: string%): addr return new AddrVal(htonl(addr)); %} +## Converts an IP address to a reverse pointer name. For example, +## ``192.168.0.1`` to ``1.0.168.192.in-addr.arpa``. ## -## .. bro:see:: addr_to_count +## a: The IP address to convert to a reverse pointer name. +## +## Returns: The reverse pointer representation of *a*. +## +## .. bro:see:: addr_to_count ptr_name_to_addr pase_dotted_addr function addr_to_ptr_name%(a: addr%): string %{ // ## Question: @@ -1588,21 +1858,54 @@ function addr_to_ptr_name%(a: addr%): string %} # Transforms n0.n1.n2.n3 -> addr. + +## Converts a decimal dotted IP address in a :bro:type:`string` to an +## :bro:type:`addr` type. +## +## s: The IP address in the form ``n0.n1.n2.n3``. +## +## Returns: The IP address as type :bro:type:`addr`. +## +## .. bro:see:: addr_to_ptr_name parse_dotted_addr addr_to_count function parse_dotted_addr%(s: string%): addr %{ return new AddrVal(dotted_to_addr(s->CheckString())); %} +## Converts a string representation of the FTP PORT command to an ``ftp_port``. +## +## s: The string of the FTP PORT command, e.g., ``"10,0,0,1,4,31"``. +## +## Returns: The FTP PORT, e.g., ``[h=10.0.0.1, p=1055/tcp, valid=T]`` +## +## .. bro:see:: parse_eftp_port parse_ftp_pasv parse_ftp_epsv fmt_ftp_port function parse_ftp_port%(s: string%): ftp_port %{ return parse_port(s->CheckString()); %} +## Converts a string representation of the FTP EPRT command to an ``ftp_port``. +## (see `RFC 2428 `_). +## The format is ``EPRT``, +## where ```` is a delimiter in the ASCII range 33-126 (usually ``|``). +## +## s: The string of the FTP PORT command, e.g., ``"10,0,0,1,4,31"``. +## +## Returns: The FTP PORT, e.g., ``[h=10.0.0.1, p=1055/tcp, valid=T]`` +## +## .. bro:see:: parse_ftp_port parse_ftp_pasv parse_ftp_epsv fmt_ftp_port function parse_eftp_port%(s: string%): ftp_port %{ return parse_eftp(s->CheckString()); %} +## Converts the result of the FTP PASV command to an ``ftp_port``. +## +## str: The string containing the result of the FTP PASV command. +## +## Returns: The FTP PORT, e.g., ``[h=10.0.0.1, p=1055/tcp, valid=T]`` +## +## .. bro:see:: parse_ftp_port parse_eftp_port parse_ftp_epsv fmt_ftp_port function parse_ftp_pasv%(str: string%): ftp_port %{ const char* s = str->CheckString(); @@ -1621,6 +1924,16 @@ function parse_ftp_pasv%(str: string%): ftp_port return parse_port(line); %} +## Converts the result of the FTP EPSV command to an ``ftp_port`` +## (see `RFC 2428 `_). +## The format is `` ()``, where ```` is a +## delimiter in the ASCII range 33-126 (usually ``|``). +## +## str: The string containing the result of the FTP PASV command. +## +## Returns: The FTP PORT, e.g., ``[h=10.0.0.1, p=1055/tcp, valid=T]`` +## +## .. bro:see:: parse_ftp_port parse_eftp_port parse_ftp_pasv fmt_ftp_port function parse_ftp_epsv%(str: string%): ftp_port %{ const char* s = str->CheckString(); @@ -1630,6 +1943,16 @@ function parse_ftp_epsv%(str: string%): ftp_port return parse_eftp(line); %} +## Formats an IP address and TCP port as an FTP PORT command. For example, +## ``10.0.0.1`` and ``1055/tcp`` yields ``"10,0,0,1,4,31"`. +## +## a: The IP address. +## +## p: The TCP port. +## +## Returns: The FTP PORT string. +## +## .. bro:see:: parse_ftp_port parse_eftp_port parse_ftp_pasv parse_ftp_epsv function fmt_ftp_port%(a: addr, p: port%): string %{ #ifdef BROv6 @@ -1648,6 +1971,13 @@ function fmt_ftp_port%(a: addr, p: port%): string pn >> 8, pn & 0xff)); %} +## Decode a `NetBIOS name `_. +## +## name: The encoded NetBIOS name, e.g., ``"FEEIEFCAEOEFFEECEJEPFDCAEOEBENEF:``. +## +## Returns: The decoded NetBIOS name, e.g., ``"THE NETBIOS NAME"``. +## +## .. bro:see:: decode_netbios_name_type function decode_netbios_name%(name: string%): string %{ char buf[16]; @@ -1676,6 +2006,14 @@ function decode_netbios_name%(name: string%): string return new StringVal(i, result); %} +## Converts a `NetBIOS name type `_ to +## its corresonding numeric value. +## +## name: The NetBIOS name type. +## +## Returns: The numeric value of *name*. +## +## .. bro:see:: decode_netbios_name function decode_netbios_name_type%(name: string%): count %{ const u_char* s = name->Bytes(); @@ -1700,7 +2038,13 @@ const char* conn_id_string(Val* c) } %%} -# Skip data of the HTTP entity on the connection +## Skips the data of the HTTP entity. +## +## c: The HTTP connection. +## +## is_orig: If true, the client data is skipped and the server data otherwise. +## +## .. bro:see:: skip_smtp_data function skip_http_entity_data%(c: connection, is_orig: bool%): any %{ AnalyzerID id = mgr.CurrentAnalyzer(); @@ -1725,16 +2069,19 @@ function skip_http_entity_data%(c: connection, is_orig: bool%): any return 0; %} -# Unescape all characters in the URI, i.e. decode every %xx group. -# -# Note that unescaping reserved characters may cause loss of -# information (see below). -# -# RFC 2396: A URI is always in an "escaped" form, since escaping or -# unescaping a completed URI might change its semantics. Normally, -# the only time escape encodings can safely be made is when the URI -# is being created from its component parts. - +## Unescapes all characters in a URI, i.e., decodes every ``%xx`` group. +## +## URI: The URI to unescape. +## +## Returns: The unescaped URI with all ``%xx`` groups decoded. +## +## .. note:: Unescaping reserved characters may cause loss of +## information (see below). +## +## RFC 2396: A URI is always in an "escaped" form, since escaping or +## unescaping a completed URI might change its semantics. Normally, +## the only time escape encodings can safely be made is when the URI +## is being created from its component parts. function unescape_URI%(URI: string%): string %{ const u_char* line = URI->Bytes(); @@ -1747,7 +2094,11 @@ function unescape_URI%(URI: string%): string #include "SMTP.h" %%} -# Skip smtp_data till next mail +## Skips SMTP data until the next email in a connection. +## +## c: The SMTP connection. +## +## .. bro:see:: skip_http_entity_data function skip_smtp_data%(c: connection%): any %{ Analyzer* sa = c->FindAnalyzer(AnalyzerTag::SMTP); @@ -1757,6 +2108,14 @@ function skip_smtp_data%(c: connection%): any %} +## Converts a string of bytes into its hexadecimal representation, e.g., +## ``"04"`` to ``"3034"``. +## +## bytestring: The string of bytes. +## +## Returns: The hexadecimal reprsentation of *bytestring*. +## +## .. bro:see:: hexdump function bytestring_to_hexstr%(bytestring: string%): string %{ bro_uint_t len = bytestring->AsString()->Len(); @@ -1774,6 +2133,13 @@ function bytestring_to_hexstr%(bytestring: string%): string extern const char* bro_version(); %%} +## Returns statistics about the number of packets *(i)* received by Bro, +## *(ii)* dropped, and *(iii)* seen on the link (not always available). +## +## Returns: A record of packet statistics. +## +## .. bro:see:: resource_usage get_matcher_stats dump_rule_stats +## get_gap_summary function net_stats%(%): NetStats %{ unsigned int recv = 0; @@ -1799,6 +2165,14 @@ function net_stats%(%): NetStats return ns; %} +## Returns Bro process statistics, such as real/user/sys CPU time, memory +## usage, page faults, number of TCP/UDP/ICMP connections, timers, and events +## queued/dispatched. +## +## Returns: A record with resource usage statistics. +## +## .. bro:see:: net_stats get_matcher_stats dump_rule_stats +## get_gap_summary function resource_usage%(%): bro_resources %{ struct rusage r; @@ -1865,6 +2239,14 @@ function resource_usage%(%): bro_resources return res; %} +## Returns statistics about the regular expression engine, such as the number +## of distinct matchers, DFA states, DFA state transitions, memory usage of +## DFA states, cache hits/misses, and average number of NFA states across all +## matchers. +## +## Returns: A record with matcher statistics. +## +## .. bro:see:: net_stats resource_usage dump_rule_stats get_gap_summary function get_matcher_stats%(%): matcher_stats %{ RuleMatcher::Stats s; @@ -1885,6 +2267,11 @@ function get_matcher_stats%(%): matcher_stats return r; %} +## Returns statistics about TCP gaps. +## +## Returns: A record with TCP gap statistics. +## +## .. bro:see:: net_stats resource_usage dump_rule_stats get_matcher_stats function get_gap_summary%(%): gap_info %{ RecordVal* r = new RecordVal(gap_info); From ab1ac72d4b5fc9a1e21d01a5513abc9584ec1279 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sat, 3 Dec 2011 14:45:02 -0800 Subject: [PATCH 134/964] Updating submodule(s). [nomail] --- aux/bro-aux | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/bro-aux b/aux/bro-aux index 4a8551ae52..4d387ce660 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 4a8551ae52d52c395b366a4eb68e63356e01999e +Subproject commit 4d387ce660468b44df99d4c87d6016ae4ed2fdc4 From 506a42638ae53f6bc86ed77a23090d567b45f54d Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 5 Dec 2011 13:02:39 -0600 Subject: [PATCH 135/964] Omit loading local-.bro scripts from base cluster framework. The loading of these is better handled by BroControl and it seems odd to load them from a base/ script anyway since they'll contain site/policy specific code. Addresses #663 --- scripts/base/frameworks/cluster/__load__.bro | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/base/frameworks/cluster/__load__.bro b/scripts/base/frameworks/cluster/__load__.bro index 3334164866..bccb35dbb1 100644 --- a/scripts/base/frameworks/cluster/__load__.bro +++ b/scripts/base/frameworks/cluster/__load__.bro @@ -28,17 +28,14 @@ redef Communication::listen_port = Cluster::nodes[Cluster::node]$p; @if ( Cluster::local_node_type() == Cluster::MANAGER ) @load ./nodes/manager -@load site/local-manager @endif @if ( Cluster::local_node_type() == Cluster::PROXY ) @load ./nodes/proxy -@load site/local-proxy @endif @if ( Cluster::local_node_type() == Cluster::WORKER ) @load ./nodes/worker -@load site/local-worker @endif @endif From 78b24da7e4f84de7f499cc9365a30e7c966a7d14 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 5 Dec 2011 15:02:03 -0800 Subject: [PATCH 136/964] start support for annotation for log field types. commit before rolling part of it back... --- src/Attr.h | 1 + src/InputMgr.cc | 10 ++++++++++ src/InputReaderAscii.h | 3 +++ src/LogMgr.cc | 8 +++++--- src/LogMgr.h | 6 ++++-- src/parse.y | 5 ++++- src/scan.l | 1 + 7 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/Attr.h b/src/Attr.h index 6c835dc61c..471acfe4ba 100644 --- a/src/Attr.h +++ b/src/Attr.h @@ -35,6 +35,7 @@ typedef enum { ATTR_GROUP, ATTR_LOG, ATTR_ERROR_HANDLER, + ATTR_TYPE_COLUMN, // for input framework ATTR_TRACKED, // hidden attribute, tracked by NotifierRegistry #define NUM_ATTRS (int(ATTR_TRACKED) + 1) } attr_tag; diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 7ec5974199..0cfb59ee90 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -561,6 +561,16 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec field->subtype = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); } else if ( field->type == TYPE_VECTOR ) { field->subtype = rec->FieldType(i)->AsVectorType()->YieldType()->Tag(); + } else if ( field->type == TYPE_PORT && + rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN) ) { + // we have an annotation for the second column + + Val* c = rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN)->AttrExpr()->Eval(0); + + assert(c); + assert(c->Type()->Tag() == TYPE_STRING); + + field->secondary_name = c->AsStringVal()->AsString()->CheckString(); } fields->push_back(field); diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 01169a3cfc..c174248454 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -12,8 +12,11 @@ struct FieldMapping { string name; TypeTag type; + // internal type for sets and vectors TypeTag subtype; int position; + // for ports: pos of the second field + int secondary_position; FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 7fc8d4ef86..dd4f9c8e5a 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -83,7 +83,8 @@ bool LogField::Read(SerializationFormat* fmt) int t; int it; - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&it, "subtype") ); + bool success = (fmt->Read(&name, "name") && fmt->Read(&secondary_name, "secondary_name") && + fmt->Read(&t, "type") && fmt->Read(&it, "subtype") ); type = (TypeTag) t; subtype = (TypeTag) it; @@ -92,7 +93,8 @@ bool LogField::Read(SerializationFormat* fmt) bool LogField::Write(SerializationFormat* fmt) const { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); + return (fmt->Write(name, "name") && fmt->Write(secondary_name, "secondary_name") && fmt->Write((int)type, "type") && + fmt->Write((int)subtype, "subtype")); } LogVal::~LogVal() @@ -151,7 +153,7 @@ bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) if ( ! t->IsSet() ) return false; - return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true); + return IsCompatibleType(t->AsSetType()->Indices()->PureType()); } case TYPE_VECTOR: diff --git a/src/LogMgr.h b/src/LogMgr.h index b8530d29ab..4ccdeb793c 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -14,13 +14,15 @@ class SerializationFormat; // Description of a log field. struct LogField { string name; + // needed by input framework. port fields have two names (one for the port, one for the type) - this specifies the secondary name. + string secondary_name; TypeTag type; - // needed by input framework. otherwise it cannot determine the inner type of a set or vector. + // needed by input framework. otherwise it cannot determine the inner type of a set. TypeTag subtype; LogField() { } LogField(const LogField& other) - : name(other.name), type(other.type), subtype(other.subtype) { } + : name(other.name), secondary_name(other.secondary_name), type(other.type), subtype(other.subtype) { } // (Un-)serialize. bool Read(SerializationFormat* fmt); diff --git a/src/parse.y b/src/parse.y index 495931aae0..988a19714b 100644 --- a/src/parse.y +++ b/src/parse.y @@ -2,7 +2,7 @@ // See the file "COPYING" in the main distribution directory for copyright. %} -%expect 88 +%expect 91 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF @@ -24,6 +24,7 @@ %token TOK_ATTR_PERSISTENT TOK_ATTR_SYNCHRONIZED %token TOK_ATTR_DISABLE_PRINT_HOOK TOK_ATTR_RAW_OUTPUT TOK_ATTR_MERGEABLE %token TOK_ATTR_PRIORITY TOK_ATTR_GROUP TOK_ATTR_LOG TOK_ATTR_ERROR_HANDLER +%token TOK_ATTR_TYPE_COLUMN %token TOK_DEBUG @@ -1313,6 +1314,8 @@ attr: { $$ = new Attr(ATTR_PRIORITY, $3); } | TOK_ATTR_GROUP '=' expr { $$ = new Attr(ATTR_GROUP, $3); } + | TOK_ATTR_TYPE_COLUMN '=' expr + { $$ = new Attr(ATTR_TYPE_COLUMN, $3); } | TOK_ATTR_LOG { $$ = new Attr(ATTR_LOG); } | TOK_ATTR_ERROR_HANDLER diff --git a/src/scan.l b/src/scan.l index 7ebd7894e1..6a85e89780 100644 --- a/src/scan.l +++ b/src/scan.l @@ -308,6 +308,7 @@ when return TOK_WHEN; &optional return TOK_ATTR_OPTIONAL; &persistent return TOK_ATTR_PERSISTENT; &priority return TOK_ATTR_PRIORITY; +&type_column return TOK_ATTR_TYPE_COLUMN; &read_expire return TOK_ATTR_EXPIRE_READ; &redef return TOK_ATTR_REDEF; &rotate_interval return TOK_ATTR_ROTATE_INTERVAL; From aecbbdd966c896b475591a3415578adc1629273e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 5 Dec 2011 16:18:54 -0800 Subject: [PATCH 137/964] make logging framework send the protocol to the writer. for use in future writers, that have a special type for port, which includes the protocol. --- src/LogMgr.cc | 32 +++++++++++++++++++++++++++++--- src/LogMgr.h | 4 ++++ src/LogWriterAscii.cc | 5 ++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index dd4f9c8e5a..ed32e4e40b 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -118,6 +118,10 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } + +// if ( type == TYPE_PORT && present ) +// delete val.port_val.proto; + } bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) @@ -190,9 +194,12 @@ bool LogVal::Read(SerializationFormat* fmt) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: return fmt->Read(&val.uint_val, "uint"); + case TYPE_PORT: + val.port_val.proto = new string; + return fmt->Read(&val.port_val.port, "port") && fmt->Read(val.port_val.proto, "proto"); + case TYPE_SUBNET: { uint32 net[4]; @@ -305,9 +312,11 @@ bool LogVal::Write(SerializationFormat* fmt) const case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: return fmt->Write(val.uint_val, "uint"); + case TYPE_PORT: + return fmt->Write(val.port_val.port, "port") && fmt->Write(*val.port_val.proto, "proto"); + case TYPE_SUBNET: { uint32 net[4]; @@ -1066,6 +1075,22 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return true; } +string LogMgr::TransportProtoToString(TransportProto p) { + switch ( p ) { + case TRANSPORT_UNKNOWN: + return "unknown"; + case TRANSPORT_TCP: + return "tcp"; + case TRANSPORT_UDP: + return "udp"; + case TRANSPORT_ICMP: + return "icmp"; + } + + assert(false); + return ""; +} + LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) { if ( ! ty ) @@ -1097,7 +1122,8 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) break; case TYPE_PORT: - lval->val.uint_val = val->AsPortVal()->Port(); + lval->val.port_val.port = val->AsPortVal()->Port(); + lval->val.port_val.proto = new string(TransportProtoToString(val->AsPortVal()->PortType())); break; case TYPE_SUBNET: diff --git a/src/LogMgr.h b/src/LogMgr.h index 4ccdeb793c..d9d61236e0 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -38,10 +38,12 @@ struct LogVal { // types we can log directly. struct set_t { bro_int_t size; LogVal** vals; }; typedef set_t vec_t; + struct port_t { bro_uint_t port; string* proto; }; union _val { bro_int_t int_val; bro_uint_t uint_val; + port_t port_val; uint32 addr_val[NUM_ADDR_WORDS]; subnet_type subnet_val; double double_val; @@ -136,6 +138,8 @@ private: Filter* FindFilter(EnumVal* id, StringVal* filter); WriterInfo* FindWriter(LogWriter* writer); + string TransportProtoToString(TransportProto p); + vector streams; // Indexed by stream enum. }; diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 9b1fda3b62..c449c1a788 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -169,10 +169,13 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: desc->Add(val->val.uint_val); break; + case TYPE_PORT: + desc->Add(val->val.port_val.port); + break; + case TYPE_SUBNET: desc->Add(dotted_addr(val->val.subnet_val.net)); desc->Add("/"); From ef2f3e7507d583ce69aed5248c75710592437d7f Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 6 Dec 2011 10:08:44 -0600 Subject: [PATCH 138/964] Fix some reST formatting issues in bro.bif that Sphinx complains about. --- src/bro.bif | 64 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index db17186427..42cf7d6093 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1475,6 +1475,7 @@ function connection_exists%(c: conn_id%): bool ## Returns: The :bro:id:`connection` record for *cid*. If *cid* does not point ## to an existing connection, the function generates a run-time error ## and returns a dummy value. +## ## .. bro:see:: connection_exists function lookup_connection%(cid: conn_id%): connection %{ @@ -1525,8 +1526,10 @@ function lookup_connection%(cid: conn_id%): connection ## Returns: False if *id* does not point to an active connection and true ## otherwise. ## -## .. note:: Bro will still generate connection-oriented events such as -## :bro:id:`connection_finished`. +## .. note:: +## +## Bro will still generate connection-oriented events such as +## :bro:id:`connection_finished`. function skip_further_processing%(cid: conn_id%): bool %{ Connection* c = sessions->FindConnection(cid); @@ -1538,7 +1541,7 @@ function skip_further_processing%(cid: conn_id%): bool %} ## Controls whether packet contents belonging to a connection should be -## recorded (when ``-w out.pcap`` is provided on the command line). +## recorded (when ``-w`` option is provided on the command line). ## ## cid: The connection identifier. ## @@ -1548,11 +1551,13 @@ function skip_further_processing%(cid: conn_id%): bool ## Returns: False if *id* does not point to an active connection and true ## otherwise. ## -## .. see:: skip_further_processing +## .. bro:see:: skip_further_processing ## -## .. note:: This is independent of whether Bro processes the packets of this -## connection, which is controlled separately by -## :bro:id:`skip_further_processing`. +## .. note:: +## +## This is independent of whether Bro processes the packets of this +## connection, which is controlled separately by +## :bro:id:`skip_further_processing`. ## ## .. bro:see: get_contents_file set_contents_file function set_record_packets%(cid: conn_id, do_record: bool%): bool @@ -1571,27 +1576,30 @@ function set_record_packets%(cid: conn_id, do_record: bool%): bool ## cid: The connection ID. ## ## direction: Controls what sides of the connection to record. The argument can -## take one the four values: +## take one the four values: ## -## - ``CONTENTS_NONE``: Stop recording the connection's content. -## - ``CONTENTS_ORIG``: Record the data sent by the connection -## originator (often the client). -## - ``CONTENTS_RESP``: Record the data sent by the connection -## responder (often the server). -## - ``CONTENTS_BOTH``: Record the data sent in both directions. Results in -## the two directions being intermixed in the file, in -## the order the data was seen by Bro. +## - ``CONTENTS_NONE``: Stop recording the connection's content. +## - ``CONTENTS_ORIG``: Record the data sent by the connection +## originator (often the client). +## - ``CONTENTS_RESP``: Record the data sent by the connection +## responder (often the server). +## - ``CONTENTS_BOTH``: Record the data sent in both directions. +## Results in the two directions being +## intermixed in the file, in the order the +## data was seen by Bro. ## ## f: The file handle of the file to write the contents to. ## ## Returns: Returns false if *id* does not point to an active connection and ## true otherwise. ## -## .. note:: The data recorded to the file reflects the byte stream, not the -## contents of individual packets. Reordering and duplicates are -## removed. If any data is missing, the recording stops at the -## missing data; this can happen, e.g., due to an -## :bro:id:`ack_above_hole` event. +## .. note:: +## +## The data recorded to the file reflects the byte stream, not the +## contents of individual packets. Reordering and duplicates are +## removed. If any data is missing, the recording stops at the +## missing data; this can happen, e.g., due to an +## :bro:id:`ack_above_hole` event. ## ## .. bro:see: get_contents_file set_record_packets function set_contents_file%(cid: conn_id, direction: count, f: file%): bool @@ -1827,7 +1835,7 @@ function ptr_name_to_addr%(s: string%): addr ## ## Returns: The reverse pointer representation of *a*. ## -## .. bro:see:: addr_to_count ptr_name_to_addr pase_dotted_addr +## .. bro:see:: addr_to_count ptr_name_to_addr parse_dotted_addr function addr_to_ptr_name%(a: addr%): string %{ // ## Question: @@ -1924,8 +1932,8 @@ function parse_ftp_pasv%(str: string%): ftp_port return parse_port(line); %} -## Converts the result of the FTP EPSV command to an ``ftp_port`` -## (see `RFC 2428 `_). +## Converts the result of the FTP EPSV command to an ``ftp_port``. +## See `RFC 2428 `_. ## The format is `` ()``, where ```` is a ## delimiter in the ASCII range 33-126 (usually ``|``). ## @@ -1944,7 +1952,7 @@ function parse_ftp_epsv%(str: string%): ftp_port %} ## Formats an IP address and TCP port as an FTP PORT command. For example, -## ``10.0.0.1`` and ``1055/tcp`` yields ``"10,0,0,1,4,31"`. +## ``10.0.0.1`` and ``1055/tcp`` yields ``"10,0,0,1,4,31"``. ## ## a: The IP address. ## @@ -1971,7 +1979,7 @@ function fmt_ftp_port%(a: addr, p: port%): string pn >> 8, pn & 0xff)); %} -## Decode a `NetBIOS name `_. +## Decode a NetBIOS name. See http://support.microsoft.com/kb/194203. ## ## name: The encoded NetBIOS name, e.g., ``"FEEIEFCAEOEFFEECEJEPFDCAEOEBENEF:``. ## @@ -2006,8 +2014,8 @@ function decode_netbios_name%(name: string%): string return new StringVal(i, result); %} -## Converts a `NetBIOS name type `_ to -## its corresonding numeric value. +## Converts a NetBIOS name type to its corresonding numeric value. +## See http://support.microsoft.com/kb/163409. ## ## name: The NetBIOS name type. ## From 4a690484ecaf5ea086090d2b6ef855cc6c913cad Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 6 Dec 2011 10:42:37 -0800 Subject: [PATCH 139/964] make port annotation work and ascii input reader way more rebust with better error messages. --- src/Attr.cc | 22 ++- src/InputMgr.cc | 44 ++++- src/InputMgr.h | 2 + src/InputReaderAscii.cc | 150 +++++++++--------- src/InputReaderAscii.h | 4 +- src/LogMgr.cc | 1 - .../scripts.base.frameworks.input.port/out | 6 + .../scripts/base/frameworks/input/port.bro | 39 +++++ 8 files changed, 187 insertions(+), 81 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.port/out create mode 100644 testing/btest/scripts/base/frameworks/input/port.bro diff --git a/src/Attr.cc b/src/Attr.cc index a5a350f452..1d610f7cb4 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -17,7 +17,7 @@ const char* attr_name(attr_tag t) "&persistent", "&synchronized", "&postprocessor", "&encrypt", "&match", "&disable_print_hook", "&raw_output", "&mergeable", "&priority", - "&group", "&log", "&error_handler", "(&tracked)", + "&group", "&log", "&error_handler", "&type_column", "(&tracked)", }; return attr_names[int(t)]; @@ -417,6 +417,26 @@ void Attributes::CheckAttr(Attr* a) Error("&log applied to a type that cannot be logged"); break; + case ATTR_TYPE_COLUMN: + { + if ( type->Tag() != TYPE_PORT ) + { + Error("type_column tag only applicable to ports"); + break; + } + + BroType* atype = a->AttrExpr()->Type(); + + if ( atype->Tag() != TYPE_STRING ) { + Error("type column needs to have a string argument"); + break; + } + + + break; + } + + default: BadTag("Attributes::CheckAttr", attr_name(a->Tag())); } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 0cfb59ee90..612461bae8 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -1167,9 +1167,14 @@ int InputMgr::GetLogValLength(const LogVal* val) { case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: length += sizeof(val->val.uint_val); break; + + case TYPE_PORT: + length += sizeof(val->val.port_val.port); + if ( val->val.port_val.proto != 0 ) + length += val->val.port_val.proto->size(); + break; case TYPE_DOUBLE: case TYPE_TIME: @@ -1228,12 +1233,24 @@ int InputMgr::CopyLogVal(char *data, const int startpos, const LogVal* val) { case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: //*(data+startpos) = val->val.uint_val; memcpy(data+startpos, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); return sizeof(val->val.uint_val); break; + case TYPE_PORT: { + int length = 0; + memcpy(data+startpos, (const void*) &(val->val.port_val.port), sizeof(val->val.port_val.port)); + length += sizeof(val->val.port_val.port); + if ( val->val.port_val.proto != 0 ) { + memcpy(data+startpos, val->val.port_val.proto->c_str(), val->val.port_val.proto->length()); + length += val->val.port_val.proto->size(); + } + return length; + break; + } + + case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: @@ -1320,6 +1337,24 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals } +TransportProto InputMgr::StringToProto(const string &proto) { + if ( proto == "unknown" ) { + return TRANSPORT_UNKNOWN; + } else if ( proto == "tcp" ) { + return TRANSPORT_TCP; + } else if ( proto == "udp" ) { + return TRANSPORT_UDP; + } else if ( proto == "icmp" ) { + return TRANSPORT_ICMP; + } + + //assert(false); + + reporter->Error("Tried to parse invalid/unknown protocol: %s", proto.c_str()); + + return TRANSPORT_UNKNOWN; +} + Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) { @@ -1357,7 +1392,10 @@ Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { } case TYPE_PORT: - return new PortVal(val->val.uint_val); + if ( val->val.port_val.proto == 0 ) + return new PortVal(val->val.port_val.port); + else + return new PortVal(val->val.port_val.port, StringToProto(*val->val.port_val.proto)); break; case TYPE_ADDR: diff --git a/src/InputMgr.h b/src/InputMgr.h index cebed231e4..ba6e208fd1 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -68,6 +68,8 @@ private: Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); RecordVal* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); + TransportProto StringToProto(const string &proto); + ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 84feb74e61..501022d58e 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -10,25 +10,27 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int : name(arg_name), type(arg_type) { position = arg_position; + secondary_position = -1; } FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position) : name(arg_name), type(arg_type), subtype(arg_subtype) { position = arg_position; + secondary_position = -1; } FieldMapping::FieldMapping(const FieldMapping& arg) : name(arg.name), type(arg.type), subtype(arg.subtype) { position = arg.position; + secondary_position = arg.secondary_position; } FieldMapping FieldMapping::subType() { return FieldMapping(name, subtype, position); } - InputReaderAscii::InputReaderAscii() { file = 0; @@ -122,45 +124,45 @@ bool InputReaderAscii::ReadHeader() { return false; } + map fields; + + // construcr list of field names. + istringstream splitstream(line); + int pos=0; + while ( splitstream ) { + string s; + if ( !getline(splitstream, s, separator[0])) + break; + + fields[s] = pos; + pos++; + } + + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - // split on tabs... - istringstream splitstream(line); - unsigned int currTab = 0; - int wantFields = 0; - while ( splitstream ) { - string s; - if ( !getline(splitstream, s, separator[0])) - break; - // current found heading in s... compare if we want it - for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { - const LogField* field = (*it).second.fields[i]; - if ( field->name == s ) { - // cool, found field. note position - FieldMapping f(field->name, field->type, field->subtype, i); - (*it).second.columnMap.push_back(f); - wantFields++; - break; // done with searching + for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { + const LogField* field = (*it).second.fields[i]; + + map::iterator fit = fields.find(field->name); + if ( fit == fields.end() ) { + Error(Fmt("Did not find requested field %s in input data file.", field->name.c_str())); + return false; + } + + + FieldMapping f(field->name, field->type, field->subtype, fields[field->name]); + if ( field->secondary_name != "" ) { + map::iterator fit2 = fields.find(field->secondary_name); + if ( fit2 == fields.end() ) { + Error(Fmt("Could not find requested port type field %s in input data file.", field->secondary_name.c_str())); + return false; } + f.secondary_position = fields[field->secondary_name]; } - - // look if we did push something... - if ( (*it).second.columnMap.size() == currTab ) { - // no, we didn't. note that... - FieldMapping empty; - (*it).second.columnMap.push_back(empty); - } - - // done - currTab++; - } - - if ( wantFields != (int) (*it).second.num_fields ) { - // we did not find all fields? - // :( - Error(Fmt("One of the requested fields could not be found in the input data file. Found %d fields, wanted %d. Filternum: %d", wantFields, (*it).second.num_fields, (*it).first)); - return false; + (*it).second.columnMap.push_back(f); } + } // well, that seems to have worked... @@ -220,10 +222,14 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: val->val.uint_val = atoi(s.c_str()); break; + case TYPE_PORT: + val->val.port_val.port = atoi(s.c_str()); + val->val.port_val.proto = 0; + break; + case TYPE_SUBNET: { int pos = s.find("/"); string width = s.substr(pos+1); @@ -346,59 +352,55 @@ bool InputReaderAscii::DoUpdate() { string line; while ( GetLine(line ) ) { + // split on tabs + istringstream splitstream(line); + + map stringfields; + int pos = 0; + while ( splitstream ) { + string s; + if ( !getline(splitstream, s, separator[0]) ) + break; + + stringfields[pos] = s; + pos++; + } + + pos--; // for easy comparisons of max element. for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - // split on tabs - - istringstream splitstream(line); - LogVal** fields = new LogVal*[(*it).second.num_fields]; - //string string_fields[num_fields]; - unsigned int currTab = 0; - unsigned int currField = 0; - while ( splitstream ) { + int fpos = 0; + for ( vector::iterator fit = (*it).second.columnMap.begin(); + fit != (*it).second.columnMap.end(); + fit++ ){ - string s; - if ( !getline(splitstream, s, separator[0]) ) - break; - - - if ( currTab >= (*it).second.columnMap.size() ) { - Error("Tabs in heading do not match tabs in data?"); - //disabled = true; + if ( (*fit).position > pos || (*fit).secondary_position > pos ) { + Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos, (*fit).position, (*fit).secondary_position)); return false; } - FieldMapping currMapping = (*it).second.columnMap[currTab]; - currTab++; - - if ( currMapping.IsEmpty() ) { - // well, that was easy - continue; - } - - if ( currField >= (*it).second.num_fields ) { - Error("internal error - fieldnum greater as possible"); - return false; - } - - LogVal* val = EntryToVal(s, currMapping); + LogVal* val = EntryToVal(stringfields[(*fit).position], *fit); if ( val == 0 ) { return false; } - fields[currMapping.position] = val; - //string_fields[currMapping.position] = s; + + if ( (*fit).secondary_position != -1 ) { + // we have a port definition :) + assert(val->type == TYPE_PORT ); + // Error(Fmt("Got type %d != PORT with secondary position!", val->type)); - currField++; - } - - if ( currField != (*it).second.num_fields ) { - Error("curr_field != num_fields in DoUpdate. Columns in file do not match column definition."); - return false; + val->val.port_val.proto = new string(stringfields[(*fit).secondary_position]); + } + + fields[fpos] = val; + + fpos++; } + assert ( (unsigned int) fpos == (*it).second.num_fields ); SendEntry((*it).first, fields); diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index c174248454..2670d785d5 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -21,10 +21,10 @@ struct FieldMapping { FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); FieldMapping(const FieldMapping& arg); - FieldMapping() { position = -1; } + FieldMapping() { position = -1; secondary_position = -1; } FieldMapping subType(); - bool IsEmpty() { return position == -1; } + //bool IsEmpty() { return position == -1; } }; diff --git a/src/LogMgr.cc b/src/LogMgr.cc index ed32e4e40b..307a2e24e8 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -1088,7 +1088,6 @@ string LogMgr::TransportProtoToString(TransportProto p) { } assert(false); - return ""; } LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.port/out b/testing/btest/Baseline/scripts.base.frameworks.input.port/out new file mode 100644 index 0000000000..6f2bd3271b --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.port/out @@ -0,0 +1,6 @@ +Trying to find field: t +{ +[1.2.3.4] = [p=80/tcp], +[1.2.4.6] = [p=30/unknown], +[1.2.3.5] = [p=52/udp] +} diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro new file mode 100644 index 0000000000..6f98c363c7 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -0,0 +1,39 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#fields i p t +1.2.3.4 80 tcp +1.2.3.5 52 udp +1.2.4.6 30 unknown +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Input::ID += { INPUT }; +} + +type Idx: record { + i: addr; +}; + +type Val: record { + p: port &type_column="t"; +}; + +global servers: table[addr] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::force_update(A::INPUT); + print servers; + Input::remove_tablefilter(A::INPUT, "ssh"); + Input::remove_stream(A::INPUT); +} From 9f32f68a13f290f01747f02b05dbe4b8b63d2790 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 6 Dec 2011 10:50:36 -0800 Subject: [PATCH 140/964] make test more robust. --- .../Baseline/scripts.base.frameworks.input.port/out | 9 +++------ testing/btest/scripts/base/frameworks/input/port.bro | 10 ++++++---- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.port/out b/testing/btest/Baseline/scripts.base.frameworks.input.port/out index 6f2bd3271b..858551aa2f 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.port/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.port/out @@ -1,6 +1,3 @@ -Trying to find field: t -{ -[1.2.3.4] = [p=80/tcp], -[1.2.4.6] = [p=30/unknown], -[1.2.3.5] = [p=52/udp] -} +[p=80/tcp] +[p=52/udp] +[p=30/unknown] diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro index 6f98c363c7..c14892ae36 100644 --- a/testing/btest/scripts/base/frameworks/input/port.bro +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -6,7 +6,7 @@ #fields i p t 1.2.3.4 80 tcp 1.2.3.5 52 udp -1.2.4.6 30 unknown +1.2.3.6 30 unknown @TEST-END-FILE redef InputAscii::empty_field = "EMPTY"; @@ -31,9 +31,11 @@ event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::INPUT); - print servers; - Input::remove_tablefilter(A::INPUT, "ssh"); + print servers[1.2.3.4]; + print servers[1.2.3.5]; + print servers[1.2.3.6]; + Input::remove_tablefilter(A::INPUT, "input"); Input::remove_stream(A::INPUT); } From eb64eeedcd27c543becf330d7da48db9aa541ea8 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 6 Dec 2011 10:56:26 -0800 Subject: [PATCH 141/964] memleak fix. --- src/LogMgr.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 307a2e24e8..dd5bc4b66a 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -119,8 +119,8 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } -// if ( type == TYPE_PORT && present ) -// delete val.port_val.proto; + if ( type == TYPE_PORT && present ) + delete val.port_val.proto; } From 9d5f79a0fa3c34998385149662297a718aec2ed6 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 6 Dec 2011 11:49:29 -0800 Subject: [PATCH 142/964] Updating submodule(s). [nomail] --- CHANGES | 5 +++++ VERSION | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index b7e69d4997..dd0b98bf72 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.0-beta-97 | 2011-12-06 11:49:29 -0800 + + * Omit loading local-.bro scripts from base cluster framework. + Addresses #663 (Jon Siwek) + 2.0-beta-94 | 2011-12-03 15:57:19 -0800 * Adapting attribute serialization when talking to Broccoli. (Robin diff --git a/VERSION b/VERSION index 11e9b6983f..d8c774d329 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0-beta-94 +2.0-beta-97 diff --git a/aux/broccoli b/aux/broccoli index d8f9d4698e..05d51b55dd 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit d8f9d4698e8e02f493a669c5adcf897506671b5d +Subproject commit 05d51b55dd5a8430151cff74f7876bd73385d5cd diff --git a/aux/broctl b/aux/broctl index be772bbada..406c689c88 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit be772bbada79b106db33fb9de5f56fa71226adc5 +Subproject commit 406c689c88997c4459eaf36bd2c65bce2e847c2d From 9295beda7ff35fddd90368a8e8ed1e1f12b97654 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 7 Dec 2011 08:19:40 -0800 Subject: [PATCH 143/964] Stepping stone events aren't deprecated but internal. --- src/event.bif | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/event.bif b/src/event.bif index 1655a52fdf..c0a6311f0e 100644 --- a/src/event.bif +++ b/src/event.bif @@ -5610,23 +5610,25 @@ event reporter_error%(t: time, msg: string, location: string%) &error_handler; ## recursively for each ``@load``. event bro_script_loaded%(path: string, level: count%); -# ##### Deprecated events. Proposed for removal. - ## Deprecated. Will be removed. event stp_create_endp%(c: connection, e: int, is_orig: bool%); -## Deprecated. Will be removed. +# ##### Internal events. Not further documented. + +## Event internal to the stepping stone detector. event stp_resume_endp%(e: int%); -## Deprecated. Will be removed. +## Event internal to the stepping stone detector. event stp_correlate_pair%(e1: int, e2: int%); -## Deprecated. Will be removed. +## Event internal to the stepping stone detector. event stp_remove_pair%(e1: int, e2: int%); -## Deprecated. Will be removed. +## Event internal to the stepping stone detector. event stp_remove_endp%(e: int%); +# ##### Deprecated events. Proposed for removal. + ## Deprecated. Will be removed. event interconn_stats%(c: connection, os: interconn_endp_stats, rs: interconn_endp_stats%); From 3c2fa085d43c176e4ae75874964eb810cd278cef Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 7 Dec 2011 07:45:48 -0800 Subject: [PATCH 144/964] Adapting diff-all so that it expands globs in both current and baseline directory. This way, it now spots if a Baseline file isn't produced anymore. Closes #677. --- CHANGES | 5 +++++ VERSION | 2 +- testing/external/scripts/diff-all | 11 +++++++++-- testing/external/scripts/skel/test.skeleton | 2 +- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index dd0b98bf72..000c7ae6e8 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.0-beta-98 | 2011-12-07 08:12:08 -0800 + + * Adapting test-suite's diff-all so that it expands globs in both + current and baseline directory. Closes #677. (Robin Sommer) + 2.0-beta-97 | 2011-12-06 11:49:29 -0800 * Omit loading local-.bro scripts from base cluster framework. diff --git a/VERSION b/VERSION index d8c774d329..6f30af8ad8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0-beta-97 +2.0-beta-98 diff --git a/testing/external/scripts/diff-all b/testing/external/scripts/diff-all index 597c769687..329bbb7f00 100755 --- a/testing/external/scripts/diff-all +++ b/testing/external/scripts/diff-all @@ -1,6 +1,10 @@ #! /usr/bin/env bash # -# Runs btest-diff on $@ and fails if any fails. +# Runs btest-diff on $@ and fails if any fails. If $@ contains globs, we expand +# them relative to *both* the current directory and the test's baseline +# directory so that we spot missing files. Note that you will need to quote +# the globals in the TEST-EXEC line as otherwise they will have been expanded relative +# to the current directory already when this scripts runs. diag=$TEST_DIAGNOSTICS @@ -14,7 +18,10 @@ fi rc=0; -for i in $@; do +files_cwd=`ls $@` +files_baseline=`cd $TEST_BASELINE && ls $@` + +for i in `echo $files_cwd $files_baseline | sort | uniq`; do if [[ "$i" != "loaded_scripts.log" && "$i" != "prof.log" ]]; then if ! btest-diff $i; then echo "" >>$diag diff --git a/testing/external/scripts/skel/test.skeleton b/testing/external/scripts/skel/test.skeleton index becd970d78..a76f3d4d09 100644 --- a/testing/external/scripts/skel/test.skeleton +++ b/testing/external/scripts/skel/test.skeleton @@ -1,5 +1,5 @@ # @TEST-EXEC: zcat $TRACES/trace.gz | bro -r - %INPUT -# @TEST-EXEC: $SCRIPTS/diff-all *.log +# @TEST-EXEC: $SCRIPTS/diff-all '*.log' @load testing-setup @load test-all-policy From f1e132cd1acb2ba37c20f90b634ad0e6c74e8cd3 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 7 Dec 2011 10:28:56 -0800 Subject: [PATCH 145/964] Adding missing script. --- testing/external/scripts/perftools-adapt-paths | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100755 testing/external/scripts/perftools-adapt-paths diff --git a/testing/external/scripts/perftools-adapt-paths b/testing/external/scripts/perftools-adapt-paths new file mode 100755 index 0000000000..2eda2477c7 --- /dev/null +++ b/testing/external/scripts/perftools-adapt-paths @@ -0,0 +1,10 @@ +#! /usr/bin/env bash +# +# Adapts relative paths in perftools stderr output to work +# directly from the top-level test directory. +# +# Returns an exit code > 0 if there's a leak. + +cat $1 | sed "s#bro *\"\./#../../../build/src/bro \".tmp/$TEST_NAME/#g" | sed 's/ *--gv//g' >$1.tmp && mv $1.tmp $1 + +grep -q "No leaks found" $1 From ab315949d6e08b622f786e3941cd66173b0ef222 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 7 Dec 2011 12:53:11 -0600 Subject: [PATCH 146/964] Remove remnant of libmagic optionality --- config.h.in | 3 --- 1 file changed, 3 deletions(-) diff --git a/config.h.in b/config.h.in index 62aa34ef43..e3c9984373 100644 --- a/config.h.in +++ b/config.h.in @@ -17,9 +17,6 @@ /* We are on a Linux system */ #cmakedefine HAVE_LINUX -/* Define if you have the header file. */ -#cmakedefine HAVE_MAGIC_H - /* Define if you have the `mallinfo' function. */ #cmakedefine HAVE_MALLINFO From 707926aaa441c24eae491768e82f194ff3bd3b0a Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 7 Dec 2011 12:12:46 -0800 Subject: [PATCH 147/964] Software framework stores ports for server software. --- scripts/base/frameworks/software/main.bro | 66 ++++++++++++------- .../policy/protocols/http/detect-webapps.bro | 2 +- scripts/policy/protocols/http/software.bro | 6 +- scripts/policy/protocols/ssh/software.bro | 2 +- 4 files changed, 49 insertions(+), 27 deletions(-) diff --git a/scripts/base/frameworks/software/main.bro b/scripts/base/frameworks/software/main.bro index 574886288a..9abac9e575 100644 --- a/scripts/base/frameworks/software/main.bro +++ b/scripts/base/frameworks/software/main.bro @@ -34,7 +34,11 @@ export { ## The time at which the software was first detected. ts: time &log; ## The IP address detected running the software. - host: addr &log; + host_a: addr &log; + ## The Port on which the software is running. Only sensible for server software. + host_p: port &log &optional; + ## The transport protocol that is being used. Only sensible for server software. + proto: transport_proto &log &optional; ## The type of software detected (e.g. WEB_SERVER) software_type: Type &log &default=UNKNOWN; ## Name of the software (e.g. Apache) @@ -71,7 +75,13 @@ export { ## still many cases where scripts may have to have their own specific ## version parsing though. global parse: function(unparsed_version: string, - host: addr, + host_a: addr, + software_type: Type): Info; + + ## This function is the equivalent to parse for software that has a specific + ## source port (i.e. server software) + global parse_with_port: function(unparsed_version: string, + host_a: addr, host_p: port, software_type: Type): Info; ## Compare two versions. @@ -107,7 +117,7 @@ event bro_init() } function parse_mozilla(unparsed_version: string, - host: addr, + host_a: addr, software_type: Type): Info { local software_name = ""; @@ -119,7 +129,7 @@ function parse_mozilla(unparsed_version: string, software_name = "Opera"; parts = split_all(unparsed_version, /Opera [0-9\.]*$/); if ( 2 in parts ) - v = parse(parts[2], host, software_type)$version; + v = parse(parts[2], host_a, software_type)$version; } else if ( / MSIE / in unparsed_version ) { @@ -134,7 +144,7 @@ function parse_mozilla(unparsed_version: string, { parts = split_all(unparsed_version, /MSIE [0-9]{1,2}\.*[0-9]*b?[0-9]*/); if ( 2 in parts ) - v = parse(parts[2], host, software_type)$version; + v = parse(parts[2], host_a, software_type)$version; } } else if ( /Version\/.*Safari\// in unparsed_version ) @@ -143,7 +153,7 @@ function parse_mozilla(unparsed_version: string, parts = split_all(unparsed_version, /Version\/[0-9\.]*/); if ( 2 in parts ) { - v = parse(parts[2], host, software_type)$version; + v = parse(parts[2], host_a, software_type)$version; if ( / Mobile\/?.* Safari/ in unparsed_version ) v$addl = "Mobile"; } @@ -153,7 +163,7 @@ function parse_mozilla(unparsed_version: string, parts = split_all(unparsed_version, /(Firefox|Netscape|Thunderbird)\/[0-9\.]*/); if ( 2 in parts ) { - local tmp_s = parse(parts[2], host, software_type); + local tmp_s = parse(parts[2], host_a, software_type); software_name = tmp_s$name; v = tmp_s$version; } @@ -163,7 +173,7 @@ function parse_mozilla(unparsed_version: string, software_name = "Chrome"; parts = split_all(unparsed_version, /Chrome\/[0-9\.]*/); if ( 2 in parts ) - v = parse(parts[2], host, software_type)$version; + v = parse(parts[2], host_a, software_type)$version; } else if ( /^Opera\// in unparsed_version ) { @@ -174,12 +184,12 @@ function parse_mozilla(unparsed_version: string, software_name = parts[2]; parts = split_all(unparsed_version, /Version\/[0-9\.]*/); if ( 2 in parts ) - v = parse(parts[2], host, software_type)$version; + v = parse(parts[2], host_a, software_type)$version; else { parts = split_all(unparsed_version, /Opera Mini\/[0-9\.]*/); if ( 2 in parts ) - v = parse(parts[2], host, software_type)$version; + v = parse(parts[2], host_a, software_type)$version; } } else @@ -187,7 +197,7 @@ function parse_mozilla(unparsed_version: string, software_name = "Opera"; parts = split_all(unparsed_version, /Version\/[0-9\.]*/); if ( 2 in parts ) - v = parse(parts[2], host, software_type)$version; + v = parse(parts[2], host_a, software_type)$version; } } else if ( /AppleWebKit\/[0-9\.]*/ in unparsed_version ) @@ -195,17 +205,17 @@ function parse_mozilla(unparsed_version: string, software_name = "Unspecified WebKit"; parts = split_all(unparsed_version, /AppleWebKit\/[0-9\.]*/); if ( 2 in parts ) - v = parse(parts[2], host, software_type)$version; + v = parse(parts[2], host_a, software_type)$version; } - return [$ts=network_time(), $host=host, $name=software_name, $version=v, + return [$ts=network_time(), $host_a=host_a, $name=software_name, $version=v, $software_type=software_type, $unparsed_version=unparsed_version]; } # Don't even try to understand this now, just make sure the tests are # working. function parse(unparsed_version: string, - host: addr, + host_a: addr, software_type: Type): Info { local software_name = ""; @@ -214,7 +224,7 @@ function parse(unparsed_version: string, # Parse browser-alike versions separately if ( /^(Mozilla|Opera)\/[0-9]\./ in unparsed_version ) { - return parse_mozilla(unparsed_version, host, software_type); + return parse_mozilla(unparsed_version, host_a, software_type); } else { @@ -276,11 +286,23 @@ function parse(unparsed_version: string, v$major = extract_count(version_numbers[1]); } } - return [$ts=network_time(), $host=host, $name=software_name, + return [$ts=network_time(), $host_a=host_a, $name=software_name, $version=v, $unparsed_version=unparsed_version, $software_type=software_type]; } +function parse_with_port(unparsed_version: string, + host_a: addr, host_p: port, + software_type: Type): Info +{ + local i: Info; + i = parse(unparsed_version, host_a, software_type); + i$host_p = host_p; + i$proto = get_port_transport_proto(host_p); + + return i; +} + function cmp_versions(v1: Version, v2: Version): int { @@ -340,9 +362,9 @@ function cmp_versions(v1: Version, v2: Version): int } } -function software_endpoint_name(id: conn_id, host: addr): string +function software_endpoint_name(id: conn_id, host_a: addr): string { - return fmt("%s %s", host, (host == id$orig_h ? "client" : "server")); + return fmt("%s %s", host_a, (host_a == id$orig_h ? "client" : "server")); } # Convert a version into a string "a.b.c-x". @@ -366,10 +388,10 @@ function software_fmt(i: Info): string event software_register(id: conn_id, info: Info) { # Host already known? - if ( info$host !in tracked ) - tracked[info$host] = table(); + if ( info$host_a !in tracked ) + tracked[info$host_a] = table(); - local ts = tracked[info$host]; + local ts = tracked[info$host_a]; # Software already registered for this host? We don't want to endlessly # log the same thing. if ( info$name in ts ) @@ -389,7 +411,7 @@ event software_register(id: conn_id, info: Info) function found(id: conn_id, info: Info): bool { - if ( info$force_log || addr_matches_host(info$host, asset_tracking) ) + if ( info$force_log || addr_matches_host(info$host_a, asset_tracking) ) { event software_register(id, info); return T; diff --git a/scripts/policy/protocols/http/detect-webapps.bro b/scripts/policy/protocols/http/detect-webapps.bro index 4a94d1adbd..63f481422a 100644 --- a/scripts/policy/protocols/http/detect-webapps.bro +++ b/scripts/policy/protocols/http/detect-webapps.bro @@ -23,7 +23,7 @@ event signature_match(state: signature_state, msg: string, data: string) &priori if ( /^webapp-/ !in state$sig_id ) return; local c = state$conn; - local si = Software::parse(msg, c$id$resp_h, WEB_APPLICATION); + local si = Software::parse_with_port(msg, c$id$resp_h, c$id$resp_p, WEB_APPLICATION); si$url = build_url_http(c$http); if ( c$id$resp_h in Software::tracked && si$name in Software::tracked[c$id$resp_h] ) diff --git a/scripts/policy/protocols/http/software.bro b/scripts/policy/protocols/http/software.bro index 8732634359..0a07ba0331 100644 --- a/scripts/policy/protocols/http/software.bro +++ b/scripts/policy/protocols/http/software.bro @@ -25,13 +25,13 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr else { if ( name == "SERVER" ) - Software::found(c$id, Software::parse(value, c$id$resp_h, SERVER)); + Software::found(c$id, Software::parse_with_port(value, c$id$resp_h, c$id$resp_p, SERVER)); else if ( name == "X-POWERED-BY" ) - Software::found(c$id, Software::parse(value, c$id$resp_h, APPSERVER)); + Software::found(c$id, Software::parse_with_port(value, c$id$resp_h, c$id$resp_p, APPSERVER)); else if ( name == "MICROSOFTSHAREPOINTTEAMSERVICES" ) { value = cat("SharePoint/", value); - Software::found(c$id, Software::parse(value, c$id$resp_h, APPSERVER)); + Software::found(c$id, Software::parse_with_port(value, c$id$resp_h, c$id$resp_p, APPSERVER)); } } } diff --git a/scripts/policy/protocols/ssh/software.bro b/scripts/policy/protocols/ssh/software.bro index a239655270..0bb6ebc43f 100644 --- a/scripts/policy/protocols/ssh/software.bro +++ b/scripts/policy/protocols/ssh/software.bro @@ -24,6 +24,6 @@ event ssh_server_version(c: connection, version: string) &priority=4 { # Get rid of the protocol information when passing to the software framework. local cleaned_version = sub(version, /SSH[0-9\.\-]{2,}/, ""); - local si = Software::parse(cleaned_version, c$id$resp_h, SERVER); + local si = Software::parse_with_port(cleaned_version, c$id$resp_h, c$id$resp_p, SERVER); Software::found(c$id, si); } From ca17a1cf46ce50a84038eecfa5da401351777d4b Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 5 Dec 2011 16:18:54 -0800 Subject: [PATCH 148/964] make logging framework send the protocol to the writer. for use in future writers, that have a special type for port, which includes the protocol. --- src/LogMgr.cc | 32 +++++++++++++++++++++++++++++--- src/LogMgr.h | 4 ++++ src/LogWriterAscii.cc | 5 ++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 0b706f6417..8873b22b46 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -114,6 +114,10 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } + +// if ( type == TYPE_PORT && present ) +// delete val.port_val.proto; + } bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) @@ -186,9 +190,12 @@ bool LogVal::Read(SerializationFormat* fmt) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: return fmt->Read(&val.uint_val, "uint"); + case TYPE_PORT: + val.port_val.proto = new string; + return fmt->Read(&val.port_val.port, "port") && fmt->Read(val.port_val.proto, "proto"); + case TYPE_SUBNET: { uint32 net[4]; @@ -301,9 +308,11 @@ bool LogVal::Write(SerializationFormat* fmt) const case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: return fmt->Write(val.uint_val, "uint"); + case TYPE_PORT: + return fmt->Write(val.port_val.port, "port") && fmt->Write(*val.port_val.proto, "proto"); + case TYPE_SUBNET: { uint32 net[4]; @@ -1062,6 +1071,22 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return true; } +string LogMgr::TransportProtoToString(TransportProto p) { + switch ( p ) { + case TRANSPORT_UNKNOWN: + return "unknown"; + case TRANSPORT_TCP: + return "tcp"; + case TRANSPORT_UDP: + return "udp"; + case TRANSPORT_ICMP: + return "icmp"; + } + + assert(false); + return ""; +} + LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) { if ( ! ty ) @@ -1093,7 +1118,8 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) break; case TYPE_PORT: - lval->val.uint_val = val->AsPortVal()->Port(); + lval->val.port_val.port = val->AsPortVal()->Port(); + lval->val.port_val.proto = new string(TransportProtoToString(val->AsPortVal()->PortType())); break; case TYPE_SUBNET: diff --git a/src/LogMgr.h b/src/LogMgr.h index 10530960cb..52acd04be5 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -34,10 +34,12 @@ struct LogVal { // types we can log directly. struct set_t { bro_int_t size; LogVal** vals; }; typedef set_t vec_t; + struct port_t { bro_uint_t port; string* proto; }; union _val { bro_int_t int_val; bro_uint_t uint_val; + port_t port_val; uint32 addr_val[NUM_ADDR_WORDS]; subnet_type subnet_val; double double_val; @@ -132,6 +134,8 @@ private: Filter* FindFilter(EnumVal* id, StringVal* filter); WriterInfo* FindWriter(LogWriter* writer); + string TransportProtoToString(TransportProto p); + vector streams; // Indexed by stream enum. }; diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 9b1fda3b62..c449c1a788 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -169,10 +169,13 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: desc->Add(val->val.uint_val); break; + case TYPE_PORT: + desc->Add(val->val.port_val.port); + break; + case TYPE_SUBNET: desc->Add(dotted_addr(val->val.subnet_val.net)); desc->Add("/"); From a0da991030836d53bc669b64ad3de9b4dba34070 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 6 Dec 2011 10:56:26 -0800 Subject: [PATCH 149/964] memleak fix. --- src/LogMgr.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 8873b22b46..729979b4ef 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -115,8 +115,8 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } -// if ( type == TYPE_PORT && present ) -// delete val.port_val.proto; + if ( type == TYPE_PORT && present ) + delete val.port_val.proto; } From 4b3cc95f7206d61614a0193508acc0f60828e3df Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 7 Dec 2011 12:43:15 -0800 Subject: [PATCH 150/964] send enum instead of string --- src/LogMgr.cc | 50 ++++++++++++++++++++++++++------------------------ src/LogMgr.h | 4 +--- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 729979b4ef..58581e2943 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -115,9 +115,6 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } - if ( type == TYPE_PORT && present ) - delete val.port_val.proto; - } bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) @@ -192,9 +189,30 @@ bool LogVal::Read(SerializationFormat* fmt) case TYPE_COUNTER: return fmt->Read(&val.uint_val, "uint"); - case TYPE_PORT: - val.port_val.proto = new string; - return fmt->Read(&val.port_val.port, "port") && fmt->Read(val.port_val.proto, "proto"); + case TYPE_PORT: { + int proto; + if ( ! (fmt->Read(&val.port_val.port, "port") && fmt->Read(&proto, "proto") ) ) { + return false; + } + + switch (proto) { + case 0: + val.port_val.proto = TRANSPORT_UNKNOWN; + break; + case 1: + val.port_val.proto = TRANSPORT_TCP; + break; + case 2: + val.port_val.proto = TRANSPORT_UDP; + break; + case 3: + val.port_val.proto = TRANSPORT_ICMP; + break; + default: + return false; + } + return true; + } case TYPE_SUBNET: { @@ -311,7 +329,7 @@ bool LogVal::Write(SerializationFormat* fmt) const return fmt->Write(val.uint_val, "uint"); case TYPE_PORT: - return fmt->Write(val.port_val.port, "port") && fmt->Write(*val.port_val.proto, "proto"); + return fmt->Write(val.port_val.port, "port") && fmt->Write(val.port_val.proto, "proto"); case TYPE_SUBNET: { @@ -1071,22 +1089,6 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return true; } -string LogMgr::TransportProtoToString(TransportProto p) { - switch ( p ) { - case TRANSPORT_UNKNOWN: - return "unknown"; - case TRANSPORT_TCP: - return "tcp"; - case TRANSPORT_UDP: - return "udp"; - case TRANSPORT_ICMP: - return "icmp"; - } - - assert(false); - return ""; -} - LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) { if ( ! ty ) @@ -1119,7 +1121,7 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) case TYPE_PORT: lval->val.port_val.port = val->AsPortVal()->Port(); - lval->val.port_val.proto = new string(TransportProtoToString(val->AsPortVal()->PortType())); + lval->val.port_val.proto = val->AsPortVal()->PortType(); break; case TYPE_SUBNET: diff --git a/src/LogMgr.h b/src/LogMgr.h index 52acd04be5..8c2c8250f8 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -34,7 +34,7 @@ struct LogVal { // types we can log directly. struct set_t { bro_int_t size; LogVal** vals; }; typedef set_t vec_t; - struct port_t { bro_uint_t port; string* proto; }; + struct port_t { bro_uint_t port; TransportProto proto; }; union _val { bro_int_t int_val; @@ -134,8 +134,6 @@ private: Filter* FindFilter(EnumVal* id, StringVal* filter); WriterInfo* FindWriter(LogWriter* writer); - string TransportProtoToString(TransportProto p); - vector streams; // Indexed by stream enum. }; From e114bdf62730769c0ba1bc6a73096f4e43d6e8a6 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 7 Dec 2011 13:04:46 -0800 Subject: [PATCH 151/964] make LogWriter output the type of data stored inside a set or vector. Now the type output is e.g. vector[string] instead of just vector. --- src/LogMgr.cc | 14 ++++++++++++-- src/LogMgr.h | 6 ++++-- src/LogWriterAscii.cc | 6 ++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 0b706f6417..28e9a2ac1f 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -81,16 +81,18 @@ struct LogMgr::Stream { bool LogField::Read(SerializationFormat* fmt) { int t; + int st; - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type")); + bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&st, "subtype") ); type = (TypeTag) t; + subtype = (TypeTag) st; return success; } bool LogField::Write(SerializationFormat* fmt) const { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type")); + return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); } LogVal::~LogVal() @@ -707,6 +709,14 @@ bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, LogField* field = new LogField(); field->name = new_path; field->type = t->Tag(); + if ( field->type == TYPE_TABLE ) + { + field->subtype = t->AsSetType()->Indices()->PureType()->Tag(); + } + else if ( field->type == TYPE_VECTOR ) + { + field->subtype = t->AsVectorType()->YieldType()->Tag(); + } filter->fields[filter->num_fields - 1] = field; } diff --git a/src/LogMgr.h b/src/LogMgr.h index 10530960cb..3eaba360d5 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -15,10 +15,12 @@ class SerializationFormat; struct LogField { string name; TypeTag type; + // inner type of sets + TypeTag subtype; - LogField() { } + LogField() { subtype = TYPE_VOID; } LogField(const LogField& other) - : name(other.name), type(other.type) { } + : name(other.name), type(other.type), subtype(other.subtype) { } // (Un-)serialize. bool Read(SerializationFormat* fmt); diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 9b1fda3b62..0213f52adb 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -125,6 +125,12 @@ bool LogWriterAscii::DoInit(string path, int num_fields, const LogField* field = fields[i]; names += field->name; types += type_name(field->type); + if ( (field->type == TYPE_TABLE) || (field->type == TYPE_VECTOR) ) + { + types += "["; + types += type_name(field->subtype); + types += "]"; + } } if ( ! (WriteHeaderField("fields", names) From 35fa52ea489cb4bae164279c17a74b0a212aaddf Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 7 Dec 2011 13:10:35 -0800 Subject: [PATCH 152/964] update baseline --- .../http.log | 2 +- .../receiver.test.log | 4 ++-- .../Baseline/scripts.base.frameworks.logging.types/ssh.log | 4 ++-- .../Baseline/scripts.base.frameworks.logging.vec/ssh.log | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log index db9ce497ed..72ba8c0ef4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log @@ -1,5 +1,5 @@ #separator \x09 #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file 1315799856.264750 UWkUyAuUGXf 10.0.1.104 64216 193.40.5.162 80 1 GET lepo.it.da.ut.ee /~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf - Wget/1.12 (darwin10.8.0) 0 346 404 Not Found - - - - - - - text/html - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log index c00e7765d5..320e1a66d8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log @@ -1,5 +1,5 @@ #separator \x09 #path test #fields b i e c p sn a d t iv s sc ss se vc ve -#types bool int enum count port subnet addr double time interval string table table table vector vector -T -42 Test::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315167054.320958 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY +#types bool int enum count port subnet addr double time interval string table[count] table[string] table[string] vector[count] vector[string] +T -42 Test::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1323292199.700588 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log index ffd579c224..9b21b85800 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log @@ -1,5 +1,5 @@ #separator \x09 #path ssh #fields b i e c p sn a d t iv s sc ss se vc ve f -#types bool int enum count port subnet addr double time interval string table table table vector vector func -T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +#types bool int enum count port subnet addr double time interval string table[count] table[string] table[string] vector[count] vector[string] func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1323292210.836187 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log index b9a54404ed..4e05b87ae5 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log @@ -1,5 +1,5 @@ #separator \x09 #path ssh #fields vec -#types vector +#types vector[string] -,2,-,-,5 From 5126b654935b5fa33d50930e7f46bc1717ce053a Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 7 Dec 2011 16:31:44 -0600 Subject: [PATCH 153/964] Add reporter bif/framework documentation. --- scripts/base/frameworks/reporter/main.bro | 21 ++++++++++++-- src/event.bif | 4 +-- src/reporter.bif | 35 +++++++++++++++++++++++ 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/scripts/base/frameworks/reporter/main.bro b/scripts/base/frameworks/reporter/main.bro index e70106f39a..3c19005364 100644 --- a/scripts/base/frameworks/reporter/main.bro +++ b/scripts/base/frameworks/reporter/main.bro @@ -1,21 +1,36 @@ ##! This framework is intended to create an output and filtering path for ##! internal messages/warnings/errors. It should typically be loaded to -##! avoid Bro spewing internal messages to standard error. +##! avoid Bro spewing internal messages to standard error and instead log +##! them to a file in a standard way. Note that this framework deals with +##! the handling of internally-generated reporter messages, for the +##! interface into actually creating reporter messages from the scripting +##! layer, use the built-in functions in :doc:`/scripts/base/reporter.bif`. module Reporter; export { + ## The reporter logging stream identifier. redef enum Log::ID += { LOG }; - + + ## An indicator of reporter message severity. type Level: enum { + ## Informational, not needing specific attention. INFO, + ## Warning of a potential problem. WARNING, + ## A non-fatal error that should be addressed, but doesn't + ## terminate program execution. ERROR }; - + + ## The record type which contains the column fields of the reporter log. type Info: record { + ## The network time at which the reporter event was generated. ts: time &log; + ## The severity of the reporter message. level: Level &log; + ## An info/warning/error message that could have either been + ## generated from the internal Bro core or at the scripting-layer. message: string &log; ## This is the location in a Bro script where the message originated. ## Not all reporter messages will have locations in them though. diff --git a/src/event.bif b/src/event.bif index c0a6311f0e..df6af21d66 100644 --- a/src/event.bif +++ b/src/event.bif @@ -5575,7 +5575,7 @@ event reporter_info%(t: time, msg: string, location: string%) &error_handler; ## location: A (potentially empty) string describing a location associated with the ## warning. ## -## .. bro:see:: reporter_warning reporter_error Reporter::info Reporter::warning +## .. bro:see:: reporter_info reporter_error Reporter::info Reporter::warning ## Reporter::error ## ## .. note:: Bro will not call reporter events recursively. If the handler of any @@ -5594,7 +5594,7 @@ event reporter_warning%(t: time, msg: string, location: string%) &error_handler; ## location: A (potentially empty) string describing a location associated with the ## error. ## -## .. bro:see:: reporter_error reporter_error Reporter::info Reporter::error +## .. bro:see:: reporter_info reporter_warning Reporter::info Reporter::warning ## Reporter::error ## ## .. note:: Bro will not call reporter events recursively. If the handler of any diff --git a/src/reporter.bif b/src/reporter.bif index 6b481eeb79..c0f83205c3 100644 --- a/src/reporter.bif +++ b/src/reporter.bif @@ -1,3 +1,11 @@ +##! The reporter built-in functions allow for the scripting layer to +##! generate messages of varying severity. If no event handlers +##! exist for reporter messages, the messages are output to stderr. +##! If event handlers do exist, it's assumed they take care of determining +##! how/where to output the messages. +##! +##! See :doc:`/scripts/base/frameworks/reporter/main` for a convenient +##! reporter message logging framework. module Reporter; @@ -5,6 +13,13 @@ module Reporter; #include "NetVar.h" %%} +## Generates an informational message. +## +## msg: The informational message to report. +## +## Returns: Always true. +## +## .. bro:see:: reporter_info function Reporter::info%(msg: string%): bool %{ reporter->PushLocation(frame->GetCall()->GetLocationInfo()); @@ -13,6 +28,13 @@ function Reporter::info%(msg: string%): bool return new Val(1, TYPE_BOOL); %} +## Generates a message that warns of a potential problem. +## +## msg: The warning message to report. +## +## Returns: Always true. +## +## .. bro:see:: reporter_warning function Reporter::warning%(msg: string%): bool %{ reporter->PushLocation(frame->GetCall()->GetLocationInfo()); @@ -21,6 +43,14 @@ function Reporter::warning%(msg: string%): bool return new Val(1, TYPE_BOOL); %} +## Generates a non-fatal error indicative of a definite problem that should +## be addressed. Program execution does not terminate. +## +## msg: The error message to report. +## +## Returns: Always true. +## +## .. bro:see:: reporter_error function Reporter::error%(msg: string%): bool %{ reporter->PushLocation(frame->GetCall()->GetLocationInfo()); @@ -29,6 +59,11 @@ function Reporter::error%(msg: string%): bool return new Val(1, TYPE_BOOL); %} +## Generates a fatal error on stderr and terminates program execution. +## +## msg: The error message to report. +## +## Returns: Always true. function Reporter::fatal%(msg: string%): bool %{ reporter->PushLocation(frame->GetCall()->GetLocationInfo()); From 7e3ebc181755aacc65670e9ddd0bcfa38776569b Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 7 Dec 2011 15:03:36 -0800 Subject: [PATCH 154/964] forgotten policy files. --- .../frameworks/software/version-changes.bro | 4 +- .../policy/frameworks/software/vulnerable.bro | 2 +- .../frameworks/software/version-parsing.bro | 98 +++++++++---------- 3 files changed, 52 insertions(+), 52 deletions(-) diff --git a/scripts/policy/frameworks/software/version-changes.bro b/scripts/policy/frameworks/software/version-changes.bro index 6d46151f0f..8365f28ae4 100644 --- a/scripts/policy/frameworks/software/version-changes.bro +++ b/scripts/policy/frameworks/software/version-changes.bro @@ -27,7 +27,7 @@ export { event log_software(rec: Info) { - local ts = tracked[rec$host]; + local ts = tracked[rec$host_a]; if ( rec$name in ts ) { @@ -40,7 +40,7 @@ event log_software(rec: Info) network_time(), rec$software_type, software_fmt_version(old$version), software_fmt(rec), rec$software_type); - NOTICE([$note=Software_Version_Change, $src=rec$host, + NOTICE([$note=Software_Version_Change, $src=rec$host_a, $msg=msg, $sub=software_fmt(rec)]); } } diff --git a/scripts/policy/frameworks/software/vulnerable.bro b/scripts/policy/frameworks/software/vulnerable.bro index 0ce949b83d..cdf7db89fc 100644 --- a/scripts/policy/frameworks/software/vulnerable.bro +++ b/scripts/policy/frameworks/software/vulnerable.bro @@ -18,6 +18,6 @@ event log_software(rec: Info) if ( rec$name in vulnerable_versions && cmp_versions(rec$version, vulnerable_versions[rec$name]) <= 0 ) { - NOTICE([$note=Vulnerable_Version, $src=rec$host, $msg=software_fmt(rec)]); + NOTICE([$note=Vulnerable_Version, $src=rec$host_a, $msg=software_fmt(rec)]); } } diff --git a/testing/btest/scripts/base/frameworks/software/version-parsing.bro b/testing/btest/scripts/base/frameworks/software/version-parsing.bro index dda3edea4b..8833b3aab6 100644 --- a/testing/btest/scripts/base/frameworks/software/version-parsing.bro +++ b/testing/btest/scripts/base/frameworks/software/version-parsing.bro @@ -2,112 +2,112 @@ # @TEST-EXEC: btest-diff output global ts = network_time(); -global host = 0.0.0.0; +global host_a = 0.0.0.0; global matched_software: table[string] of Software::Info = { ["OpenSSH_4.4"] = - [$name="OpenSSH", $version=[$major=4,$minor=4], $host=host, $ts=ts], + [$name="OpenSSH", $version=[$major=4,$minor=4], $host_a=host_a, $ts=ts], ["OpenSSH_5.2"] = - [$name="OpenSSH", $version=[$major=5,$minor=2], $host=host, $ts=ts], + [$name="OpenSSH", $version=[$major=5,$minor=2], $host_a=host_a, $ts=ts], ["Apache/2.0.63 (Unix) mod_auth_kerb/5.3 mod_ssl/2.0.63 OpenSSL/0.9.7a mod_fastcgi/2.4.2"] = - [$name="Apache", $version=[$major=2,$minor=0,$minor2=63,$addl="Unix"], $host=host, $ts=ts], + [$name="Apache", $version=[$major=2,$minor=0,$minor2=63,$addl="Unix"], $host_a=host_a, $ts=ts], ["Apache/1.3.19 (Unix)"] = - [$name="Apache", $version=[$major=1,$minor=3,$minor2=19,$addl="Unix"], $host=host, $ts=ts], + [$name="Apache", $version=[$major=1,$minor=3,$minor2=19,$addl="Unix"], $host_a=host_a, $ts=ts], ["ProFTPD 1.2.5rc1 Server (Debian)"] = - [$name="ProFTPD", $version=[$major=1,$minor=2,$minor2=5,$addl="rc1"], $host=host, $ts=ts], + [$name="ProFTPD", $version=[$major=1,$minor=2,$minor2=5,$addl="rc1"], $host_a=host_a, $ts=ts], ["wu-2.4.2-academ[BETA-18-VR14](1)"] = - [$name="wu", $version=[$major=2,$minor=4,$minor2=2,$addl="academ"], $host=host, $ts=ts], + [$name="wu", $version=[$major=2,$minor=4,$minor2=2,$addl="academ"], $host_a=host_a, $ts=ts], ["wu-2.6.2(1)"] = - [$name="wu", $version=[$major=2,$minor=6,$minor2=2,$addl="1"], $host=host, $ts=ts], + [$name="wu", $version=[$major=2,$minor=6,$minor2=2,$addl="1"], $host_a=host_a, $ts=ts], ["Java1.2.2-JDeveloper"] = - [$name="Java", $version=[$major=1,$minor=2,$minor2=2,$addl="JDeveloper"], $host=host, $ts=ts], + [$name="Java", $version=[$major=1,$minor=2,$minor2=2,$addl="JDeveloper"], $host_a=host_a, $ts=ts], ["Java/1.6.0_13"] = - [$name="Java", $version=[$major=1,$minor=6,$minor2=0,$addl="13"], $host=host, $ts=ts], + [$name="Java", $version=[$major=1,$minor=6,$minor2=0,$addl="13"], $host_a=host_a, $ts=ts], ["Python-urllib/3.1"] = - [$name="Python-urllib", $version=[$major=3,$minor=1], $host=host, $ts=ts], + [$name="Python-urllib", $version=[$major=3,$minor=1], $host_a=host_a, $ts=ts], ["libwww-perl/5.820"] = - [$name="libwww-perl", $version=[$major=5,$minor=820], $host=host, $ts=ts], + [$name="libwww-perl", $version=[$major=5,$minor=820], $host_a=host_a, $ts=ts], ["Wget/1.9+cvs-stable (Red Hat modified)"] = - [$name="Wget", $version=[$major=1,$minor=9,$addl="+cvs"], $host=host, $ts=ts], + [$name="Wget", $version=[$major=1,$minor=9,$addl="+cvs"], $host_a=host_a, $ts=ts], ["Wget/1.11.4 (Red Hat modified)"] = - [$name="Wget", $version=[$major=1,$minor=11,$minor2=4,$addl="Red Hat modified"], $host=host, $ts=ts], + [$name="Wget", $version=[$major=1,$minor=11,$minor2=4,$addl="Red Hat modified"], $host_a=host_a, $ts=ts], ["curl/7.15.1 (i486-pc-linux-gnu) libcurl/7.15.1 OpenSSL/0.9.8a zlib/1.2.3 libidn/0.5.18"] = - [$name="curl", $version=[$major=7,$minor=15,$minor2=1,$addl="i486-pc-linux-gnu"], $host=host, $ts=ts], + [$name="curl", $version=[$major=7,$minor=15,$minor2=1,$addl="i486-pc-linux-gnu"], $host_a=host_a, $ts=ts], ["Apache"] = - [$name="Apache", $host=host, $ts=ts], + [$name="Apache", $host_a=host_a, $ts=ts], ["Zope/(Zope 2.7.8-final, python 2.3.5, darwin) ZServer/1.1 Plone/Unknown"] = - [$name="Zope/(Zope", $version=[$major=2,$minor=7,$minor2=8,$addl="final"], $host=host, $ts=ts], + [$name="Zope/(Zope", $version=[$major=2,$minor=7,$minor2=8,$addl="final"], $host_a=host_a, $ts=ts], ["The Bat! (v2.00.9) Personal"] = - [$name="The Bat!", $version=[$major=2,$minor=0,$minor2=9,$addl="Personal"], $host=host, $ts=ts], + [$name="The Bat!", $version=[$major=2,$minor=0,$minor2=9,$addl="Personal"], $host_a=host_a, $ts=ts], ["Flash/10,2,153,1"] = - [$name="Flash", $version=[$major=10,$minor=2,$minor2=153,$addl="1"], $host=host, $ts=ts], + [$name="Flash", $version=[$major=10,$minor=2,$minor2=153,$addl="1"], $host_a=host_a, $ts=ts], ["mt2/1.2.3.967 Oct 13 2010-13:40:24 ord-pixel-x2 pid 0x35a3 13731"] = - [$name="mt2", $version=[$major=1,$minor=2,$minor2=3,$addl="967"], $host=host, $ts=ts], + [$name="mt2", $version=[$major=1,$minor=2,$minor2=3,$addl="967"], $host_a=host_a, $ts=ts], ["CacheFlyServe v26b"] = - [$name="CacheFlyServe", $version=[$major=26,$addl="b"], $host=host, $ts=ts], + [$name="CacheFlyServe", $version=[$major=26,$addl="b"], $host_a=host_a, $ts=ts], ["Apache/2.0.46 (Win32) mod_ssl/2.0.46 OpenSSL/0.9.7b mod_jk2/2.0.4"] = - [$name="Apache", $version=[$major=2,$minor=0,$minor2=46,$addl="Win32"], $host=host, $ts=ts], + [$name="Apache", $version=[$major=2,$minor=0,$minor2=46,$addl="Win32"], $host_a=host_a, $ts=ts], # I have no clue how I'd support this without a special case. #["Apache mod_fcgid/2.3.6 mod_auth_passthrough/2.1 mod_bwlimited/1.4 FrontPage/5.0.2.2635"] = - # [$name="Apache", $version=[], $host=host, $ts=ts], + # [$name="Apache", $version=[], $host_a=host_a, $ts=ts], ["Apple iPhone v4.3.1 Weather v1.0.0.8G4"] = - [$name="Apple iPhone", $version=[$major=4,$minor=3,$minor2=1,$addl="Weather"], $host=host, $ts=ts], + [$name="Apple iPhone", $version=[$major=4,$minor=3,$minor2=1,$addl="Weather"], $host_a=host_a, $ts=ts], ["Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5"] = - [$name="Safari", $version=[$major=5,$minor=0,$minor2=2,$addl="Mobile"], $host=host, $ts=ts], + [$name="Safari", $version=[$major=5,$minor=0,$minor2=2,$addl="Mobile"], $host_a=host_a, $ts=ts], ["Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.205 Safari/534.16"] = - [$name="Chrome", $version=[$major=10,$minor=0,$minor2=648,$addl="205"], $host=host, $ts=ts], + [$name="Chrome", $version=[$major=10,$minor=0,$minor2=648,$addl="205"], $host_a=host_a, $ts=ts], ["Opera/9.80 (Windows NT 6.1; U; sv) Presto/2.7.62 Version/11.01"] = - [$name="Opera", $version=[$major=11,$minor=1], $host=host, $ts=ts], + [$name="Opera", $version=[$major=11,$minor=1], $host_a=host_a, $ts=ts], ["Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2.11) Gecko/20101013 Lightning/1.0b2 Thunderbird/3.1.5"] = - [$name="Thunderbird", $version=[$major=3,$minor=1,$minor2=5], $host=host, $ts=ts], + [$name="Thunderbird", $version=[$major=3,$minor=1,$minor2=5], $host_a=host_a, $ts=ts], ["iTunes/9.0 (Macintosh; Intel Mac OS X 10.5.8) AppleWebKit/531.9"] = - [$name="iTunes", $version=[$major=9,$minor=0,$addl="Macintosh"], $host=host, $ts=ts], + [$name="iTunes", $version=[$major=9,$minor=0,$addl="Macintosh"], $host_a=host_a, $ts=ts], ["Java1.3.1_04"] = - [$name="Java", $version=[$major=1,$minor=3,$minor2=1,$addl="04"], $host=host, $ts=ts], + [$name="Java", $version=[$major=1,$minor=3,$minor2=1,$addl="04"], $host_a=host_a, $ts=ts], ["Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"] = - [$name="Safari", $version=[$major=4,$minor=0,$addl="Mobile"], $host=host, $ts=ts], + [$name="Safari", $version=[$major=4,$minor=0,$addl="Mobile"], $host_a=host_a, $ts=ts], ["Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-us) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27"] = - [$name="Safari", $version=[$major=5,$minor=0,$minor2=4], $host=host, $ts=ts], + [$name="Safari", $version=[$major=5,$minor=0,$minor2=4], $host_a=host_a, $ts=ts], ["Mozilla/5.0 (iPod; U; CPU iPhone OS 4_0 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8A293 Safari/6531.22.7"] = - [$name="Safari", $version=[$major=4,$minor=0,$minor2=5,$addl="Mobile"], $host=host, $ts=ts], + [$name="Safari", $version=[$major=4,$minor=0,$minor2=5,$addl="Mobile"], $host_a=host_a, $ts=ts], ["Opera/9.80 (J2ME/MIDP; Opera Mini/9.80 (S60; SymbOS; Opera Mobi/23.348; U; en) Presto/2.5.25 Version/10.54"] = - [$name="Opera Mini", $version=[$major=10,$minor=54], $host=host, $ts=ts], + [$name="Opera Mini", $version=[$major=10,$minor=54], $host_a=host_a, $ts=ts], ["Opera/9.80 (J2ME/MIDP; Opera Mini/5.0.18741/18.794; U; en) Presto/2.4.15"] = - [$name="Opera Mini", $version=[$major=5,$minor=0,$minor2=18741], $host=host, $ts=ts], + [$name="Opera Mini", $version=[$major=5,$minor=0,$minor2=18741], $host_a=host_a, $ts=ts], ["Opera/9.80 (Windows NT 5.1; Opera Mobi/49; U; en) Presto/2.4.18 Version/10.00"] = - [$name="Opera Mobi", $version=[$major=10,$minor=0], $host=host, $ts=ts], + [$name="Opera Mobi", $version=[$major=10,$minor=0], $host_a=host_a, $ts=ts], ["Mozilla/4.0 (compatible; MSIE 8.0; Android 2.2.2; Linux; Opera Mobi/ADR-1103311355; en) Opera 11.00"] = - [$name="Opera", $version=[$major=11,$minor=0], $host=host, $ts=ts], + [$name="Opera", $version=[$major=11,$minor=0], $host_a=host_a, $ts=ts], ["Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.2) Gecko/20040804 Netscape/7.2 (ax)"] = - [$name="Netscape", $version=[$major=7,$minor=2], $host=host, $ts=ts], + [$name="Netscape", $version=[$major=7,$minor=2], $host_a=host_a, $ts=ts], ["Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; GTB5; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; InfoPath.2)"] = - [$name="MSIE", $version=[$major=7,$minor=0], $host=host, $ts=ts], + [$name="MSIE", $version=[$major=7,$minor=0], $host_a=host_a, $ts=ts], ["Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.1; Media Center PC 3.0; .NET CLR 1.0.3705; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1)"] = - [$name="MSIE", $version=[$major=7,$minor=0,$addl="b"], $host=host, $ts=ts], + [$name="MSIE", $version=[$major=7,$minor=0,$addl="b"], $host_a=host_a, $ts=ts], ["Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Tablet PC 2.0; InfoPath.2; InfoPath.3)"] = - [$name="MSIE", $version=[$major=8,$minor=0], $host=host, $ts=ts], + [$name="MSIE", $version=[$major=8,$minor=0], $host_a=host_a, $ts=ts], ["Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)"] = - [$name="MSIE", $version=[$major=9,$minor=0], $host=host, $ts=ts], + [$name="MSIE", $version=[$major=9,$minor=0], $host_a=host_a, $ts=ts], ["Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; InfoPath.3; Creative AutoUpdate v1.40.02)"] = - [$name="MSIE", $version=[$major=9,$minor=0], $host=host, $ts=ts], + [$name="MSIE", $version=[$major=9,$minor=0], $host_a=host_a, $ts=ts], ["Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)"] = - [$name="MSIE", $version=[$major=10,$minor=0], $host=host, $ts=ts], + [$name="MSIE", $version=[$major=10,$minor=0], $host_a=host_a, $ts=ts], ["The Bat! (3.0.1 RC3) Professional"] = - [$name="The Bat!", $version=[$major=3,$minor=0,$minor2=1,$addl="RC3"], $host=host, $ts=ts], + [$name="The Bat!", $version=[$major=3,$minor=0,$minor2=1,$addl="RC3"], $host_a=host_a, $ts=ts], # This is an FTP client (found with CLNT command) ["Total Commander"] = - [$name="Total Commander", $version=[], $host=host, $ts=ts], + [$name="Total Commander", $version=[], $host_a=host_a, $ts=ts], ["(vsFTPd 2.0.5)"] = - [$name="vsFTPd", $version=[$major=2,$minor=0,$minor2=5], $host=host, $ts=ts], + [$name="vsFTPd", $version=[$major=2,$minor=0,$minor2=5], $host_a=host_a, $ts=ts], ["Apple Mail (2.1084)"] = - [$name="Apple Mail", $version=[$major=2,$minor=1084], $host=host, $ts=ts], + [$name="Apple Mail", $version=[$major=2,$minor=1084], $host_a=host_a, $ts=ts], }; event bro_init() { for ( sw in matched_software ) { - local output = Software::parse(sw, host, Software::UNKNOWN); + local output = Software::parse(sw, host_a, Software::UNKNOWN); local baseline: Software::Info; baseline = matched_software[sw]; if ( baseline$name == output$name && From 80b24513e770c7f935be2da5297b2ea53bfadae3 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 7 Dec 2011 17:08:38 -0600 Subject: [PATCH 155/964] Fix Broxygen confusing scoped id at start of line as function parameter. --- doc/scripts/example.bro | 2 ++ src/scan.l | 4 ++-- testing/btest/Baseline/doc.autogen-reST-example/example.rst | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/scripts/example.bro b/doc/scripts/example.bro index d2d0ec6879..e7c86f4bd3 100644 --- a/doc/scripts/example.bro +++ b/doc/scripts/example.bro @@ -183,6 +183,8 @@ export { ## Summarize "an_event" here. ## Give more details about "an_event" here. + ## Example::an_event should not be confused as a parameter + ## ## name: describe the argument here global an_event: event(name: string); diff --git a/src/scan.l b/src/scan.l index 7ebd7894e1..b167d69943 100644 --- a/src/scan.l +++ b/src/scan.l @@ -167,7 +167,7 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) return TOK_POST_DOC; } -##{OWS}{ID}:.* { +##{OWS}{ID}:[^:].* { const char* id_start = skip_whitespace(yytext + 2); yylval.str = copy_string(canon_doc_func_param(id_start).c_str()); return TOK_DOC; @@ -181,7 +181,7 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) } } -##{OWS}{ID}:.* { +##{OWS}{ID}:[^:].* { if ( generate_documentation ) { // Comment is documenting either a function parameter or return type, diff --git a/testing/btest/Baseline/doc.autogen-reST-example/example.rst b/testing/btest/Baseline/doc.autogen-reST-example/example.rst index 7c61277187..fbda81771b 100644 --- a/testing/btest/Baseline/doc.autogen-reST-example/example.rst +++ b/testing/btest/Baseline/doc.autogen-reST-example/example.rst @@ -217,6 +217,8 @@ Events Summarize "an_event" here. Give more details about "an_event" here. + Example::an_event should not be confused as a parameter + :param name: describe the argument here From 04e2773d30337af3162d2d142928ab3988da4e05 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 8 Dec 2011 13:06:45 -0500 Subject: [PATCH 156/964] Fixed some bugs with capturing data in the base DNS script. --- scripts/base/protocols/dns/main.bro | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro index d738e4943c..5f2f68e807 100644 --- a/scripts/base/protocols/dns/main.bro +++ b/scripts/base/protocols/dns/main.bro @@ -139,16 +139,12 @@ function set_session(c: connection, msg: dns_msg, is_query: bool) event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) &priority=5 { set_session(c, msg, F); - - c$dns$AA = msg$AA; - c$dns$RA = msg$RA; - if ( ! c$dns?$TTLs ) - c$dns$TTLs = vector(); - c$dns$TTLs[|c$dns$TTLs|] = ans$TTL; - if ( ans$answer_type == DNS_ANS ) { + c$dns$AA = msg$AA; + c$dns$RA = msg$RA; + if ( msg$id in c$dns_state$finished_answers ) event conn_weird("dns_reply_seen_after_done", c, ""); @@ -157,6 +153,10 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) if ( ! c$dns?$answers ) c$dns$answers = vector(); c$dns$answers[|c$dns$answers|] = reply; + + if ( ! c$dns?$TTLs ) + c$dns$TTLs = vector(); + c$dns$TTLs[|c$dns$TTLs|] = ans$TTL; } if ( c$dns?$answers && |c$dns$answers| == c$dns$total_answers ) From 3391270527f0eb75dba943d9dd75531032c462e7 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 8 Dec 2011 14:25:52 -0500 Subject: [PATCH 157/964] Fixed a really dumb bug that was causing the malware hash registry script to break. --- scripts/policy/protocols/http/detect-MHR.bro | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/policy/protocols/http/detect-MHR.bro b/scripts/policy/protocols/http/detect-MHR.bro index 3b2e8bf968..fca3bb1ac9 100644 --- a/scripts/policy/protocols/http/detect-MHR.bro +++ b/scripts/policy/protocols/http/detect-MHR.bro @@ -7,9 +7,12 @@ @load base/frameworks/notice @load base/protocols/http +module HTTP; + export { redef enum Notice::Type += { - ## If the MD5 sum of a file transferred over HTTP + ## The MD5 sum of a file transferred over HTTP matched in the + ## malware hash registry. Malware_Hash_Registry_Match }; } From 6d3b29b0ec085be2f13ed541a7bef5852270eee8 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 8 Dec 2011 15:55:38 -0600 Subject: [PATCH 158/964] Add builtin type documentation, clean up format of attribute docs. --- doc/scripts/builtins.rst | 578 +++++++++++++++++++++++++++++++++++---- 1 file changed, 518 insertions(+), 60 deletions(-) diff --git a/doc/scripts/builtins.rst b/doc/scripts/builtins.rst index 6abbd9110b..942e053b5f 100644 --- a/doc/scripts/builtins.rst +++ b/doc/scripts/builtins.rst @@ -6,159 +6,617 @@ Types The Bro scripting language supports the following built-in types. -.. TODO: add documentation - -.. bro:type:: void - .. bro:type:: bool + Reflects a value with one of two meanings: true or false. The two + ``bool`` constants are ``T`` and ``F``. + .. bro:type:: int + A numeric type representing a signed integer. An ``int`` constant + is a string of digits preceded by a ``+`` or ``-`` sign, e.g. + ``-42`` or ``+5``. When using type inferencing use care so that the + intended type is inferred, e.g. ``local size_difference = 0`` will + infer the :bro:type:`count` while ``local size_difference = +0`` + will infer :bro:type:`int`. + .. bro:type:: count + A numeric type representing an unsigned integer. A ``count`` + constant is a string of digits, e.g. ``1234`` or ``0``. + .. bro:type:: counter + An alias to :bro:type:`count` + +.. TODO: is there anything special about this type? + .. bro:type:: double + A numeric type representing a double-precision floating-point + number. Floating-point constants are written as a string of digits + with an optional decimal point, optional scale-factor in scientific + notation, and optional ``+`` or ``-`` sign. Examples are ``-1234``, + ``-1234e0``, ``3.14159``, and ``.003e-23``. + .. bro:type:: time + A temporal type representing an absolute time. There is currently + no way to specify a ``time`` constant, but one can use the + :bro:id:`current_time` or :bro:id:`network_time` built-in functions + to assign a value to a ``time``-typed variable. + .. bro:type:: interval + A temporal type representing a relative time. An ``interval`` + constant can be written as a numeric constant followed by a time + unit where the time unit is one of ``usec``, ``sec``, ``min``, + ``hr``, or ``day`` which respectively represent microseconds, + seconds, minutes, hours, and days. Whitespace between the numeric + constant and time unit is optional. Appending the letter "s" to the + time unit in order to pluralize it is also optional (to no semantic + effect). Examples of ``interval`` constants are ``3.5 min`` and + ``3.5mins``. An ``interval`` can also be negated, for example ``- + 12 hr`` represents "twelve hours in the past". Intervals also + support addition, subtraction, multiplication, division, and + comparison operations. + .. bro:type:: string + A type used to hold character-string values which represent text. + String constants are created by enclosing text in double quotes (") + and the backslash character (\) introduces escape sequences. + + Note that Bro represents strings internally as a count and vector of + bytes rather than a NUL-terminated byte string (although string + constants are also automatically NUL-terminated). This is because + network traffic can easily introduce NULs into strings either by + nature of an application, inadvertently, or maliciously. And while + NULs are allowed in Bro strings, when present in strings passed as + arguments to many functions, a run-time error can occur as their + presence likely indicates a sort of problem. In that case, the + string will also only be represented to the user as the literal + "" string. + .. bro:type:: pattern + A type representing regular-expression patterns which can be used + for fast text-searching operations. Pattern constants are created + by enclosing text within forward slashes (/) and is the same syntax + as the patterns supported by the `flex lexical analyzer + `_. The speed of + regular expression matching does not depend on the complexity or + size of the patterns. Patterns support two types of matching, exact + and embedded. + + In exact matching the ``==`` equality relational operator is used + with one :bro:type:`string` operand and one :bro:type:`pattern` + operand to check whether the full string exactly matches the + pattern. In this case, the ``^`` beginning-of-line and ``$`` + end-of-line anchors are redundant since pattern is implicitly + anchored to the beginning and end of the line to facilitate an exact + match. For example:: + + "foo" == /foo|bar/ + + yields true, while:: + + /foo|bar/ == "foobar" + + yields false. The ``!=`` operator would yield the negation of ``==``. + + In embedded matching the ``in`` operator is again used with one + :bro:type:`string` operand and one :bro:type:`pattern` operand + (which must be on the left-hand side), but tests whether the pattern + appears anywhere within the given string. For example:: + + /foo|bar/ in "foobar" + + yields true, while:: + + /^oob/ in "foobar" + + is false since "oob" does not appear at the start of "foobar". The + ``!in`` operator would yield the negation of ``in``. + .. bro:type:: enum + A type allowing the specification of a set of related values that + have no further structure. The only operations allowed on + enumerations are equality comparisons and they do not have + associated values or ordering. An example declaration: + + .. code:: bro + + type color: enum { Red, White, Blue, }; + + The last comma is after ``Blue`` is optional. + .. bro:type:: timer +.. TODO: is this a type that's exposed to users? + .. bro:type:: port + A type representing transport-level port numbers. Besides TCP and + UDP ports, there is a concept of an ICMP "port" where the source + port is the ICMP message type and the destination port the ICMP + message code. A ``port`` constant is written as an unsigned integer + followed by one of ``/tcp``, ``/udp``, ``/icmp``, or ``/unknown``. + + Ports can be compared for equality and also for ordering. When + comparing order across transport-level protocols, ``/unknown`` < + ``/tcp`` < ``/udp`` < ``icmp``, for example ``65535/tcp`` is smaller + than ``0/udp``. + .. bro:type:: addr -.. bro:type:: net + A type representing an IP address. Currently, Bro defaults to only + supporting IPv4 addresses unless configured/built with + ``--enable-brov6``, in which case, IPv6 addresses are supported. + + IPv4 address constants are written in "dotted quad" format, + ``A1.A2.A3.A4``, where Ai all lie between 0 and 255. + + IPv6 address constants are written as colon-separated hexadecimal form + as described by :rfc:`2373`. + + Hostname constants can also be used, but since a hostname can + correspond to multiple IP addresses, the type of such variable is a + :bro:type:`set` of :bro:type:`addr` elements. For example: + + .. code:: bro + + local a = www.google.com; + + Addresses can be compared for (in)equality using ``==`` and ``!=``. + They can also be masked with ``/`` to produce a :bro:type:`subnet`: + + .. code:: bro + + local a: addr = 192.168.1.100; + local s: subnet = 192.168.0.0/16; + if ( a/16 == s ) + print "true"; + + And checked for inclusion within a :bro:type:`subnet` using ``in`` : + + .. code:: bro + + local a: addr = 192.168.1.100; + local s: subnet = 192.168.0.0/16; + if ( a in s ) + print "true"; .. bro:type:: subnet + A type representing a block of IP addresses in CIDR notation. A + ``subnet`` constant is written as an :bro:type:`addr` followed by a + slash (/) and then the network prefix size specified as a decimal + number. For example, ``192.168.0.0/16``. + .. bro:type:: any + Used to bypass strong typing. For example, a function can take an + argument of type ``any`` when it may be of different types. + .. bro:type:: table -.. bro:type:: union + An associate array that maps from one set of values to another. The + values being mapped are termed the *index* or *indices* and the + result of the mapping is called the *yield*. Indexing into tables + is very efficient, and internally it is just a single hash table + lookup. -.. bro:type:: record + The table declaration syntax is:: -.. bro:type:: types + table [ type^+ ] of type -.. bro:type:: func + where *type^+* is one or more types, separated by commas. For example: -.. bro:type:: file + .. code:: bro -.. bro:type:: vector + global a: table[count] of string; -.. TODO: below are kind of "special cases" that bro knows about? + declares a table indexed by :bro:type:`count` values and yielding + :bro:type:`string` values. The yield type can also be more complex: + + .. code:: bro + + global a: table[count] of table[addr, port] of string; + + which declared a table indexed by :bro:type:`count` and yielding + another :bro:type:`table` which is indexed by an :bro:type:`addr` + and :bro:type:`port` to yield a :bro:type:`string`. + + Initialization of tables occurs by enclosing a set of initializers within + braces, for example: + + .. code:: bro + + global t: table[count] of string = { + [11] = "eleven", + [5] = "five", + }; + + Accessing table elements if provided by enclosing values within square + brackets (``[]``), for example: + + .. code:: bro + + t[13] = "thirteen"; + + And membership can be tested with ``in``: + + .. code:: bro + + if ( 13 in t ) + ... + + Iterate over tables with a ``for`` loop: + + .. code:: bro + + local t: table[count] of string; + for ( n in t ) + ... + + local services: table[addr, port] of string; + for ( [a, p] in services ) + ... + + Remove individual table elements with ``delete``: + + .. code:: bro + + delete t[13]; + + Nothing happens if the element with value ``13`` isn't present in + the table. + + Table size can be obtained by placing the table identifier between + vertical pipe (|) characters: + + .. code:: bro + + |t| .. bro:type:: set + A set is like a :bro:type:`table`, but it is a collection of indices + that do not map to any yield value. They are declared with the + syntax:: + + set [ type^+ ] + + where *type^+* is one or more types separated by commas. + + Sets are initialized by listing elements enclosed by curly braces: + + .. code:: bro + + global s: set[port] = { 21/tcp, 23/tcp, 80/tcp, 443/tcp }; + global s2: set[port, string] = { [21/tcp, "ftp"], [23/tcp, "telnet"] }; + + The types are explicitly shown in the example above, but they could + have been left to type inference. + + Set membership is tested with ``in``: + + .. code:: bro + + if ( 21/tcp in s ) + ... + + Elements are added with ``add``: + + .. code:: bro + + add s[22/tcp]; + + And removed with ``delete``: + + .. code:: bro + + delete s[21/tcp]; + + Set size can be obtained by placing the set identifier between + vertical pipe (|) characters: + + .. code:: bro + + |s| + +.. bro:type:: vector + + A vector is like a :bro:type:`table`, except it's always indexed by a + :bro:type:`count`. A vector is declared like: + + .. code:: bro + + global v: vector of string; + + And can be initialized with the vector constructor: + + .. code:: bro + + global v: vector of string = vector("one", "two", "three"); + + Adding an element to a vector involves accessing/assigning it: + + .. code:: bro + + v[3] = "four" + + Note how the vector indexing is 0-based. + + Vector size can be obtained by placing the vector identifier between + vertical pipe (|) characters: + + .. code:: bro + + |v| + +.. bro:type:: record + + A ``record`` is a collection of values. Each value has a field name + and a type. Values do not need to have the same type and the types + have no restrictions. An example record type definition: + + .. code:: bro + + type MyRecordType: record { + c: count; + s: string &optional; + }; + + Access to a record field uses the dollar sign (``$``) operator: + + .. code:: bro + + global r: MyRecordType; + r$c = 13; + + Record assignment can be done field by field or as a whole like: + + .. code:: bro + + r = [$c = 13, $s = "thirteen"]; + + When assigning a whole record value, all fields that are not + :bro:attr:`&optional` or have a :bro:attr:`&default` attribute must + be specified. + + To test for existence of field that is :bro:attr:`&optional`, use the + ``?$`` operator: + + .. code:: bro + + if ( r?$s ) + ... + +.. bro:type:: file + + Bro supports writing to files, but not reading from them. For + example, declare, open, and write to a file and finally close it + like: + + .. code:: bro + + global f: file = open("myfile"); + print f, "hello, world"; + close(f); + + Writing to files like this for logging usually isn't recommend, for better + logging support see :doc:`/logging`. + .. bro:type:: function + Function types in Bro are declared using:: + + function( argument* ): type + + where *argument* is a (possibly empty) comma-separated list of + arguments, and *type* is an optional return type. For example: + + .. code:: bro + + global greeting: function(name: string): string; + + Here ``greeting`` is an identifier with a certain function type. + The function body is not defined yet and ``greeting`` could even + have different function body values at different times. To define + a function including a body value, the syntax is like: + + .. code:: bro + + function greeting(name: string): string + { + return "Hello, " + name; + } + + Note that in the definition above, it's not necessary for us to have + done the first (forward) declaration of ``greeting`` as a function + type, but when it is, the argument list and return type much match + exactly. + + Function types don't need to have a name and can be assigned anonymously: + + .. code:: bro + + greeting = function(name: string): string { return "Hi, " + name; }; + + And finally, the function can be called like: + + .. code:: bro + + print greeting("Dave"); + .. bro:type:: event + Event handlers are nearly identical in both syntax and semantics to + a :bro:type:`function`, with the two differences being that event + handlers have no return type since they never return a value, and + you cannot call an event handler. Instead of directly calling an + event handler from a script, event handler bodies are executed when + they are invoked by one of three different methods: + + - From the event engine + + When the event engine detects an event for which you have + defined a corresponding event handler, it queues an event for + that handler. The handler is invoked as soon as the event + engine finishes processing the current packet and flushing the + invocation of other event handlers that were queued first. + + - With the ``event`` statement from a script + + Immediately queuing invocation of an event handler occurs like: + + .. code:: bro + + event password_exposed(user, password); + + This assumes that ``password_exposed`` was previously declared + as an event handler type with compatible arguments. + + - Via the ``schedule`` expression in a script + + This delays the invocation of event handlers until some time in + the future. For example: + + .. code:: bro + + schedule 5 secs { password_exposed(user, password) }; + + Multiple event handler bodies can be defined for the same event handler + identifier and the body of each will be executed in turn. Ordering + of execution can be influenced with :bro:attr:`&priority`. + Attributes ---------- Attributes occur at the end of type/event declarations and change their -behavior. The syntax is ``&key`` or ``&key=val``, e.g., -``type T: set[count] &read_expire=5min`` or ``event foo() &priority=-3``. -The Bro scripting language supports the following built-in attributes. +behavior. The syntax is ``&key`` or ``&key=val``, e.g., ``type T: +set[count] &read_expire=5min`` or ``event foo() &priority=-3``. The Bro +scripting language supports the following built-in attributes. -## Allows record field to be missing. For example the type -## ``record { a: int, b: port &optional }`` could be instantiated both as -## singleton ``[$a=127.0.0.1]`` or pair ``[$a=127.0.0.1, $b=80/tcp]``. .. bro:attr:: &optional -## Uses a default value for a record field or container elements. For example, -## ``table[int] of string &default="foo" }`` would create table that returns -## The :bro:type:`string` ``"foo"`` for any non-existing index. + Allows record field to be missing. For example the type ``record { + a: int, b: port &optional }`` could be instantiated both as + singleton ``[$a=127.0.0.1]`` or pair ``[$a=127.0.0.1, $b=80/tcp]``. + .. bro:attr:: &default -## Allows for redefinition of initial object values. This is typically used -## with constants, for example, ``const clever = T &redef;`` would allow the -## constant to be redifined at some later point during script execution. + Uses a default value for a record field or container elements. For + example, ``table[int] of string &default="foo" }`` would create + table that returns The :bro:type:`string` ``"foo"`` for any + non-existing index. + .. bro:attr:: &redef -## Rotates a file after a specified interval. + Allows for redefinition of initial object values. This is typically + used with constants, for example, ``const clever = T &redef;`` would + allow the constant to be redifined at some later point during script + execution. + .. bro:attr:: &rotate_interval -## Rotates af file after it has reached a given size in bytes. + Rotates a file after a specified interval. + .. bro:attr:: &rotate_size -## ..TODO: needs to be documented. + Rotates af file after it has reached a given size in bytes. + .. bro:attr:: &add_func -## ..TODO: needs to be documented. +.. TODO: needs to be documented. + .. bro:attr:: &delete_func -## Called right before a container element expires. +.. TODO: needs to be documented. + .. bro:attr:: &expire_func -## Specifies a read expiration timeout for container elements. That is, the -## element expires after the given amount of time since the last time it has -## been read. Note that a write also counts as a read. + Called right before a container element expires. + .. bro:attr:: &read_expire -## Specifies a write expiration timeout for container elements. That is, the -## element expires after the given amount of time since the last time it has -## been written. + Specifies a read expiration timeout for container elements. That is, + the element expires after the given amount of time since the last + time it has been read. Note that a write also counts as a read. + .. bro:attr:: &write_expire -## Specifies a creation expiration timeout for container elements. That is, the -## element expires after the given amount of time since it has been inserted -## into the container, regardless of any reads or writes. + Specifies a write expiration timeout for container elements. That + is, the element expires after the given amount of time since the + last time it has been written. + .. bro:attr:: &create_expire -## Makes a variable persistent, i.e., its value is writen to disk (per default -## at shutdown time). + Specifies a creation expiration timeout for container elements. That + is, the element expires after the given amount of time since it has + been inserted into the container, regardless of any reads or writes. + .. bro:attr:: &persistent -## Synchronizes variable accesses across nodes. The value of a -## ``&synchronized`` variable is automatically propagated to all peers when it -## changes. + Makes a variable persistent, i.e., its value is writen to disk (per + default at shutdown time). + .. bro:attr:: &synchronized -## ..TODO: needs to be documented. + Synchronizes variable accesses across nodes. The value of a + ``&synchronized`` variable is automatically propagated to all peers + when it changes. + .. bro:attr:: &postprocessor -## Encryptes files right before writing them to disk. -## ..TODO: needs to be documented in more detail. +.. TODO: needs to be documented. + .. bro:attr:: &encrypt -## ..TODO: needs to be documented. + Encrypts files right before writing them to disk. + +.. TODO: needs to be documented in more detail. + .. bro:attr:: &match -## Deprecated. Will be removed. +.. TODO: needs to be documented. + .. bro:attr:: &disable_print_hook -## Opens a file in raw mode, i.e., non-ASCII characters are not escaped. + Deprecated. Will be removed. + .. bro:attr:: &raw_output -## Prefers set union to assignment for synchronized state. This attribute is -## used in conjunction with :bro:attr:`synchronized` container types: when the -## same container is updated at two peers with different value, the propagation -## of the state causes a race condition, where the last update succeeds. This -## can cause inconsistencies and can be avoided by unifying the two sets, -## rather than merely overwriting the old value. + Opens a file in raw mode, i.e., non-ASCII characters are not + escaped. + .. bro:attr:: &mergeable -## Specifies the execution priority of an event handler. Higher values are -## executed before lower ones. The default value is 0. + Prefers set union to assignment for synchronized state. This + attribute is used in conjunction with :bro:attr:`&synchronized` + container types: when the same container is updated at two peers + with different value, the propagation of the state causes a race + condition, where the last update succeeds. This can cause + inconsistencies and can be avoided by unifying the two sets, rather + than merely overwriting the old value. + .. bro:attr:: &priority -## Groups event handlers such that those in the same group can be jointly -## activated or deactivated. + Specifies the execution priority of an event handler. Higher values + are executed before lower ones. The default value is 0. + .. bro:attr:: &group -## Writes a record field to the associated log stream. + Groups event handlers such that those in the same group can be + jointly activated or deactivated. + .. bro:attr:: &log + Writes a record field to the associated log stream. + .. bro:attr:: (&tracked) + +.. TODO: needs documented or removed if it's not used anywhere. From e0b7dc04512a228b332e2b9aaddd39300646d5cc Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 8 Dec 2011 14:12:59 -0800 Subject: [PATCH 159/964] fix compile warnings --- src/InputMgr.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 00a1a26311..1fe3a82abe 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -866,8 +866,8 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { //while ( ( ih = i->lastDict->NextEntry(c) ) ) { while ( ( ih = filter->lastDict->NextEntry(lastDictIdxKey, c) ) ) { - ListVal * idx; - Val *val; + ListVal * idx = 0; + Val *val = 0; if ( filter->pred || filter->event ) { idx = filter->tab->RecoverIndex(ih->idxkey); From 311cd1b1165f256a1dc89b58bd3b6219aadc5c10 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 8 Dec 2011 14:25:46 -0800 Subject: [PATCH 160/964] after talking to seth - change host_a field in record back to host. --- scripts/base/frameworks/software/main.bro | 50 +++++----- .../frameworks/software/version-changes.bro | 4 +- .../policy/frameworks/software/vulnerable.bro | 2 +- .../frameworks/software/version-parsing.bro | 98 +++++++++---------- 4 files changed, 77 insertions(+), 77 deletions(-) diff --git a/scripts/base/frameworks/software/main.bro b/scripts/base/frameworks/software/main.bro index 9abac9e575..817ae92e40 100644 --- a/scripts/base/frameworks/software/main.bro +++ b/scripts/base/frameworks/software/main.bro @@ -34,7 +34,7 @@ export { ## The time at which the software was first detected. ts: time &log; ## The IP address detected running the software. - host_a: addr &log; + host: addr &log; ## The Port on which the software is running. Only sensible for server software. host_p: port &log &optional; ## The transport protocol that is being used. Only sensible for server software. @@ -75,13 +75,13 @@ export { ## still many cases where scripts may have to have their own specific ## version parsing though. global parse: function(unparsed_version: string, - host_a: addr, + host: addr, software_type: Type): Info; ## This function is the equivalent to parse for software that has a specific ## source port (i.e. server software) global parse_with_port: function(unparsed_version: string, - host_a: addr, host_p: port, + host: addr, host_p: port, software_type: Type): Info; ## Compare two versions. @@ -117,7 +117,7 @@ event bro_init() } function parse_mozilla(unparsed_version: string, - host_a: addr, + host: addr, software_type: Type): Info { local software_name = ""; @@ -129,7 +129,7 @@ function parse_mozilla(unparsed_version: string, software_name = "Opera"; parts = split_all(unparsed_version, /Opera [0-9\.]*$/); if ( 2 in parts ) - v = parse(parts[2], host_a, software_type)$version; + v = parse(parts[2], host, software_type)$version; } else if ( / MSIE / in unparsed_version ) { @@ -144,7 +144,7 @@ function parse_mozilla(unparsed_version: string, { parts = split_all(unparsed_version, /MSIE [0-9]{1,2}\.*[0-9]*b?[0-9]*/); if ( 2 in parts ) - v = parse(parts[2], host_a, software_type)$version; + v = parse(parts[2], host, software_type)$version; } } else if ( /Version\/.*Safari\// in unparsed_version ) @@ -153,7 +153,7 @@ function parse_mozilla(unparsed_version: string, parts = split_all(unparsed_version, /Version\/[0-9\.]*/); if ( 2 in parts ) { - v = parse(parts[2], host_a, software_type)$version; + v = parse(parts[2], host, software_type)$version; if ( / Mobile\/?.* Safari/ in unparsed_version ) v$addl = "Mobile"; } @@ -163,7 +163,7 @@ function parse_mozilla(unparsed_version: string, parts = split_all(unparsed_version, /(Firefox|Netscape|Thunderbird)\/[0-9\.]*/); if ( 2 in parts ) { - local tmp_s = parse(parts[2], host_a, software_type); + local tmp_s = parse(parts[2], host, software_type); software_name = tmp_s$name; v = tmp_s$version; } @@ -173,7 +173,7 @@ function parse_mozilla(unparsed_version: string, software_name = "Chrome"; parts = split_all(unparsed_version, /Chrome\/[0-9\.]*/); if ( 2 in parts ) - v = parse(parts[2], host_a, software_type)$version; + v = parse(parts[2], host, software_type)$version; } else if ( /^Opera\// in unparsed_version ) { @@ -184,12 +184,12 @@ function parse_mozilla(unparsed_version: string, software_name = parts[2]; parts = split_all(unparsed_version, /Version\/[0-9\.]*/); if ( 2 in parts ) - v = parse(parts[2], host_a, software_type)$version; + v = parse(parts[2], host, software_type)$version; else { parts = split_all(unparsed_version, /Opera Mini\/[0-9\.]*/); if ( 2 in parts ) - v = parse(parts[2], host_a, software_type)$version; + v = parse(parts[2], host, software_type)$version; } } else @@ -197,7 +197,7 @@ function parse_mozilla(unparsed_version: string, software_name = "Opera"; parts = split_all(unparsed_version, /Version\/[0-9\.]*/); if ( 2 in parts ) - v = parse(parts[2], host_a, software_type)$version; + v = parse(parts[2], host, software_type)$version; } } else if ( /AppleWebKit\/[0-9\.]*/ in unparsed_version ) @@ -205,17 +205,17 @@ function parse_mozilla(unparsed_version: string, software_name = "Unspecified WebKit"; parts = split_all(unparsed_version, /AppleWebKit\/[0-9\.]*/); if ( 2 in parts ) - v = parse(parts[2], host_a, software_type)$version; + v = parse(parts[2], host, software_type)$version; } - return [$ts=network_time(), $host_a=host_a, $name=software_name, $version=v, + return [$ts=network_time(), $host=host, $name=software_name, $version=v, $software_type=software_type, $unparsed_version=unparsed_version]; } # Don't even try to understand this now, just make sure the tests are # working. function parse(unparsed_version: string, - host_a: addr, + host: addr, software_type: Type): Info { local software_name = ""; @@ -224,7 +224,7 @@ function parse(unparsed_version: string, # Parse browser-alike versions separately if ( /^(Mozilla|Opera)\/[0-9]\./ in unparsed_version ) { - return parse_mozilla(unparsed_version, host_a, software_type); + return parse_mozilla(unparsed_version, host, software_type); } else { @@ -286,17 +286,17 @@ function parse(unparsed_version: string, v$major = extract_count(version_numbers[1]); } } - return [$ts=network_time(), $host_a=host_a, $name=software_name, + return [$ts=network_time(), $host=host, $name=software_name, $version=v, $unparsed_version=unparsed_version, $software_type=software_type]; } function parse_with_port(unparsed_version: string, - host_a: addr, host_p: port, + host: addr, host_p: port, software_type: Type): Info { local i: Info; - i = parse(unparsed_version, host_a, software_type); + i = parse(unparsed_version, host, software_type); i$host_p = host_p; i$proto = get_port_transport_proto(host_p); @@ -362,9 +362,9 @@ function cmp_versions(v1: Version, v2: Version): int } } -function software_endpoint_name(id: conn_id, host_a: addr): string +function software_endpoint_name(id: conn_id, host: addr): string { - return fmt("%s %s", host_a, (host_a == id$orig_h ? "client" : "server")); + return fmt("%s %s", host, (host == id$orig_h ? "client" : "server")); } # Convert a version into a string "a.b.c-x". @@ -388,10 +388,10 @@ function software_fmt(i: Info): string event software_register(id: conn_id, info: Info) { # Host already known? - if ( info$host_a !in tracked ) - tracked[info$host_a] = table(); + if ( info$host !in tracked ) + tracked[info$host] = table(); - local ts = tracked[info$host_a]; + local ts = tracked[info$host]; # Software already registered for this host? We don't want to endlessly # log the same thing. if ( info$name in ts ) @@ -411,7 +411,7 @@ event software_register(id: conn_id, info: Info) function found(id: conn_id, info: Info): bool { - if ( info$force_log || addr_matches_host(info$host_a, asset_tracking) ) + if ( info$force_log || addr_matches_host(info$host, asset_tracking) ) { event software_register(id, info); return T; diff --git a/scripts/policy/frameworks/software/version-changes.bro b/scripts/policy/frameworks/software/version-changes.bro index 8365f28ae4..6d46151f0f 100644 --- a/scripts/policy/frameworks/software/version-changes.bro +++ b/scripts/policy/frameworks/software/version-changes.bro @@ -27,7 +27,7 @@ export { event log_software(rec: Info) { - local ts = tracked[rec$host_a]; + local ts = tracked[rec$host]; if ( rec$name in ts ) { @@ -40,7 +40,7 @@ event log_software(rec: Info) network_time(), rec$software_type, software_fmt_version(old$version), software_fmt(rec), rec$software_type); - NOTICE([$note=Software_Version_Change, $src=rec$host_a, + NOTICE([$note=Software_Version_Change, $src=rec$host, $msg=msg, $sub=software_fmt(rec)]); } } diff --git a/scripts/policy/frameworks/software/vulnerable.bro b/scripts/policy/frameworks/software/vulnerable.bro index cdf7db89fc..0ce949b83d 100644 --- a/scripts/policy/frameworks/software/vulnerable.bro +++ b/scripts/policy/frameworks/software/vulnerable.bro @@ -18,6 +18,6 @@ event log_software(rec: Info) if ( rec$name in vulnerable_versions && cmp_versions(rec$version, vulnerable_versions[rec$name]) <= 0 ) { - NOTICE([$note=Vulnerable_Version, $src=rec$host_a, $msg=software_fmt(rec)]); + NOTICE([$note=Vulnerable_Version, $src=rec$host, $msg=software_fmt(rec)]); } } diff --git a/testing/btest/scripts/base/frameworks/software/version-parsing.bro b/testing/btest/scripts/base/frameworks/software/version-parsing.bro index 8833b3aab6..dda3edea4b 100644 --- a/testing/btest/scripts/base/frameworks/software/version-parsing.bro +++ b/testing/btest/scripts/base/frameworks/software/version-parsing.bro @@ -2,112 +2,112 @@ # @TEST-EXEC: btest-diff output global ts = network_time(); -global host_a = 0.0.0.0; +global host = 0.0.0.0; global matched_software: table[string] of Software::Info = { ["OpenSSH_4.4"] = - [$name="OpenSSH", $version=[$major=4,$minor=4], $host_a=host_a, $ts=ts], + [$name="OpenSSH", $version=[$major=4,$minor=4], $host=host, $ts=ts], ["OpenSSH_5.2"] = - [$name="OpenSSH", $version=[$major=5,$minor=2], $host_a=host_a, $ts=ts], + [$name="OpenSSH", $version=[$major=5,$minor=2], $host=host, $ts=ts], ["Apache/2.0.63 (Unix) mod_auth_kerb/5.3 mod_ssl/2.0.63 OpenSSL/0.9.7a mod_fastcgi/2.4.2"] = - [$name="Apache", $version=[$major=2,$minor=0,$minor2=63,$addl="Unix"], $host_a=host_a, $ts=ts], + [$name="Apache", $version=[$major=2,$minor=0,$minor2=63,$addl="Unix"], $host=host, $ts=ts], ["Apache/1.3.19 (Unix)"] = - [$name="Apache", $version=[$major=1,$minor=3,$minor2=19,$addl="Unix"], $host_a=host_a, $ts=ts], + [$name="Apache", $version=[$major=1,$minor=3,$minor2=19,$addl="Unix"], $host=host, $ts=ts], ["ProFTPD 1.2.5rc1 Server (Debian)"] = - [$name="ProFTPD", $version=[$major=1,$minor=2,$minor2=5,$addl="rc1"], $host_a=host_a, $ts=ts], + [$name="ProFTPD", $version=[$major=1,$minor=2,$minor2=5,$addl="rc1"], $host=host, $ts=ts], ["wu-2.4.2-academ[BETA-18-VR14](1)"] = - [$name="wu", $version=[$major=2,$minor=4,$minor2=2,$addl="academ"], $host_a=host_a, $ts=ts], + [$name="wu", $version=[$major=2,$minor=4,$minor2=2,$addl="academ"], $host=host, $ts=ts], ["wu-2.6.2(1)"] = - [$name="wu", $version=[$major=2,$minor=6,$minor2=2,$addl="1"], $host_a=host_a, $ts=ts], + [$name="wu", $version=[$major=2,$minor=6,$minor2=2,$addl="1"], $host=host, $ts=ts], ["Java1.2.2-JDeveloper"] = - [$name="Java", $version=[$major=1,$minor=2,$minor2=2,$addl="JDeveloper"], $host_a=host_a, $ts=ts], + [$name="Java", $version=[$major=1,$minor=2,$minor2=2,$addl="JDeveloper"], $host=host, $ts=ts], ["Java/1.6.0_13"] = - [$name="Java", $version=[$major=1,$minor=6,$minor2=0,$addl="13"], $host_a=host_a, $ts=ts], + [$name="Java", $version=[$major=1,$minor=6,$minor2=0,$addl="13"], $host=host, $ts=ts], ["Python-urllib/3.1"] = - [$name="Python-urllib", $version=[$major=3,$minor=1], $host_a=host_a, $ts=ts], + [$name="Python-urllib", $version=[$major=3,$minor=1], $host=host, $ts=ts], ["libwww-perl/5.820"] = - [$name="libwww-perl", $version=[$major=5,$minor=820], $host_a=host_a, $ts=ts], + [$name="libwww-perl", $version=[$major=5,$minor=820], $host=host, $ts=ts], ["Wget/1.9+cvs-stable (Red Hat modified)"] = - [$name="Wget", $version=[$major=1,$minor=9,$addl="+cvs"], $host_a=host_a, $ts=ts], + [$name="Wget", $version=[$major=1,$minor=9,$addl="+cvs"], $host=host, $ts=ts], ["Wget/1.11.4 (Red Hat modified)"] = - [$name="Wget", $version=[$major=1,$minor=11,$minor2=4,$addl="Red Hat modified"], $host_a=host_a, $ts=ts], + [$name="Wget", $version=[$major=1,$minor=11,$minor2=4,$addl="Red Hat modified"], $host=host, $ts=ts], ["curl/7.15.1 (i486-pc-linux-gnu) libcurl/7.15.1 OpenSSL/0.9.8a zlib/1.2.3 libidn/0.5.18"] = - [$name="curl", $version=[$major=7,$minor=15,$minor2=1,$addl="i486-pc-linux-gnu"], $host_a=host_a, $ts=ts], + [$name="curl", $version=[$major=7,$minor=15,$minor2=1,$addl="i486-pc-linux-gnu"], $host=host, $ts=ts], ["Apache"] = - [$name="Apache", $host_a=host_a, $ts=ts], + [$name="Apache", $host=host, $ts=ts], ["Zope/(Zope 2.7.8-final, python 2.3.5, darwin) ZServer/1.1 Plone/Unknown"] = - [$name="Zope/(Zope", $version=[$major=2,$minor=7,$minor2=8,$addl="final"], $host_a=host_a, $ts=ts], + [$name="Zope/(Zope", $version=[$major=2,$minor=7,$minor2=8,$addl="final"], $host=host, $ts=ts], ["The Bat! (v2.00.9) Personal"] = - [$name="The Bat!", $version=[$major=2,$minor=0,$minor2=9,$addl="Personal"], $host_a=host_a, $ts=ts], + [$name="The Bat!", $version=[$major=2,$minor=0,$minor2=9,$addl="Personal"], $host=host, $ts=ts], ["Flash/10,2,153,1"] = - [$name="Flash", $version=[$major=10,$minor=2,$minor2=153,$addl="1"], $host_a=host_a, $ts=ts], + [$name="Flash", $version=[$major=10,$minor=2,$minor2=153,$addl="1"], $host=host, $ts=ts], ["mt2/1.2.3.967 Oct 13 2010-13:40:24 ord-pixel-x2 pid 0x35a3 13731"] = - [$name="mt2", $version=[$major=1,$minor=2,$minor2=3,$addl="967"], $host_a=host_a, $ts=ts], + [$name="mt2", $version=[$major=1,$minor=2,$minor2=3,$addl="967"], $host=host, $ts=ts], ["CacheFlyServe v26b"] = - [$name="CacheFlyServe", $version=[$major=26,$addl="b"], $host_a=host_a, $ts=ts], + [$name="CacheFlyServe", $version=[$major=26,$addl="b"], $host=host, $ts=ts], ["Apache/2.0.46 (Win32) mod_ssl/2.0.46 OpenSSL/0.9.7b mod_jk2/2.0.4"] = - [$name="Apache", $version=[$major=2,$minor=0,$minor2=46,$addl="Win32"], $host_a=host_a, $ts=ts], + [$name="Apache", $version=[$major=2,$minor=0,$minor2=46,$addl="Win32"], $host=host, $ts=ts], # I have no clue how I'd support this without a special case. #["Apache mod_fcgid/2.3.6 mod_auth_passthrough/2.1 mod_bwlimited/1.4 FrontPage/5.0.2.2635"] = - # [$name="Apache", $version=[], $host_a=host_a, $ts=ts], + # [$name="Apache", $version=[], $host=host, $ts=ts], ["Apple iPhone v4.3.1 Weather v1.0.0.8G4"] = - [$name="Apple iPhone", $version=[$major=4,$minor=3,$minor2=1,$addl="Weather"], $host_a=host_a, $ts=ts], + [$name="Apple iPhone", $version=[$major=4,$minor=3,$minor2=1,$addl="Weather"], $host=host, $ts=ts], ["Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5"] = - [$name="Safari", $version=[$major=5,$minor=0,$minor2=2,$addl="Mobile"], $host_a=host_a, $ts=ts], + [$name="Safari", $version=[$major=5,$minor=0,$minor2=2,$addl="Mobile"], $host=host, $ts=ts], ["Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.205 Safari/534.16"] = - [$name="Chrome", $version=[$major=10,$minor=0,$minor2=648,$addl="205"], $host_a=host_a, $ts=ts], + [$name="Chrome", $version=[$major=10,$minor=0,$minor2=648,$addl="205"], $host=host, $ts=ts], ["Opera/9.80 (Windows NT 6.1; U; sv) Presto/2.7.62 Version/11.01"] = - [$name="Opera", $version=[$major=11,$minor=1], $host_a=host_a, $ts=ts], + [$name="Opera", $version=[$major=11,$minor=1], $host=host, $ts=ts], ["Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2.11) Gecko/20101013 Lightning/1.0b2 Thunderbird/3.1.5"] = - [$name="Thunderbird", $version=[$major=3,$minor=1,$minor2=5], $host_a=host_a, $ts=ts], + [$name="Thunderbird", $version=[$major=3,$minor=1,$minor2=5], $host=host, $ts=ts], ["iTunes/9.0 (Macintosh; Intel Mac OS X 10.5.8) AppleWebKit/531.9"] = - [$name="iTunes", $version=[$major=9,$minor=0,$addl="Macintosh"], $host_a=host_a, $ts=ts], + [$name="iTunes", $version=[$major=9,$minor=0,$addl="Macintosh"], $host=host, $ts=ts], ["Java1.3.1_04"] = - [$name="Java", $version=[$major=1,$minor=3,$minor2=1,$addl="04"], $host_a=host_a, $ts=ts], + [$name="Java", $version=[$major=1,$minor=3,$minor2=1,$addl="04"], $host=host, $ts=ts], ["Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"] = - [$name="Safari", $version=[$major=4,$minor=0,$addl="Mobile"], $host_a=host_a, $ts=ts], + [$name="Safari", $version=[$major=4,$minor=0,$addl="Mobile"], $host=host, $ts=ts], ["Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-us) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27"] = - [$name="Safari", $version=[$major=5,$minor=0,$minor2=4], $host_a=host_a, $ts=ts], + [$name="Safari", $version=[$major=5,$minor=0,$minor2=4], $host=host, $ts=ts], ["Mozilla/5.0 (iPod; U; CPU iPhone OS 4_0 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8A293 Safari/6531.22.7"] = - [$name="Safari", $version=[$major=4,$minor=0,$minor2=5,$addl="Mobile"], $host_a=host_a, $ts=ts], + [$name="Safari", $version=[$major=4,$minor=0,$minor2=5,$addl="Mobile"], $host=host, $ts=ts], ["Opera/9.80 (J2ME/MIDP; Opera Mini/9.80 (S60; SymbOS; Opera Mobi/23.348; U; en) Presto/2.5.25 Version/10.54"] = - [$name="Opera Mini", $version=[$major=10,$minor=54], $host_a=host_a, $ts=ts], + [$name="Opera Mini", $version=[$major=10,$minor=54], $host=host, $ts=ts], ["Opera/9.80 (J2ME/MIDP; Opera Mini/5.0.18741/18.794; U; en) Presto/2.4.15"] = - [$name="Opera Mini", $version=[$major=5,$minor=0,$minor2=18741], $host_a=host_a, $ts=ts], + [$name="Opera Mini", $version=[$major=5,$minor=0,$minor2=18741], $host=host, $ts=ts], ["Opera/9.80 (Windows NT 5.1; Opera Mobi/49; U; en) Presto/2.4.18 Version/10.00"] = - [$name="Opera Mobi", $version=[$major=10,$minor=0], $host_a=host_a, $ts=ts], + [$name="Opera Mobi", $version=[$major=10,$minor=0], $host=host, $ts=ts], ["Mozilla/4.0 (compatible; MSIE 8.0; Android 2.2.2; Linux; Opera Mobi/ADR-1103311355; en) Opera 11.00"] = - [$name="Opera", $version=[$major=11,$minor=0], $host_a=host_a, $ts=ts], + [$name="Opera", $version=[$major=11,$minor=0], $host=host, $ts=ts], ["Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.2) Gecko/20040804 Netscape/7.2 (ax)"] = - [$name="Netscape", $version=[$major=7,$minor=2], $host_a=host_a, $ts=ts], + [$name="Netscape", $version=[$major=7,$minor=2], $host=host, $ts=ts], ["Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; GTB5; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; InfoPath.2)"] = - [$name="MSIE", $version=[$major=7,$minor=0], $host_a=host_a, $ts=ts], + [$name="MSIE", $version=[$major=7,$minor=0], $host=host, $ts=ts], ["Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.1; Media Center PC 3.0; .NET CLR 1.0.3705; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1)"] = - [$name="MSIE", $version=[$major=7,$minor=0,$addl="b"], $host_a=host_a, $ts=ts], + [$name="MSIE", $version=[$major=7,$minor=0,$addl="b"], $host=host, $ts=ts], ["Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Tablet PC 2.0; InfoPath.2; InfoPath.3)"] = - [$name="MSIE", $version=[$major=8,$minor=0], $host_a=host_a, $ts=ts], + [$name="MSIE", $version=[$major=8,$minor=0], $host=host, $ts=ts], ["Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)"] = - [$name="MSIE", $version=[$major=9,$minor=0], $host_a=host_a, $ts=ts], + [$name="MSIE", $version=[$major=9,$minor=0], $host=host, $ts=ts], ["Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; InfoPath.3; Creative AutoUpdate v1.40.02)"] = - [$name="MSIE", $version=[$major=9,$minor=0], $host_a=host_a, $ts=ts], + [$name="MSIE", $version=[$major=9,$minor=0], $host=host, $ts=ts], ["Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)"] = - [$name="MSIE", $version=[$major=10,$minor=0], $host_a=host_a, $ts=ts], + [$name="MSIE", $version=[$major=10,$minor=0], $host=host, $ts=ts], ["The Bat! (3.0.1 RC3) Professional"] = - [$name="The Bat!", $version=[$major=3,$minor=0,$minor2=1,$addl="RC3"], $host_a=host_a, $ts=ts], + [$name="The Bat!", $version=[$major=3,$minor=0,$minor2=1,$addl="RC3"], $host=host, $ts=ts], # This is an FTP client (found with CLNT command) ["Total Commander"] = - [$name="Total Commander", $version=[], $host_a=host_a, $ts=ts], + [$name="Total Commander", $version=[], $host=host, $ts=ts], ["(vsFTPd 2.0.5)"] = - [$name="vsFTPd", $version=[$major=2,$minor=0,$minor2=5], $host_a=host_a, $ts=ts], + [$name="vsFTPd", $version=[$major=2,$minor=0,$minor2=5], $host=host, $ts=ts], ["Apple Mail (2.1084)"] = - [$name="Apple Mail", $version=[$major=2,$minor=1084], $host_a=host_a, $ts=ts], + [$name="Apple Mail", $version=[$major=2,$minor=1084], $host=host, $ts=ts], }; event bro_init() { for ( sw in matched_software ) { - local output = Software::parse(sw, host_a, Software::UNKNOWN); + local output = Software::parse(sw, host, Software::UNKNOWN); local baseline: Software::Info; baseline = matched_software[sw]; if ( baseline$name == output$name && From 031303997723c178f387c63fe1d999392fb444d3 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 8 Dec 2011 14:44:45 -0800 Subject: [PATCH 161/964] log protocol in notices. --- scripts/base/frameworks/notice/main.bro | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index 7d98c6464c..1e150a3506 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -64,6 +64,10 @@ export { conn: connection &optional; iconn: icmp_conn &optional; + ## The transport protocol. Filled automatically when either conn, iconn + ## or p are specified. + proto: transport_proto &log &optional; + ## The :bro:enum:`Notice::Type` of the notice. note: Type &log; ## The human readable message for the notice. @@ -458,8 +462,12 @@ function apply_policy(n: Notice::Info) n$p = n$id$resp_p; } + if ( n?$p ) + n$proto = get_port_transport_proto(n$p); + if ( n?$iconn ) { + n$proto = icmp; if ( ! n?$src ) n$src = n$iconn$orig_h; if ( ! n?$dst ) From 1f57827e5431cb35b7d38304bd652762811aa3b1 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 9 Dec 2011 14:30:21 -0600 Subject: [PATCH 162/964] Add more logging framework documentation. --- doc/logging.rst | 83 ++++--- doc/scripts/builtins.rst | 4 + scripts/base/frameworks/logging/main.bro | 208 ++++++++++++++++-- .../frameworks/logging/postprocessors/scp.bro | 28 ++- .../base/frameworks/logging/writers/ascii.bro | 6 +- src/logging.bif | 2 +- 6 files changed, 272 insertions(+), 59 deletions(-) diff --git a/doc/logging.rst b/doc/logging.rst index 2817cadd45..e7734915da 100644 --- a/doc/logging.rst +++ b/doc/logging.rst @@ -43,13 +43,14 @@ Basics ====== The data fields that a stream records are defined by a record type -specified when it is created. Let's look at the script generating -Bro's connection summaries as an example, -``base/protocols/conn/main.bro``. It defines a record ``Conn::Info`` -that lists all the fields that go into ``conn.log``, each marked with -a ``&log`` attribute indicating that it is part of the information -written out. To write a log record, the script then passes an instance -of ``Conn::Info`` to the logging framework's ``Log::write`` function. +specified when it is created. Let's look at the script generating Bro's +connection summaries as an example, +:doc:`scripts/base/protocols/conn/main`. It defines a record +:bro:type:`Conn::Info` that lists all the fields that go into +``conn.log``, each marked with a ``&log`` attribute indicating that it +is part of the information written out. To write a log record, the +script then passes an instance of :bro:type:`Conn::Info` to the logging +framework's :bro:id:`Log::write` function. By default, each stream automatically gets a filter named ``default`` that generates the normal output by recording all record fields into a @@ -66,7 +67,7 @@ To create new a new output file for an existing stream, you can add a new filter. A filter can, e.g., restrict the set of fields being logged: -.. code:: bro: +.. code:: bro event bro_init() { @@ -85,14 +86,15 @@ Note the fields that are set for the filter: ``path`` The filename for the output file, without any extension (which may be automatically added by the writer). Default path values - are generated by taking the stream's ID and munging it - slightly. ``Conn::LOG`` is converted into ``conn``, - ``PacketFilter::LOG`` is converted into ``packet_filter``, and - ``Notice::POLICY_LOG`` is converted into ``notice_policy``. + are generated by taking the stream's ID and munging it slightly. + :bro:enum:`Conn::LOG` is converted into ``conn``, + :bro:enum:`PacketFilter::LOG` is converted into + ``packet_filter``, and :bro:enum:`Notice::POLICY_LOG` is + converted into ``notice_policy``. ``include`` A set limiting the fields to the ones given. The names - correspond to those in the ``Conn::LOG`` record, with + correspond to those in the :bro:type:`Conn::Info` record, with sub-records unrolled by concatenating fields (separated with dots). @@ -158,10 +160,10 @@ further for example to log information by subnets or even by IP address. Be careful, however, as it is easy to create many files very quickly ... -.. sidebar: +.. sidebar:: A More Generic Path Function - The show ``split_log`` method has one draw-back: it can be used - only with the ``Conn::Log`` stream as the record type is hardcoded + The ``split_log`` method has one draw-back: it can be used + only with the :bro:enum:`Conn::Log` stream as the record type is hardcoded into its argument list. However, Bro allows to do a more generic variant: @@ -201,8 +203,8 @@ Extending You can add further fields to a log stream by extending the record type that defines its content. Let's say we want to add a boolean -field ``is_private`` to ``Conn::Info`` that indicates whether the -originator IP address is part of the RFC1918 space: +field ``is_private`` to :bro:type:`Conn::Info` that indicates whether the +originator IP address is part of the :rfc:`1918` space: .. code:: bro @@ -234,10 +236,10 @@ Notes: - For extending logs this way, one needs a bit of knowledge about how the script that creates the log stream is organizing its state keeping. Most of the standard Bro scripts attach their log state to - the ``connection`` record where it can then be accessed, just as the - ``c$conn`` above. For example, the HTTP analysis adds a field ``http - : HTTP::Info`` to the ``connection`` record. See the script - reference for more information. + the :bro:type:`connection` record where it can then be accessed, just + as the ``c$conn`` above. For example, the HTTP analysis adds a field + ``http`` of type :bro:type:`HTTP::Info` to the :bro:type:`connection` + record. See the script reference for more information. - When extending records as shown above, the new fields must always be declared either with a ``&default`` value or as ``&optional``. @@ -251,8 +253,8 @@ Sometimes it is helpful to do additional analysis of the information being logged. For these cases, a stream can specify an event that will be generated every time a log record is written to it. All of Bro's default log streams define such an event. For example, the connection -log stream raises the event ``Conn::log_conn(rec: Conn::Info)``: You -could use that for example for flagging when an a connection to +log stream raises the event :bro:id:`Conn::log_conn`. You +could use that for example for flagging when a connection to specific destination exceeds a certain duration: .. code:: bro @@ -279,11 +281,32 @@ real-time. Rotation -------- +By default, no log rotation occurs, but it's globally controllable for all +filters by redefining the :bro:id:`Log::default_rotation_interval` option: + +.. code:: bro + + redef Log::default_rotation_interval = 1 hr; + +Or specifically for certain :bro:type:`Log::Filter` instances by setting +their ``interv`` field. Here's an example of changing just the +:bro:enum:`Conn::LOG` stream's default filter rotation. + +.. code:: bro + + event bro_init() + { + local f = Log::get_filter(Conn::LOG, "default"); + f$interv = 1 min; + Log::remove_filter(Conn::LOG, "default"); + Log::add_filter(Conn::LOG, f); + } + ASCII Writer Configuration -------------------------- The ASCII writer has a number of options for customizing the format of -its output, see XXX.bro. +its output, see :doc:`scripts/base/frameworks/logging/writers/ascii`. Adding Streams ============== @@ -321,8 +344,8 @@ example for the ``Foo`` module: Log::create_stream(Foo::LOG, [$columns=Info, $ev=log_foo]); } -You can also the state to the ``connection`` record to make it easily -accessible across event handlers: +You can also add the state to the :bro:type:`connection` record to make +it easily accessible across event handlers: .. code:: bro @@ -330,7 +353,7 @@ accessible across event handlers: foo: Info &optional; } -Now you can use the ``Log::write`` method to output log records and +Now you can use the :bro:id:`Log::write` method to output log records and save the logged ``Foo::Info`` record into the connection record: .. code:: bro @@ -343,9 +366,9 @@ save the logged ``Foo::Info`` record into the connection record: } See the existing scripts for how to work with such a new connection -field. A simple example is ``base/protocols/syslog/main.bro``. +field. A simple example is :doc:`scripts/base/protocols/syslog/main`. -When you are developing scripts that add data to the ``connection`` +When you are developing scripts that add data to the :bro:type:`connection` record, care must be given to when and how long data is stored. Normally data saved to the connection record will remain there for the duration of the connection and from a practical perspective it's not diff --git a/doc/scripts/builtins.rst b/doc/scripts/builtins.rst index 942e053b5f..4a3316c04f 100644 --- a/doc/scripts/builtins.rst +++ b/doc/scripts/builtins.rst @@ -410,6 +410,10 @@ The Bro scripting language supports the following built-in types. Writing to files like this for logging usually isn't recommend, for better logging support see :doc:`/logging`. +.. bro:type:: func + + See :bro:type:`function`. + .. bro:type:: function Function types in Bro are declared using:: diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro index 440773233d..2c36b3001e 100644 --- a/scripts/base/frameworks/logging/main.bro +++ b/scripts/base/frameworks/logging/main.bro @@ -1,16 +1,16 @@ ##! The Bro logging interface. ##! -##! See XXX for a introduction to Bro's logging framework. +##! See :doc:`/logging` for a introduction to Bro's logging framework. module Log; -# Log::ID and Log::Writer are defined in bro.init due to circular dependencies. +# Log::ID and Log::Writer are defined in types.bif due to circular dependencies. export { - ## If true, is local logging is by default enabled for all filters. + ## If true, local logging is by default enabled for all filters. const enable_local_logging = T &redef; - ## If true, is remote logging is by default enabled for all filters. + ## If true, remote logging is by default enabled for all filters. const enable_remote_logging = T &redef; ## Default writer to use if a filter does not specify @@ -23,21 +23,24 @@ export { columns: any; ## Event that will be raised once for each log entry. - ## The event receives a single same parameter, an instance of type ``columns``. + ## The event receives a single same parameter, an instance of type + ## ``columns``. ev: any &optional; }; - ## Default function for building the path values for log filters if not - ## speficied otherwise by a filter. The default implementation uses ``id`` + ## Builds the default path values for log filters if not otherwise + ## specified by a filter. The default implementation uses *id* ## to derive a name. ## - ## id: The log stream. + ## id: The ID associated with the log stream. + ## ## path: A suggested path value, which may be either the filter's ## ``path`` if defined, else a previous result from the function. ## If no ``path`` is defined for the filter, then the first call ## to the function will contain an empty string. + ## ## rec: An instance of the streams's ``columns`` type with its - ## fields set to the values to logged. + ## fields set to the values to be logged. ## ## Returns: The path to be used for the filter. global default_path_func: function(id: ID, path: string, rec: any) : string &redef; @@ -46,7 +49,7 @@ export { ## Information passed into rotation callback functions. type RotationInfo: record { - writer: Writer; ##< Writer. + writer: Writer; ##< The :bro:type:`Log::Writer` being used. fname: string; ##< Full name of the rotated file. path: string; ##< Original path value. open: time; ##< Time when opened. @@ -57,25 +60,26 @@ export { ## Default rotation interval. Zero disables rotation. const default_rotation_interval = 0secs &redef; - ## Default naming format for timestamps embedded into filenames. Uses a strftime() style. + ## Default naming format for timestamps embedded into filenames. + ## Uses a ``strftime()`` style. const default_rotation_date_format = "%Y-%m-%d-%H-%M-%S" &redef; ## Default shell command to run on rotated files. Empty for none. const default_rotation_postprocessor_cmd = "" &redef; - ## Specifies the default postprocessor function per writer type. Entries in this - ## table are initialized by each writer type. + ## Specifies the default postprocessor function per writer type. + ## Entries in this table are initialized by each writer type. const default_rotation_postprocessors: table[Writer] of function(info: RotationInfo) : bool &redef; - ## Filter customizing logging. + ## A filter type describes how to customize logging streams. type Filter: record { ## Descriptive name to reference this filter. name: string; - ## The writer to use. + ## The logging writer implementation to use. writer: Writer &default=default_writer; - ## Predicate indicating whether a log entry should be recorded. + ## Indicates whether a log entry should be recorded. ## If not given, all entries are recorded. ## ## rec: An instance of the streams's ``columns`` type with its @@ -101,13 +105,15 @@ export { ## easy to flood the disk by returning a new string for each ## connection ... ## - ## id: The log stream. + ## id: The ID associated with the log stream. + ## ## path: A suggested path value, which may be either the filter's ## ``path`` if defined, else a previous result from the function. ## If no ``path`` is defined for the filter, then the first call ## to the function will contain an empty string. + ## ## rec: An instance of the streams's ``columns`` type with its - ## fields set to the values to logged. + ## fields set to the values to be logged. ## ## Returns: The path to be used for the filter. path_func: function(id: ID, path: string, rec: any): string &optional; @@ -129,27 +135,183 @@ export { ## Rotation interval. interv: interval &default=default_rotation_interval; - ## Callback function to trigger for rotated files. If not set, - ## the default comes out of default_rotation_postprocessors. + ## Callback function to trigger for rotated files. If not set, the + ## default comes out of :bro:id:`Log::default_rotation_postprocessors`. postprocessor: function(info: RotationInfo) : bool &optional; }; ## Sentinel value for indicating that a filter was not found when looked up. - const no_filter: Filter = [$name=""]; # Sentinel. + const no_filter: Filter = [$name=""]; - # TODO: Document. + ## Creates a new logging stream with the default filter. + ## + ## id: The ID enum to be associated with the new logging stream. + ## + ## stream: A record defining the content that the new stream will log. + ## + ## Returns: True if a new logging stream was successfully created and + ## a default filter added to it. + ## + ## .. bro:see:: Log::add_default_filter Log::remove_default_filter global create_stream: function(id: ID, stream: Stream) : bool; + + ## Enables a previously disabled logging stream. Disabled streams + ## will not be written to until they are enabled again. New streams + ## are enabled by default. + ## + ## id: The ID associated with the logging stream to enable. + ## + ## Returns: True if the stream is re-enabled or was not previously disabled. + ## + ## .. bro:see:: Log::disable_stream global enable_stream: function(id: ID) : bool; + + ## Disables a currently enabled logging stream. Disabled streams + ## will not be written to until they are enabled again. New streams + ## are enabled by default. + ## + ## id: The ID associated with the logging stream to disable. + ## + ## Returns: True if the stream is now disabled or was already disabled. + ## + ## .. bro:see:: Log::enable_stream global disable_stream: function(id: ID) : bool; + + ## Adds a custom filter to an existing logging stream. If a filter + ## with a matching ``name`` field already exists for the stream, it + ## is removed when the new filter is successfully added. + ## + ## id: The ID associated with the logging stream to filter. + ## + ## filter: A record describing the desired logging parameters. + ## + ## Returns: True if the filter was sucessfully added, false if + ## the filter was not added or the *filter* argument was not + ## the correct type. + ## + ## .. bro:see:: Log::remove_filter Log::add_default_filter + ## Log::remove_default_filter global add_filter: function(id: ID, filter: Filter) : bool; + + ## Removes a filter from an existing logging stream. + ## + ## id: The ID associated with the logging stream from which to + ## remove a filter. + ## + ## name: A string to match against the ``name`` field of a + ## :bro:type:`Log::Filter` for identification purposes. + ## + ## Returns: True if the logging stream's filter was removed or + ## if no filter associated with *name* was found. + ## + ## .. bro:see:: Log::remove_filter Log::add_default_filter + ## Log::remove_default_filter global remove_filter: function(id: ID, name: string) : bool; - global get_filter: function(id: ID, name: string) : Filter; # Returns no_filter if not found. + + ## Gets a filter associated with an existing logging stream. + ## + ## id: The ID associated with a logging stream from which to + ## obtain one of its filters. + ## + ## name: A string to match against the ``name`` field of a + ## :bro:type:`Log::Filter` for identification purposes. + ## + ## Returns: A filter attached to the logging stream *id* matching + ## *name* or, if no matches are found returns the + ## :bro:id:`Log::no_filter` sentinel value. + ## + ## .. bro:see:: Log::add_filter Log::remove_filter Log::add_default_filter + ## Log::remove_default_filter + global get_filter: function(id: ID, name: string) : Filter; + + ## Writes a new log line/entry to a logging stream. + ## + ## id: The ID associated with a logging stream to be written to. + ## + ## columns: A record value describing the values of each field/column + ## to write to the log stream. + ## + ## Returns: True if the stream was found and no error occurred in writing + ## to it or if the stream was disabled and nothing was written. + ## False if the stream was was not found, or the *columns* + ## argument did not match what the stream was initially defined + ## to handle, or one of the stream's filters has an invalid + ## ``path_func``. + ## + ## .. bro:see: Log::enable_stream Log::disable_stream global write: function(id: ID, columns: any) : bool; + + ## Sets the buffering status for all the writers of a given logging stream. + ## A given writer implementation may or may not support buffering and if it + ## doesn't then toggling buffering with this function has no effect. + ## + ## id: The ID associated with a logging stream for which to + ## enable/disable buffering. + ## + ## buffered: Whether to enable or disable log buffering. + ## + ## Returns: True if buffering status was set, false if the logging stream + ## does not exist. + ## + ## .. bro:see:: Log::flush global set_buf: function(id: ID, buffered: bool): bool; + + ## Flushes any currently buffered output for all the writers of a given + ## logging stream. + ## + ## id: The ID associated with a logging stream for which to flush buffered + ## data. + ## + ## Returns: True if all writers of a log stream were signalled to flush + ## buffered data or if the logging stream is disabled, + ## false if the logging stream does not exist. + ## + ## .. bro:see:: Log::set_buf Log::enable_stream Log::disable_stream global flush: function(id: ID): bool; + + ## Adds a default :bro:type:`Log::Filter` record with ``name`` field + ## set as "default" to a given logging stream. + ## + ## id: The ID associated with a logging stream for which to add a default + ## filter. + ## + ## Returns: The status of a call to :bro:id:`Log::add_filter` using a + ## default :bro:type:`Log::Filter` argument with ``name`` field + ## set to "default". + ## + ## .. bro:see:: Log::add_filter Log::remove_filter + ## Log::remove_default_filter global add_default_filter: function(id: ID) : bool; + + ## Removes the :bro:type:`Log::Filter` with ``name`` field equal to + ## "default". + ## + ## id: The ID associated with a logging stream from which to remove the + ## default filter. + ## + ## Returns: The status of a call to :bro:id:`Log::remove_filter` using + ## "default" as the argument. + ## + ## .. bro:see:: Log::add_filter Log::remove_filter Log::add_default_filter global remove_default_filter: function(id: ID) : bool; + ## Runs a command given by :bro:id:`Log::default_rotation_postprocessor_cmd` + ## on a rotated file. Meant to be called from postprocessor functions + ## that are added to :bro:id:`Log::default_rotation_postprocessors`. + ## + ## info: A record holding meta-information about the log being rotated. + ## + ## npath: The new path of the file (after already being rotated/processed + ## by writer-specific postprocessor as defined in + ## :bro:id:`Log::default_rotation_postprocessors`. + ## + ## Returns: True when :bro:id:`Log::default_rotation_postprocessor_cmd` + ## is empty or the system command given by it has been invoked + ## to postprocess a rotated log file. + ## + ## .. bro:see:: Log::default_rotation_date_format + ## Log::default_rotation_postprocessor_cmd + ## Log::default_rotation_postprocessors global run_rotation_postprocessor_cmd: function(info: RotationInfo, npath: string) : bool; } diff --git a/scripts/base/frameworks/logging/postprocessors/scp.bro b/scripts/base/frameworks/logging/postprocessors/scp.bro index f27e748ae5..8f35aa99f2 100644 --- a/scripts/base/frameworks/logging/postprocessors/scp.bro +++ b/scripts/base/frameworks/logging/postprocessors/scp.bro @@ -1,29 +1,51 @@ ##! This script defines a postprocessing function that can be applied ##! to a logging filter in order to automatically SCP (secure copy) ##! a log stream (or a subset of it) to a remote host at configurable -##! rotation time intervals. +##! rotation time intervals. Generally, to use this functionality +##! you must handle the :bro:id:`bro_init` event and do the following +##! in your handler: +##! +##! 1) Create a new :bro:type:`Log::Filter` record that defines a name/path, +##! rotation interval, and set the ``postprocessor`` to +##! :bro:id:`Log::scp_postprocessor`. +##! 2) Add the filter to a logging stream using :bro:id:`Log::add_filter`. +##! 3) Add a table entry to :bro:id:`Log::scp_destinations` for the filter's +##! writer/path pair which defines a set of :bro:type:`Log::SCPDestination` +##! records. module Log; export { - ## This postprocessor SCP's the rotated-log to all the remote hosts + ## Secure-copies the rotated-log to all the remote hosts ## defined in :bro:id:`Log::scp_destinations` and then deletes ## the local copy of the rotated-log. It's not active when ## reading from trace files. + ## + ## info: A record holding meta-information about the log file to be + ## postprocessed. + ## + ## Returns: True if secure-copy system command was initiated or + ## if no destination was configured for the log as described + ## by *info*. global scp_postprocessor: function(info: Log::RotationInfo): bool; ## A container that describes the remote destination for the SCP command ## argument as ``user@host:path``. type SCPDestination: record { + ## The remote user to log in as. A trust mechanism should be + ## pre-established. user: string; + ## The remote host to which to transfer logs. host: string; + ## The path/directory on the remote host to send logs. path: string; }; ## A table indexed by a particular log writer and filter path, that yields ## a set remote destinations. The :bro:id:`Log::scp_postprocessor` ## function queries this table upon log rotation and performs a secure - ## copy of the rotated-log to each destination in the set. + ## copy of the rotated-log to each destination in the set. This + ## table can be modified at run-time. global scp_destinations: table[Writer, string] of set[SCPDestination]; } diff --git a/scripts/base/frameworks/logging/writers/ascii.bro b/scripts/base/frameworks/logging/writers/ascii.bro index 5c04fdd3d9..3f00787f57 100644 --- a/scripts/base/frameworks/logging/writers/ascii.bro +++ b/scripts/base/frameworks/logging/writers/ascii.bro @@ -1,4 +1,5 @@ -##! Interface for the ascii log writer. +##! Interface for the ASCII log writer. Redefinable options are available +##! to tweak the output format of ASCII logs. module LogAscii; @@ -7,7 +8,8 @@ export { ## into files. This is primarily for debugging purposes. const output_to_stdout = F &redef; - ## If true, include a header line with column names. + ## If true, include a header line with column names and description + ## of the other ASCII logging options that were used. const include_header = T &redef; ## Prefix for the header line if included. diff --git a/src/logging.bif b/src/logging.bif index 501eb899d9..31e1bebacd 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -1,4 +1,4 @@ -# Internal functions and types used by the logging framework. +##! Internal functions and types used by the logging framework. module Log; From 2cf7bb578886ec64d11df509cc52adc725965b08 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 9 Dec 2011 15:39:31 -0600 Subject: [PATCH 163/964] Teach Broxygen to more generally reference attribute values by name. --- src/Attr.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Attr.cc b/src/Attr.cc index 12e1c888af..aed9165182 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -60,16 +60,19 @@ void Attr::DescribeReST(ODesc* d) const d->Add("="); d->SP(); - if ( expr->Type()->Tag() == TYPE_FUNC ) - d->Add(":bro:type:`func`"); - else if ( expr->Type()->Tag() == TYPE_ENUM ) + if ( expr->Tag() == EXPR_NAME ) { - d->Add(":bro:enum:`"); + d->Add(":bro:see:`"); expr->Describe(d); d->Add("`"); } + else if ( expr->Type()->Tag() == TYPE_FUNC ) + { + d->Add(":bro:type:`func`"); + } + else { d->Add("``"); From ec721dffeca84727d969b634d80a5e466323ded5 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Fri, 9 Dec 2011 16:56:12 -0500 Subject: [PATCH 164/964] Added is_orig fields to the SSL events and adapted script. - Added a field named $last_alert to the SSL log. This doesn't even indicate the direction the alert was sent, but we need to start somewhere. - The x509_certificate function has an is_orig field now instead of is_server and it's position in the argument list has moved. - A bit of reorganization and cleanup in the core analyzer. --- scripts/base/protocols/ssl/consts.bro | 41 ++++++++++++++++++- scripts/base/protocols/ssl/main.bro | 17 ++++++-- scripts/policy/protocols/ssl/cert-hash.bro | 4 +- .../policy/protocols/ssl/expiring-certs.bro | 5 ++- scripts/policy/protocols/ssl/known-certs.bro | 4 +- src/event.bif | 10 ++--- src/ssl-analyzer.pac | 20 +++++---- src/ssl-protocol.pac | 27 +++++------- 8 files changed, 89 insertions(+), 39 deletions(-) diff --git a/scripts/base/protocols/ssl/consts.bro b/scripts/base/protocols/ssl/consts.bro index 2026f9bfa2..9d8bc68fd5 100644 --- a/scripts/base/protocols/ssl/consts.bro +++ b/scripts/base/protocols/ssl/consts.bro @@ -13,6 +13,44 @@ export { [TLSv11] = "TLSv11", } &default="UNKNOWN"; + const alert_levels: table[count] of string = { + [1] = "warning", + [2] = "fatal", + } &default=function(i: count):string { return fmt("unknown-%d", i); }; + + const alert_descriptions: table[count] of string = { + [0] = "close_notify", + [10] = "unexpected_message", + [20] = "bad_record_mac", + [21] = "decryption_failed", + [22] = "record_overflow", + [30] = "decompression_failure", + [40] = "handshake_failure", + [41] = "no_certificate", + [42] = "bad_certificate", + [43] = "unsupported_certificate", + [44] = "certificate_revoked", + [45] = "certificate_expired", + [46] = "certificate_unknown", + [47] = "illegal_parameter", + [48] = "unknown_ca", + [49] = "access_denied", + [50] = "decode_error", + [51] = "decrypt_error", + [60] = "export_restriction", + [70] = "protocol_version", + [71] = "insufficient_security", + [80] = "internal_error", + [90] = "user_canceled", + [100] = "no_renegotiation", + [110] = "unsupported_extension", + [111] = "certificate_unobtainable", + [112] = "unrecognized_name", + [113] = "bad_certificate_status_response", + [114] = "bad_certificate_hash_value", + [115] = "unknown_psk_identity", + } &default=function(i: count):string { return fmt("unknown-%d", i); }; + # http://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xml const extensions: table[count] of string = { [0] = "server_name", @@ -526,8 +564,7 @@ export { [30] = "akid issuer serial mismatch", [31] = "keyusage no certsign", [32] = "unable to get crl issuer", - [33] = "unhandled critical extension" - + [33] = "unhandled critical extension", }; } diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index c3c04d3c93..990b4d56f7 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -16,6 +16,7 @@ export { subject: string &log &optional; not_valid_before: time &log &optional; not_valid_after: time &log &optional; + last_alert: string &log &optional; cert: string &optional; cert_chain: vector of string &optional; @@ -112,10 +113,13 @@ event ssl_server_hello(c: connection, version: count, possible_ts: time, session c$ssl$cipher = cipher_desc[cipher]; } -event x509_certificate(c: connection, cert: X509, is_server: bool, chain_idx: count, chain_len: count, der_cert: string) &priority=5 +event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=5 { set_session(c); + # We aren't doing anything with client certificates yet. + if ( is_orig ) return; + if ( chain_idx == 0 ) { # Save the primary cert. @@ -133,14 +137,21 @@ event x509_certificate(c: connection, cert: X509, is_server: bool, chain_idx: co } } -event ssl_extension(c: connection, code: count, val: string) &priority=5 +event ssl_extension(c: connection, is_orig: bool, code: count, val: string) &priority=5 { set_session(c); - if ( extensions[code] == "server_name" ) + if ( is_orig && extensions[code] == "server_name" ) c$ssl$server_name = sub_bytes(val, 6, |val|); } +event ssl_alert(c: connection, is_orig: bool, level: count, desc: count) &priority=5 + { + set_session(c); + + c$ssl$last_alert = alert_descriptions[desc]; + } + event ssl_established(c: connection) &priority=5 { set_session(c); diff --git a/scripts/policy/protocols/ssl/cert-hash.bro b/scripts/policy/protocols/ssl/cert-hash.bro index 80a937f670..1e47ccac2e 100644 --- a/scripts/policy/protocols/ssl/cert-hash.bro +++ b/scripts/policy/protocols/ssl/cert-hash.bro @@ -10,11 +10,11 @@ export { }; } -event x509_certificate(c: connection, cert: X509, is_server: bool, chain_idx: count, chain_len: count, der_cert: string) &priority=4 +event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=4 { # We aren't tracking client certificates yet and we are also only tracking # the primary cert. Watch that this came from an SSL analyzed session too. - if ( ! is_server || chain_idx != 0 || ! c?$ssl ) + if ( is_orig || chain_idx != 0 || ! c?$ssl ) return; c$ssl$cert_hash = md5_hash(der_cert); diff --git a/scripts/policy/protocols/ssl/expiring-certs.bro b/scripts/policy/protocols/ssl/expiring-certs.bro index 50480b3a09..0e4db56bc3 100644 --- a/scripts/policy/protocols/ssl/expiring-certs.bro +++ b/scripts/policy/protocols/ssl/expiring-certs.bro @@ -33,10 +33,11 @@ export { const notify_when_cert_expiring_in = 30days &redef; } -event x509_certificate(c: connection, cert: X509, is_server: bool, chain_idx: count, chain_len: count, der_cert: string) &priority=3 +event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=3 { # If this isn't the host cert or we aren't interested in the server, just return. - if ( chain_idx != 0 || + if ( is_orig || + chain_idx != 0 || ! c$ssl?$cert_hash || ! addr_matches_host(c$id$resp_h, notify_certs_expiration) ) return; diff --git a/scripts/policy/protocols/ssl/known-certs.bro b/scripts/policy/protocols/ssl/known-certs.bro index 90f6ee6186..669432e4d9 100644 --- a/scripts/policy/protocols/ssl/known-certs.bro +++ b/scripts/policy/protocols/ssl/known-certs.bro @@ -44,10 +44,10 @@ event bro_init() &priority=5 Log::create_stream(Known::CERTS_LOG, [$columns=CertsInfo, $ev=log_known_certs]); } -event x509_certificate(c: connection, cert: X509, is_server: bool, chain_idx: count, chain_len: count, der_cert: string) &priority=3 +event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=3 { # Make sure this is the server cert and we have a hash for it. - if ( chain_idx != 0 || ! c$ssl?$cert_hash ) + if ( is_orig || chain_idx != 0 || ! c$ssl?$cert_hash ) return; local host = c$id$resp_h; diff --git a/src/event.bif b/src/event.bif index d953ac78fe..e5408f36a6 100644 --- a/src/event.bif +++ b/src/event.bif @@ -279,13 +279,13 @@ event ssh_server_version%(c: connection, version: string%); event ssl_client_hello%(c: connection, version: count, possible_ts: time, session_id: string, ciphers: count_set%); event ssl_server_hello%(c: connection, version: count, possible_ts: time, session_id: string, cipher: count, comp_method: count%); -event ssl_extension%(c: connection, code: count, val: string%); +event ssl_extension%(c: connection, is_orig: bool, code: count, val: string%); +event ssl_alert%(c: connection, is_orig: bool, level: count, desc: count%); event ssl_established%(c: connection%); -event ssl_alert%(c: connection, level: count, desc: count%); -event x509_certificate%(c: connection, cert: X509, is_server: bool, chain_idx: count, chain_len: count, der_cert: string%); -event x509_extension%(c: connection, data: string%); -event x509_error%(c: connection, err: count%); +event x509_certificate%(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string%); +event x509_extension%(c: connection, is_orig: bool, data: string%); +event x509_error%(c: connection, is_orig: bool, err: count%); event stp_create_endp%(c: connection, e: int, is_orig: bool%); event stp_resume_endp%(e: int%); diff --git a/src/ssl-analyzer.pac b/src/ssl-analyzer.pac index 6471d9c4a4..6278e79459 100644 --- a/src/ssl-analyzer.pac +++ b/src/ssl-analyzer.pac @@ -22,11 +22,17 @@ } }; + string orig_label(bool is_orig); void free_X509(void *); X509* d2i_X509_binpac(X509** px, const uint8** in, int len); %} %code{ +string orig_label(bool is_orig) + { + return string(is_orig ? "originator" :"responder"); + } + void free_X509(void* cert) { X509_free((X509*) cert); @@ -117,7 +123,7 @@ refine connection SSL_Conn += { function proc_alert(rec: SSLRecord, level : int, desc : int) : bool %{ BifEvent::generate_ssl_alert(bro_analyzer(), bro_analyzer()->Conn(), - level, desc); + ${rec.is_orig}, level, desc); return true; %} @@ -200,11 +206,11 @@ refine connection SSL_Conn += { return true; %} - function proc_ssl_extension(type: int, data: bytestring) : bool + function proc_ssl_extension(rec: SSLRecord, type: int, data: bytestring) : bool %{ if ( ssl_extension ) BifEvent::generate_ssl_extension(bro_analyzer(), - bro_analyzer()->Conn(), type, + bro_analyzer()->Conn(), ${rec.is_orig}, type, new StringVal(data.length(), (const char*) data.data())); return true; %} @@ -231,7 +237,7 @@ refine connection SSL_Conn += { if ( ! pTemp ) { BifEvent::generate_x509_error(bro_analyzer(), bro_analyzer()->Conn(), - ERR_get_error()); + ${rec.is_orig}, ERR_get_error()); return false; } @@ -257,8 +263,8 @@ refine connection SSL_Conn += { StringVal* der_cert = new StringVal(cert.length(), (const char*) cert.data()); BifEvent::generate_x509_certificate(bro_analyzer(), bro_analyzer()->Conn(), + ${rec.is_orig}, pX509Cert, - ! ${rec.is_orig}, i, certificates->size(), der_cert); @@ -284,7 +290,7 @@ refine connection SSL_Conn += { StringVal* value = new StringVal(length, (char*)pBuffer); BifEvent::generate_x509_extension(bro_analyzer(), - bro_analyzer()->Conn(), value); + bro_analyzer()->Conn(), ${rec.is_orig}, value); OPENSSL_free(pBuffer); } } @@ -445,5 +451,5 @@ refine typeattr CiphertextRecord += &let { } refine typeattr SSLExtension += &let { - proc : bool = $context.connection.proc_ssl_extension(type, data); + proc : bool = $context.connection.proc_ssl_extension(rec, type, data); }; diff --git a/src/ssl-protocol.pac b/src/ssl-protocol.pac index f60d73b27e..24207ac78b 100644 --- a/src/ssl-protocol.pac +++ b/src/ssl-protocol.pac @@ -22,7 +22,6 @@ type uint24 = record { }; string state_label(int state_nr); - string orig_label(bool is_orig); double get_time_from_asn1(const ASN1_TIME * atime); string handshake_type_label(int type); %} @@ -35,7 +34,7 @@ type SSLRecord(is_orig: bool) = record { head2 : uint8; head3 : uint8; head4 : uint8; - rec : RecordText(this, is_orig)[] &length=length, &requires(content_type); + rec : RecordText(this)[] &length=length, &requires(content_type); } &length = length+5, &byteorder=bigendian, &let { version : int = @@ -54,25 +53,25 @@ type SSLRecord(is_orig: bool) = record { }; }; -type RecordText(rec: SSLRecord, is_orig: bool) = case $context.connection.state() of { +type RecordText(rec: SSLRecord) = case $context.connection.state() of { STATE_ABBREV_SERVER_ENCRYPTED, STATE_CLIENT_ENCRYPTED, STATE_COMM_ENCRYPTED, STATE_CONN_ESTABLISHED - -> ciphertext : CiphertextRecord(rec, is_orig); + -> ciphertext : CiphertextRecord(rec); default - -> plaintext : PlaintextRecord(rec, is_orig); + -> plaintext : PlaintextRecord(rec); }; -type PossibleEncryptedHandshake(rec: SSLRecord, is_orig: bool) = case $context.connection.state() of { +type PossibleEncryptedHandshake(rec: SSLRecord) = case $context.connection.state() of { # Deal with encrypted handshakes before the server cipher spec change. STATE_CLIENT_FINISHED, STATE_CLIENT_ENCRYPTED - -> ct : CiphertextRecord(rec, is_orig); + -> ct : CiphertextRecord(rec); default -> hs : Handshake(rec); }; -type PlaintextRecord(rec: SSLRecord, is_orig: bool) = case rec.content_type of { +type PlaintextRecord(rec: SSLRecord) = case rec.content_type of { CHANGE_CIPHER_SPEC -> ch_cipher : ChangeCipherSpec(rec); ALERT -> alert : Alert(rec); - HANDSHAKE -> handshake : PossibleEncryptedHandshake(rec, is_orig); + HANDSHAKE -> handshake : PossibleEncryptedHandshake(rec); APPLICATION_DATA -> app_data : ApplicationData(rec); V2_ERROR -> v2_error : V2Error(rec); V2_CLIENT_HELLO -> v2_client_hello : V2ClientHello(rec); @@ -81,7 +80,7 @@ type PlaintextRecord(rec: SSLRecord, is_orig: bool) = case rec.content_type of { default -> unknown_record : UnknownRecord(rec); }; -type SSLExtension = record { +type SSLExtension(rec: SSLRecord) = record { type: uint16; data_len: uint16; data: bytestring &length=data_len; @@ -156,10 +155,6 @@ enum AnalyzerState { } } - string orig_label(bool is_orig) - { - return string(is_orig ? "originator" :"responder"); - } double get_time_from_asn1(const ASN1_TIME * atime) { @@ -389,7 +384,7 @@ type ClientHello(rec: SSLRecord) = record { # This weirdness is to deal with the possible existence or absence # of the following fields. ext_len: uint16[] &until($element == 0 || $element != 0); - extensions : SSLExtension[] &until($input.length() == 0); + extensions : SSLExtension(rec)[] &until($input.length() == 0); } &let { state_changed : bool = $context.connection.transition(STATE_INITIAL, @@ -663,7 +658,7 @@ type UnknownRecord(rec: SSLRecord) = record { state_changed : bool = $context.connection.lost_track(); }; -type CiphertextRecord(rec: SSLRecord, is_orig: bool) = record { +type CiphertextRecord(rec: SSLRecord) = record { cont : bytestring &restofdata &transient; } &let { state_changed : bool = From 8e89d78788eb39784c54878d75bb2f0c255310c9 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 9 Dec 2011 17:31:47 -0600 Subject: [PATCH 165/964] Add more cluster and communication framework documentation. --- scripts/base/frameworks/cluster/__load__.bro | 8 ++-- scripts/base/frameworks/cluster/main.bro | 46 ++++++++++++++++--- .../base/frameworks/cluster/nodes/proxy.bro | 4 ++ .../base/frameworks/cluster/nodes/worker.bro | 4 ++ .../frameworks/cluster/setup-connections.bro | 3 ++ .../base/frameworks/communication/main.bro | 37 +++++++++++++-- 6 files changed, 86 insertions(+), 16 deletions(-) diff --git a/scripts/base/frameworks/cluster/__load__.bro b/scripts/base/frameworks/cluster/__load__.bro index bccb35dbb1..0f9003514d 100644 --- a/scripts/base/frameworks/cluster/__load__.bro +++ b/scripts/base/frameworks/cluster/__load__.bro @@ -9,10 +9,10 @@ redef peer_description = Cluster::node; # Add a cluster prefix. @prefixes += cluster -## If this script isn't found anywhere, the cluster bombs out. -## Loading the cluster framework requires that a script by this name exists -## somewhere in the BROPATH. The only thing in the file should be the -## cluster definition in the :bro:id:`Cluster::nodes` variable. +# If this script isn't found anywhere, the cluster bombs out. +# Loading the cluster framework requires that a script by this name exists +# somewhere in the BROPATH. The only thing in the file should be the +# cluster definition in the :bro:id:`Cluster::nodes` variable. @load cluster-layout @if ( Cluster::node in Cluster::nodes ) diff --git a/scripts/base/frameworks/cluster/main.bro b/scripts/base/frameworks/cluster/main.bro index 2e7fc32d8b..1e89e9b2a7 100644 --- a/scripts/base/frameworks/cluster/main.bro +++ b/scripts/base/frameworks/cluster/main.bro @@ -1,21 +1,45 @@ +##! A framework for establishing and controlling a cluster of Bro instances. +##! In order to use the cluster framework, a script named +##! ``cluster-layout.bro`` must exist somewhere in Bro's script search path +##! which has a cluster definition of the :bro:id:`Cluster::nodes` variable. +##! The ``CLUSTER_NODE`` environment variable or :bro:id:`Cluster::node` +##! must also be sent and the cluster framework loaded as a package like +##! ``@load base/frameworks/cluster``. + @load base/frameworks/control module Cluster; export { + ## The cluster logging stream identifier. redef enum Log::ID += { LOG }; - + + ## The record type which contains the column fields of the cluster log. type Info: record { + ## The time at which a cluster message was generated. ts: time; + ## A message indicating information about the cluster's operation. message: string; } &log; - + + ## Types of nodes that are allowed to participate in the cluster + ## configuration. type NodeType: enum { + ## A dummy node type indicating the local node is not operating + ## within a cluster. NONE, + ## A node type which is allowed to view/manipulate the configuration + ## of other nodes in the cluster. CONTROL, + ## A node type responsible for log and policy management. MANAGER, + ## A node type for relaying worker node communication and synchronizing + ## worker node state. PROXY, + ## The node type doing all the actual traffic analysis. WORKER, + ## A node acting as a traffic recorder using the + ## `Time Machine `_ software. TIME_MACHINE, }; @@ -49,30 +73,38 @@ export { ## Record type to indicate a node in a cluster. type Node: record { + ## Identifies the type of cluster node in this node's configuration. node_type: NodeType; + ## The IP address of the cluster node. ip: addr; + ## The port to which the this local node can connect when + ## establishing communication. p: port; - ## Identifier for the interface a worker is sniffing. interface: string &optional; - - ## Manager node this node uses. For workers and proxies. + ## Name of the manager node this node uses. For workers and proxies. manager: string &optional; - ## Proxy node this node uses. For workers and managers. + ## Name of the proxy node this node uses. For workers and managers. proxy: string &optional; - ## Worker nodes that this node connects with. For managers and proxies. + ## Names of worker nodes that this node connects with. + ## For managers and proxies. workers: set[string] &optional; + ## Name of a time machine node with which this node connects. time_machine: string &optional; }; ## This function can be called at any time to determine if the cluster ## framework is being enabled for this run. + ## + ## Returns: True if :bro:id:`Cluster::node` has been set. global is_enabled: function(): bool; ## This function can be called at any time to determine what type of ## cluster node the current Bro instance is going to be acting as. ## If :bro:id:`Cluster::is_enabled` returns false, then ## :bro:enum:`Cluster::NONE` is returned. + ## + ## Returns: The :bro:type:`Cluster::NodeType` the calling node acts as. global local_node_type: function(): NodeType; ## This gives the value for the number of workers currently connected to, diff --git a/scripts/base/frameworks/cluster/nodes/proxy.bro b/scripts/base/frameworks/cluster/nodes/proxy.bro index 426a5c26fc..e38a5e9109 100644 --- a/scripts/base/frameworks/cluster/nodes/proxy.bro +++ b/scripts/base/frameworks/cluster/nodes/proxy.bro @@ -1,3 +1,7 @@ +##! Redefines the options common to all proxy nodes within a Bro cluster. +##! In particular, proxies are not meant to produce logs locally and they +##! do not forward events anywhere, they mainly synchronize state between +##! worker nodes. @prefixes += cluster-proxy diff --git a/scripts/base/frameworks/cluster/nodes/worker.bro b/scripts/base/frameworks/cluster/nodes/worker.bro index 454cf57c85..61d5228c88 100644 --- a/scripts/base/frameworks/cluster/nodes/worker.bro +++ b/scripts/base/frameworks/cluster/nodes/worker.bro @@ -1,3 +1,7 @@ +##! Redefines some options common to all worker nodes within a Bro cluster. +##! In particular, worker nodes do not produce logs locally, instead they +##! send them off to a manager node for processing. + @prefixes += cluster-worker ## Don't do any local logging. diff --git a/scripts/base/frameworks/cluster/setup-connections.bro b/scripts/base/frameworks/cluster/setup-connections.bro index 059b984d61..395c11953a 100644 --- a/scripts/base/frameworks/cluster/setup-connections.bro +++ b/scripts/base/frameworks/cluster/setup-connections.bro @@ -1,3 +1,6 @@ +##! This script establishes communication among all nodes in a cluster +##! as defined by :bro:id:`Cluster::nodes`. + @load ./main @load base/frameworks/communication diff --git a/scripts/base/frameworks/communication/main.bro b/scripts/base/frameworks/communication/main.bro index 01c608c8db..04772f57aa 100644 --- a/scripts/base/frameworks/communication/main.bro +++ b/scripts/base/frameworks/communication/main.bro @@ -1,11 +1,13 @@ -##! Connect to remote Bro or Broccoli instances to share state and/or transfer -##! events. +##! Facilitates connecting to remote Bro or Broccoli instances to share state +##! and/or transfer events. @load base/frameworks/packet-filter module Communication; export { + + ## The communication logging stream identifier. redef enum Log::ID += { LOG }; ## Which interface to listen on (0.0.0.0 for any interface). @@ -21,14 +23,25 @@ export { ## compression. global compression_level = 0 &redef; + ## A record type containing the column fields of the communication log. type Info: record { + ## The network time at which a communication event occurred. ts: time &log; + ## The peer name (if any) for which a communication event is concerned. peer: string &log &optional; + ## Where the communication event message originated from, that is, + ## either from the scripting layer or inside the Bro process. src_name: string &log &optional; + ## .. todo:: currently unused. connected_peer_desc: string &log &optional; + ## .. todo:: currently unused. connected_peer_addr: addr &log &optional; + ## .. todo:: currently unused. connected_peer_port: port &log &optional; + ## The severity of the communication event message. level: string &log &optional; + ## A message describing the communication event between Bro or + ## Broccoli instances. message: string &log; }; @@ -77,7 +90,7 @@ export { auth: bool &default = F; ## If not set, no capture filter is sent. - ## If set to "", the default cature filter is sent. + ## If set to "", the default capture filter is sent. capture_filter: string &optional; ## Whether to use SSL-based communication. @@ -96,11 +109,25 @@ export { ## The table of Bro or Broccoli nodes that Bro will initiate connections ## to or respond to connections from. global nodes: table[string] of Node &redef; - + + ## A table of peer nodes for which this node issued a + ## :bro:id:`Communication::connect_peer` call but with which a connection + ## has not yet been established or with which a connection has been + ## closed and is currently in the process of retrying to establish. + ## When a connection is successfully established, the peer is removed + ## from the table. global pending_peers: table[peer_id] of Node; + + ## A table of peer nodes for which this node has an established connection. + ## Peers are automatically removed if their connection is closed and + ## automatically added back if a connection is re-established later. global connected_peers: table[peer_id] of Node; - ## Connect to nodes[node], independent of its "connect" flag. + ## Connect to a node in :bro:id:`Communication::nodes` independent + ## of its "connect" flag. + ## + ## peer: the string used to index a particular node within the + ## :bro:id:`Communication::nodes` table. global connect_peer: function(peer: string); } From dcc7fe3c38a56f1941375ad30ab5cf4d533324a0 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 8 Dec 2011 15:27:47 -0800 Subject: [PATCH 166/964] start reworking interface of software framework. working apart from detect-webapps.bro, which direcly manipulates a no longer available interface... --- scripts/base/frameworks/software/main.bro | 112 ++++++++++-------- scripts/policy/protocols/ftp/software.bro | 3 +- .../http/software-browser-plugins.bro | 7 +- scripts/policy/protocols/http/software.bro | 8 +- scripts/policy/protocols/smtp/software.bro | 3 +- scripts/policy/protocols/ssh/software.bro | 6 +- .../frameworks/software/version-parsing.bro | 104 ++++++++-------- 7 files changed, 124 insertions(+), 119 deletions(-) diff --git a/scripts/base/frameworks/software/main.bro b/scripts/base/frameworks/software/main.bro index 817ae92e40..5cfd249982 100644 --- a/scripts/base/frameworks/software/main.bro +++ b/scripts/base/frameworks/software/main.bro @@ -30,6 +30,12 @@ export { addl: string &optional; ##< Additional version string (e.g. "beta42") } &log; + type SoftwareDescription: record { + version: Version; + name: string; + unparsed_version: string; + }; + type Info: record { ## The time at which the software was first detected. ts: time &log; @@ -48,6 +54,21 @@ export { ## The full unparsed version string found because the version parsing ## doesn't work 100% reliably and this acts as a fall back in the logs. unparsed_version: string &log &optional; + }; + + type AddItem: record { + ## The connection + id: conn_id; + ## The unparsed string representing the software version + banner: string; + ## Pre-parsed version. If this field is present, banner should only contain the name of the software + version: Version &optional; + ## The IP address detected running the software. + host: addr; + ## The port on which the software is running (if applicable). + host_p: port &optional; + ## The type of software detected (e.g. WEB_SERVER) + sw_type: Type; ## This can indicate that this software being detected should ## definitely be sent onward to the logging framework. By @@ -58,7 +79,7 @@ export { ## needs to happen in a specific way to the software. force_log: bool &default=F; }; - + ## The hosts whose software should be detected and tracked. ## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS const asset_tracking = LOCAL_HOSTS &redef; @@ -68,22 +89,14 @@ export { ## unparsed_version: This is the full string from which the ## :bro:type:`Software::Info` was extracted. ## Returns: T if the software was logged, F otherwise. - global found: function(id: conn_id, info: Software::Info): bool; + global found: function(i: AddItem): bool; ## This function can take many software version strings and parse them ## into a sensible :bro:type:`Software::Version` record. There are ## still many cases where scripts may have to have their own specific ## version parsing though. - global parse: function(unparsed_version: string, - host: addr, - software_type: Type): Info; + global parse: function(unparsed_version: string): SoftwareDescription; - ## This function is the equivalent to parse for software that has a specific - ## source port (i.e. server software) - global parse_with_port: function(unparsed_version: string, - host: addr, host_p: port, - software_type: Type): Info; - ## Compare two versions. ## Returns: -1 for v1 < v2, 0 for v1 == v2, 1 for v1 > v2. ## If the numerical version numbers match, the addl string @@ -116,9 +129,7 @@ event bro_init() Log::create_stream(Software::LOG, [$columns=Info, $ev=log_software]); } -function parse_mozilla(unparsed_version: string, - host: addr, - software_type: Type): Info +function parse_mozilla(unparsed_version: string): SoftwareDescription { local software_name = ""; local v: Version; @@ -129,7 +140,7 @@ function parse_mozilla(unparsed_version: string, software_name = "Opera"; parts = split_all(unparsed_version, /Opera [0-9\.]*$/); if ( 2 in parts ) - v = parse(parts[2], host, software_type)$version; + v = parse(parts[2])$version; } else if ( / MSIE / in unparsed_version ) { @@ -144,7 +155,7 @@ function parse_mozilla(unparsed_version: string, { parts = split_all(unparsed_version, /MSIE [0-9]{1,2}\.*[0-9]*b?[0-9]*/); if ( 2 in parts ) - v = parse(parts[2], host, software_type)$version; + v = parse(parts[2])$version; } } else if ( /Version\/.*Safari\// in unparsed_version ) @@ -153,7 +164,7 @@ function parse_mozilla(unparsed_version: string, parts = split_all(unparsed_version, /Version\/[0-9\.]*/); if ( 2 in parts ) { - v = parse(parts[2], host, software_type)$version; + v = parse(parts[2])$version; if ( / Mobile\/?.* Safari/ in unparsed_version ) v$addl = "Mobile"; } @@ -163,7 +174,7 @@ function parse_mozilla(unparsed_version: string, parts = split_all(unparsed_version, /(Firefox|Netscape|Thunderbird)\/[0-9\.]*/); if ( 2 in parts ) { - local tmp_s = parse(parts[2], host, software_type); + local tmp_s = parse(parts[2]); software_name = tmp_s$name; v = tmp_s$version; } @@ -173,7 +184,7 @@ function parse_mozilla(unparsed_version: string, software_name = "Chrome"; parts = split_all(unparsed_version, /Chrome\/[0-9\.]*/); if ( 2 in parts ) - v = parse(parts[2], host, software_type)$version; + v = parse(parts[2])$version; } else if ( /^Opera\// in unparsed_version ) { @@ -184,12 +195,12 @@ function parse_mozilla(unparsed_version: string, software_name = parts[2]; parts = split_all(unparsed_version, /Version\/[0-9\.]*/); if ( 2 in parts ) - v = parse(parts[2], host, software_type)$version; + v = parse(parts[2])$version; else { parts = split_all(unparsed_version, /Opera Mini\/[0-9\.]*/); if ( 2 in parts ) - v = parse(parts[2], host, software_type)$version; + v = parse(parts[2])$version; } } else @@ -197,7 +208,7 @@ function parse_mozilla(unparsed_version: string, software_name = "Opera"; parts = split_all(unparsed_version, /Version\/[0-9\.]*/); if ( 2 in parts ) - v = parse(parts[2], host, software_type)$version; + v = parse(parts[2])$version; } } else if ( /AppleWebKit\/[0-9\.]*/ in unparsed_version ) @@ -205,26 +216,24 @@ function parse_mozilla(unparsed_version: string, software_name = "Unspecified WebKit"; parts = split_all(unparsed_version, /AppleWebKit\/[0-9\.]*/); if ( 2 in parts ) - v = parse(parts[2], host, software_type)$version; + v = parse(parts[2])$version; } - return [$ts=network_time(), $host=host, $name=software_name, $version=v, - $software_type=software_type, $unparsed_version=unparsed_version]; + return [$version=v, $unparsed_version=unparsed_version, $name=software_name]; } # Don't even try to understand this now, just make sure the tests are # working. -function parse(unparsed_version: string, - host: addr, - software_type: Type): Info +function parse(unparsed_version: string): SoftwareDescription { local software_name = ""; local v: Version; - + + # Parse browser-alike versions separately if ( /^(Mozilla|Opera)\/[0-9]\./ in unparsed_version ) { - return parse_mozilla(unparsed_version, host, software_type); + return parse_mozilla(unparsed_version); } else { @@ -249,7 +258,7 @@ function parse(unparsed_version: string, if ( 4 in version_numbers && version_numbers[4] != "" ) v$addl = strip(version_numbers[4]); else if ( 3 in version_parts && version_parts[3] != "" && - version_parts[3] != ")" ) + version_parts[3] != ")" ) { if ( /^[[:blank:]]*\([a-zA-Z0-9\-\._[:blank:]]*\)/ in version_parts[3] ) { @@ -286,22 +295,10 @@ function parse(unparsed_version: string, v$major = extract_count(version_numbers[1]); } } - return [$ts=network_time(), $host=host, $name=software_name, - $version=v, $unparsed_version=unparsed_version, - $software_type=software_type]; + + return [$version=v, $unparsed_version=unparsed_version, $name=software_name]; } -function parse_with_port(unparsed_version: string, - host: addr, host_p: port, - software_type: Type): Info -{ - local i: Info; - i = parse(unparsed_version, host, software_type); - i$host_p = host_p; - i$proto = get_port_transport_proto(host_p); - - return i; -} function cmp_versions(v1: Version, v2: Version): int @@ -385,7 +382,7 @@ function software_fmt(i: Info): string # Insert a mapping into the table # Overides old entries for the same software and generates events if needed. -event software_register(id: conn_id, info: Info) +event software_register(id: conn_id, force_log: bool, info: Info) { # Host already known? if ( info$host !in tracked ) @@ -401,7 +398,7 @@ event software_register(id: conn_id, info: Info) # If the version hasn't changed, then we're just redetecting the # same thing, then we don't care. This results in no extra logging. # But if the $force_log value is set then we'll continue. - if ( ! info$force_log && cmp_versions(old$version, info$version) == 0 ) + if ( ! force_log && cmp_versions(old$version, info$version) == 0 ) return; } ts[info$name] = info; @@ -409,11 +406,26 @@ event software_register(id: conn_id, info: Info) Log::write(Software::LOG, info); } -function found(id: conn_id, info: Info): bool +function found(i: AddItem): bool { - if ( info$force_log || addr_matches_host(info$host, asset_tracking) ) + if ( i$force_log || addr_matches_host(i$host, asset_tracking) ) { - event software_register(id, info); + + local sw: SoftwareDescription; + + if ( i?$version ) # already fully parsed, banner should contain the software name + { + sw = [$version=i$version, $name=i$banner, $unparsed_version=i$banner]; + } + else + { + sw = parse(i$banner); + } + + event software_register(i$id, i$force_log, [$ts=network_time(), $host=i$host, $host_p=i$host_p, $name=sw$name, + $version=sw$version, $unparsed_version=sw$unparsed_version, + $software_type=i$sw_type] ); + return T; } else diff --git a/scripts/policy/protocols/ftp/software.bro b/scripts/policy/protocols/ftp/software.bro index 622357a608..24e7ff0b0a 100644 --- a/scripts/policy/protocols/ftp/software.bro +++ b/scripts/policy/protocols/ftp/software.bro @@ -21,7 +21,6 @@ event ftp_request(c: connection, command: string, arg: string) &priority=4 { if ( command == "CLNT" ) { - local si = Software::parse(arg, c$id$orig_h, FTP_CLIENT); - Software::found(c$id, si); + Software::found([$id=c$id, $banner=arg, $host=c$id$orig_h, $sw_type=FTP_CLIENT]); } } diff --git a/scripts/policy/protocols/http/software-browser-plugins.bro b/scripts/policy/protocols/http/software-browser-plugins.bro index db9eafd1a7..21bc8d07cb 100644 --- a/scripts/policy/protocols/http/software-browser-plugins.bro +++ b/scripts/policy/protocols/http/software-browser-plugins.bro @@ -26,8 +26,7 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr # Flash doesn't include it's name so we'll add it here since it # simplifies the version parsing. value = cat("Flash/", value); - local flash_version = Software::parse(value, c$id$orig_h, BROWSER_PLUGIN); - Software::found(c$id, flash_version); + Software::found([$id=c$id, $banner=flash_version, $host=c$id$orig_h, $sw_type=BROWSER_PLUGIN]); } } else @@ -54,7 +53,7 @@ event log_http(rec: Info) local plugins = split(sw, /[[:blank:]]*;[[:blank:]]*/); for ( i in plugins ) - Software::found(rec$id, Software::parse(plugins[i], rec$id$orig_h, BROWSER_PLUGIN)); + Software::found([$id=rec$id, $banner=plugins[i], $host=rec$id$orig_h, $sw_type=BROWSER_PLUGIN]); } } - } \ No newline at end of file + } diff --git a/scripts/policy/protocols/http/software.bro b/scripts/policy/protocols/http/software.bro index 0a07ba0331..43552798e6 100644 --- a/scripts/policy/protocols/http/software.bro +++ b/scripts/policy/protocols/http/software.bro @@ -20,18 +20,18 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr if ( is_orig ) { if ( name == "USER-AGENT" && ignored_user_agents !in value ) - Software::found(c$id, Software::parse(value, c$id$orig_h, BROWSER)); + Software::found([$id=c$id, $banner=value, $host=c$id$orig_h, $sw_type=BROWSER]); } else { if ( name == "SERVER" ) - Software::found(c$id, Software::parse_with_port(value, c$id$resp_h, c$id$resp_p, SERVER)); + Software::found([$id=c$id, $banner=value, $host=c$id$resp_h, $host_p=c$id$resp_p, $sw_type=SERVER]); else if ( name == "X-POWERED-BY" ) - Software::found(c$id, Software::parse_with_port(value, c$id$resp_h, c$id$resp_p, APPSERVER)); + Software::found([$id=c$id, $banner=value, $host=c$id$resp_h, $host_p=c$id$resp_p, $sw_type=APPSERVER]); else if ( name == "MICROSOFTSHAREPOINTTEAMSERVICES" ) { value = cat("SharePoint/", value); - Software::found(c$id, Software::parse_with_port(value, c$id$resp_h, c$id$resp_p, APPSERVER)); + Software::found([$id=c$id, $banner=value, $host=c$id$resp_h, $host_p=c$id$resp_p, $sw_type=APPSERVER]); } } } diff --git a/scripts/policy/protocols/smtp/software.bro b/scripts/policy/protocols/smtp/software.bro index 3c4c870885..881848abc2 100644 --- a/scripts/policy/protocols/smtp/software.bro +++ b/scripts/policy/protocols/smtp/software.bro @@ -75,8 +75,7 @@ event log_smtp(rec: Info) if ( addr_matches_host(rec$id$orig_h, detect_clients_in_messages_from) ) { - local s = Software::parse(rec$user_agent, client_ip, s_type); - Software::found(rec$id, s); + Software::found([$id=rec$id, $banner=rec$user_agent, $host=client_ip, $sw_type=s_type]); } } } diff --git a/scripts/policy/protocols/ssh/software.bro b/scripts/policy/protocols/ssh/software.bro index 0bb6ebc43f..a0160ee7ba 100644 --- a/scripts/policy/protocols/ssh/software.bro +++ b/scripts/policy/protocols/ssh/software.bro @@ -16,14 +16,12 @@ event ssh_client_version(c: connection, version: string) &priority=4 { # Get rid of the protocol information when passing to the software framework. local cleaned_version = sub(version, /^SSH[0-9\.\-]+/, ""); - local si = Software::parse(cleaned_version, c$id$orig_h, CLIENT); - Software::found(c$id, si); + Software::found([$id=c$id, $banner=cleaned_version, $host=c$id$orig_h, $sw_type=CLIENT]); } event ssh_server_version(c: connection, version: string) &priority=4 { # Get rid of the protocol information when passing to the software framework. local cleaned_version = sub(version, /SSH[0-9\.\-]{2,}/, ""); - local si = Software::parse_with_port(cleaned_version, c$id$resp_h, c$id$resp_p, SERVER); - Software::found(c$id, si); + Software::found([$id=c$id, $banner=cleaned_version, $host=c$id$resp_h, $host_p=c$id$resp_p, $sw_type=SERVER]); } diff --git a/testing/btest/scripts/base/frameworks/software/version-parsing.bro b/testing/btest/scripts/base/frameworks/software/version-parsing.bro index dda3edea4b..c0c2147313 100644 --- a/testing/btest/scripts/base/frameworks/software/version-parsing.bro +++ b/testing/btest/scripts/base/frameworks/software/version-parsing.bro @@ -1,116 +1,114 @@ # @TEST-EXEC: bro %INPUT > output # @TEST-EXEC: btest-diff output -global ts = network_time(); -global host = 0.0.0.0; - -global matched_software: table[string] of Software::Info = { +global matched_software: table[string] of Software::SoftwareDescription = { ["OpenSSH_4.4"] = - [$name="OpenSSH", $version=[$major=4,$minor=4], $host=host, $ts=ts], + [$name="OpenSSH", $version=[$major=4,$minor=4], $unparsed_version=""], ["OpenSSH_5.2"] = - [$name="OpenSSH", $version=[$major=5,$minor=2], $host=host, $ts=ts], + [$name="OpenSSH", $version=[$major=5,$minor=2], $unparsed_version=""], ["Apache/2.0.63 (Unix) mod_auth_kerb/5.3 mod_ssl/2.0.63 OpenSSL/0.9.7a mod_fastcgi/2.4.2"] = - [$name="Apache", $version=[$major=2,$minor=0,$minor2=63,$addl="Unix"], $host=host, $ts=ts], + [$name="Apache", $version=[$major=2,$minor=0,$minor2=63,$addl="Unix"], $unparsed_version=""], ["Apache/1.3.19 (Unix)"] = - [$name="Apache", $version=[$major=1,$minor=3,$minor2=19,$addl="Unix"], $host=host, $ts=ts], + [$name="Apache", $version=[$major=1,$minor=3,$minor2=19,$addl="Unix"], $unparsed_version=""], ["ProFTPD 1.2.5rc1 Server (Debian)"] = - [$name="ProFTPD", $version=[$major=1,$minor=2,$minor2=5,$addl="rc1"], $host=host, $ts=ts], + [$name="ProFTPD", $version=[$major=1,$minor=2,$minor2=5,$addl="rc1"], $unparsed_version=""], ["wu-2.4.2-academ[BETA-18-VR14](1)"] = - [$name="wu", $version=[$major=2,$minor=4,$minor2=2,$addl="academ"], $host=host, $ts=ts], + [$name="wu", $version=[$major=2,$minor=4,$minor2=2,$addl="academ"], $unparsed_version=""], ["wu-2.6.2(1)"] = - [$name="wu", $version=[$major=2,$minor=6,$minor2=2,$addl="1"], $host=host, $ts=ts], + [$name="wu", $version=[$major=2,$minor=6,$minor2=2,$addl="1"], $unparsed_version=""], ["Java1.2.2-JDeveloper"] = - [$name="Java", $version=[$major=1,$minor=2,$minor2=2,$addl="JDeveloper"], $host=host, $ts=ts], + [$name="Java", $version=[$major=1,$minor=2,$minor2=2,$addl="JDeveloper"], $unparsed_version=""], ["Java/1.6.0_13"] = - [$name="Java", $version=[$major=1,$minor=6,$minor2=0,$addl="13"], $host=host, $ts=ts], + [$name="Java", $version=[$major=1,$minor=6,$minor2=0,$addl="13"], $unparsed_version=""], ["Python-urllib/3.1"] = - [$name="Python-urllib", $version=[$major=3,$minor=1], $host=host, $ts=ts], + [$name="Python-urllib", $version=[$major=3,$minor=1], $unparsed_version=""], ["libwww-perl/5.820"] = - [$name="libwww-perl", $version=[$major=5,$minor=820], $host=host, $ts=ts], + [$name="libwww-perl", $version=[$major=5,$minor=820], $unparsed_version=""], ["Wget/1.9+cvs-stable (Red Hat modified)"] = - [$name="Wget", $version=[$major=1,$minor=9,$addl="+cvs"], $host=host, $ts=ts], + [$name="Wget", $version=[$major=1,$minor=9,$addl="+cvs"], $unparsed_version=""], ["Wget/1.11.4 (Red Hat modified)"] = - [$name="Wget", $version=[$major=1,$minor=11,$minor2=4,$addl="Red Hat modified"], $host=host, $ts=ts], + [$name="Wget", $version=[$major=1,$minor=11,$minor2=4,$addl="Red Hat modified"], $unparsed_version=""], ["curl/7.15.1 (i486-pc-linux-gnu) libcurl/7.15.1 OpenSSL/0.9.8a zlib/1.2.3 libidn/0.5.18"] = - [$name="curl", $version=[$major=7,$minor=15,$minor2=1,$addl="i486-pc-linux-gnu"], $host=host, $ts=ts], + [$name="curl", $version=[$major=7,$minor=15,$minor2=1,$addl="i486-pc-linux-gnu"], $unparsed_version=""], ["Apache"] = - [$name="Apache", $host=host, $ts=ts], + [$name="Apache", $unparsed_version=""], ["Zope/(Zope 2.7.8-final, python 2.3.5, darwin) ZServer/1.1 Plone/Unknown"] = - [$name="Zope/(Zope", $version=[$major=2,$minor=7,$minor2=8,$addl="final"], $host=host, $ts=ts], + [$name="Zope/(Zope", $version=[$major=2,$minor=7,$minor2=8,$addl="final"], $unparsed_version=""], ["The Bat! (v2.00.9) Personal"] = - [$name="The Bat!", $version=[$major=2,$minor=0,$minor2=9,$addl="Personal"], $host=host, $ts=ts], + [$name="The Bat!", $version=[$major=2,$minor=0,$minor2=9,$addl="Personal"], $unparsed_version=""], ["Flash/10,2,153,1"] = - [$name="Flash", $version=[$major=10,$minor=2,$minor2=153,$addl="1"], $host=host, $ts=ts], + [$name="Flash", $version=[$major=10,$minor=2,$minor2=153,$addl="1"], $unparsed_version=""], ["mt2/1.2.3.967 Oct 13 2010-13:40:24 ord-pixel-x2 pid 0x35a3 13731"] = - [$name="mt2", $version=[$major=1,$minor=2,$minor2=3,$addl="967"], $host=host, $ts=ts], + [$name="mt2", $version=[$major=1,$minor=2,$minor2=3,$addl="967"], $unparsed_version=""], ["CacheFlyServe v26b"] = - [$name="CacheFlyServe", $version=[$major=26,$addl="b"], $host=host, $ts=ts], + [$name="CacheFlyServe", $version=[$major=26,$addl="b"], $unparsed_version=""], ["Apache/2.0.46 (Win32) mod_ssl/2.0.46 OpenSSL/0.9.7b mod_jk2/2.0.4"] = - [$name="Apache", $version=[$major=2,$minor=0,$minor2=46,$addl="Win32"], $host=host, $ts=ts], + [$name="Apache", $version=[$major=2,$minor=0,$minor2=46,$addl="Win32"], $unparsed_version=""], # I have no clue how I'd support this without a special case. #["Apache mod_fcgid/2.3.6 mod_auth_passthrough/2.1 mod_bwlimited/1.4 FrontPage/5.0.2.2635"] = - # [$name="Apache", $version=[], $host=host, $ts=ts], + # [$name="Apache", $version=[], $unparsed_version=""], ["Apple iPhone v4.3.1 Weather v1.0.0.8G4"] = - [$name="Apple iPhone", $version=[$major=4,$minor=3,$minor2=1,$addl="Weather"], $host=host, $ts=ts], + [$name="Apple iPhone", $version=[$major=4,$minor=3,$minor2=1,$addl="Weather"], $unparsed_version=""], ["Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5"] = - [$name="Safari", $version=[$major=5,$minor=0,$minor2=2,$addl="Mobile"], $host=host, $ts=ts], + [$name="Safari", $version=[$major=5,$minor=0,$minor2=2,$addl="Mobile"], $unparsed_version=""], ["Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.205 Safari/534.16"] = - [$name="Chrome", $version=[$major=10,$minor=0,$minor2=648,$addl="205"], $host=host, $ts=ts], + [$name="Chrome", $version=[$major=10,$minor=0,$minor2=648,$addl="205"], $unparsed_version=""], ["Opera/9.80 (Windows NT 6.1; U; sv) Presto/2.7.62 Version/11.01"] = - [$name="Opera", $version=[$major=11,$minor=1], $host=host, $ts=ts], + [$name="Opera", $version=[$major=11,$minor=1], $unparsed_version=""], ["Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2.11) Gecko/20101013 Lightning/1.0b2 Thunderbird/3.1.5"] = - [$name="Thunderbird", $version=[$major=3,$minor=1,$minor2=5], $host=host, $ts=ts], + [$name="Thunderbird", $version=[$major=3,$minor=1,$minor2=5], $unparsed_version=""], ["iTunes/9.0 (Macintosh; Intel Mac OS X 10.5.8) AppleWebKit/531.9"] = - [$name="iTunes", $version=[$major=9,$minor=0,$addl="Macintosh"], $host=host, $ts=ts], + [$name="iTunes", $version=[$major=9,$minor=0,$addl="Macintosh"], $unparsed_version=""], ["Java1.3.1_04"] = - [$name="Java", $version=[$major=1,$minor=3,$minor2=1,$addl="04"], $host=host, $ts=ts], + [$name="Java", $version=[$major=1,$minor=3,$minor2=1,$addl="04"], $unparsed_version=""], ["Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"] = - [$name="Safari", $version=[$major=4,$minor=0,$addl="Mobile"], $host=host, $ts=ts], + [$name="Safari", $version=[$major=4,$minor=0,$addl="Mobile"], $unparsed_version=""], ["Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-us) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27"] = - [$name="Safari", $version=[$major=5,$minor=0,$minor2=4], $host=host, $ts=ts], + [$name="Safari", $version=[$major=5,$minor=0,$minor2=4], $unparsed_version=""], ["Mozilla/5.0 (iPod; U; CPU iPhone OS 4_0 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8A293 Safari/6531.22.7"] = - [$name="Safari", $version=[$major=4,$minor=0,$minor2=5,$addl="Mobile"], $host=host, $ts=ts], + [$name="Safari", $version=[$major=4,$minor=0,$minor2=5,$addl="Mobile"], $unparsed_version=""], ["Opera/9.80 (J2ME/MIDP; Opera Mini/9.80 (S60; SymbOS; Opera Mobi/23.348; U; en) Presto/2.5.25 Version/10.54"] = - [$name="Opera Mini", $version=[$major=10,$minor=54], $host=host, $ts=ts], + [$name="Opera Mini", $version=[$major=10,$minor=54], $unparsed_version=""], ["Opera/9.80 (J2ME/MIDP; Opera Mini/5.0.18741/18.794; U; en) Presto/2.4.15"] = - [$name="Opera Mini", $version=[$major=5,$minor=0,$minor2=18741], $host=host, $ts=ts], + [$name="Opera Mini", $version=[$major=5,$minor=0,$minor2=18741], $unparsed_version=""], ["Opera/9.80 (Windows NT 5.1; Opera Mobi/49; U; en) Presto/2.4.18 Version/10.00"] = - [$name="Opera Mobi", $version=[$major=10,$minor=0], $host=host, $ts=ts], + [$name="Opera Mobi", $version=[$major=10,$minor=0], $unparsed_version=""], ["Mozilla/4.0 (compatible; MSIE 8.0; Android 2.2.2; Linux; Opera Mobi/ADR-1103311355; en) Opera 11.00"] = - [$name="Opera", $version=[$major=11,$minor=0], $host=host, $ts=ts], + [$name="Opera", $version=[$major=11,$minor=0], $unparsed_version=""], ["Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.2) Gecko/20040804 Netscape/7.2 (ax)"] = - [$name="Netscape", $version=[$major=7,$minor=2], $host=host, $ts=ts], + [$name="Netscape", $version=[$major=7,$minor=2], $unparsed_version=""], ["Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; GTB5; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; InfoPath.2)"] = - [$name="MSIE", $version=[$major=7,$minor=0], $host=host, $ts=ts], + [$name="MSIE", $version=[$major=7,$minor=0], $unparsed_version=""], ["Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.1; Media Center PC 3.0; .NET CLR 1.0.3705; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1)"] = - [$name="MSIE", $version=[$major=7,$minor=0,$addl="b"], $host=host, $ts=ts], + [$name="MSIE", $version=[$major=7,$minor=0,$addl="b"], $unparsed_version=""], ["Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Tablet PC 2.0; InfoPath.2; InfoPath.3)"] = - [$name="MSIE", $version=[$major=8,$minor=0], $host=host, $ts=ts], + [$name="MSIE", $version=[$major=8,$minor=0], $unparsed_version=""], ["Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)"] = - [$name="MSIE", $version=[$major=9,$minor=0], $host=host, $ts=ts], + [$name="MSIE", $version=[$major=9,$minor=0], $unparsed_version=""], ["Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; InfoPath.3; Creative AutoUpdate v1.40.02)"] = - [$name="MSIE", $version=[$major=9,$minor=0], $host=host, $ts=ts], + [$name="MSIE", $version=[$major=9,$minor=0], $unparsed_version=""], ["Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)"] = - [$name="MSIE", $version=[$major=10,$minor=0], $host=host, $ts=ts], + [$name="MSIE", $version=[$major=10,$minor=0], $unparsed_version=""], ["The Bat! (3.0.1 RC3) Professional"] = - [$name="The Bat!", $version=[$major=3,$minor=0,$minor2=1,$addl="RC3"], $host=host, $ts=ts], + [$name="The Bat!", $version=[$major=3,$minor=0,$minor2=1,$addl="RC3"], $unparsed_version=""], # This is an FTP client (found with CLNT command) ["Total Commander"] = - [$name="Total Commander", $version=[], $host=host, $ts=ts], + [$name="Total Commander", $version=[], $unparsed_version=""], ["(vsFTPd 2.0.5)"] = - [$name="vsFTPd", $version=[$major=2,$minor=0,$minor2=5], $host=host, $ts=ts], + [$name="vsFTPd", $version=[$major=2,$minor=0,$minor2=5], $unparsed_version=""], ["Apple Mail (2.1084)"] = - [$name="Apple Mail", $version=[$major=2,$minor=1084], $host=host, $ts=ts], + [$name="Apple Mail", $version=[$major=2,$minor=1084], $unparsed_version=""], }; event bro_init() { for ( sw in matched_software ) { - local output = Software::parse(sw, host, Software::UNKNOWN); - local baseline: Software::Info; + local output = Software::parse(sw); + local baseline: Software::SoftwareDescription; baseline = matched_software[sw]; if ( baseline$name == output$name && + sw == output$unparsed_version && Software::cmp_versions(baseline$version,output$version) == 0 ) print fmt("success on: %s", sw); else From 00fb187927b6369f941d30e98a396c85f9e8218d Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Sat, 10 Dec 2011 00:13:37 -0500 Subject: [PATCH 167/964] SSH::Interesting_Hostname_Login cleanup. Fixes #664. --- scripts/policy/protocols/ssh/interesting-hostnames.bro | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/policy/protocols/ssh/interesting-hostnames.bro b/scripts/policy/protocols/ssh/interesting-hostnames.bro index 29886d0eb0..5771770916 100644 --- a/scripts/policy/protocols/ssh/interesting-hostnames.bro +++ b/scripts/policy/protocols/ssh/interesting-hostnames.bro @@ -36,7 +36,9 @@ event SSH::heuristic_successful_login(c: connection) if ( interesting_hostnames in hostname ) { NOTICE([$note=Interesting_Hostname_Login, - $msg=fmt("Interesting login from hostname: %s", hostname), + $msg=fmt("Possible SSH login involving a %s %s with an interesting hostname.", + Site::is_local_addr(host) ? "local" : "remote", + host == c$id$orig_h ? "client" : "server"), $sub=hostname, $conn=c]); } } From 6478b4acafaaef51f5bcbc2d86996a59dedc39e1 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Sat, 10 Dec 2011 00:18:10 -0500 Subject: [PATCH 168/964] Removing Off_Port_Protocol_Found notice. - Other very small cleanup. --- .../policy/frameworks/dpd/detect-protocols.bro | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/scripts/policy/frameworks/dpd/detect-protocols.bro b/scripts/policy/frameworks/dpd/detect-protocols.bro index 8e1ea1267f..8f4e892ce4 100644 --- a/scripts/policy/frameworks/dpd/detect-protocols.bro +++ b/scripts/policy/frameworks/dpd/detect-protocols.bro @@ -8,7 +8,6 @@ module ProtocolDetector; export { redef enum Notice::Type += { - Off_Port_Protocol_Found, # raised for each connection found Protocol_Found, Server_Found, }; @@ -155,13 +154,10 @@ function report_protocols(c: connection) { if ( [a, c$id$resp_h, c$id$resp_p] in valids ) do_notice(c, a, valids[a, c$id$resp_h, c$id$resp_p]); - else if ( [a, 0.0.0.0, c$id$resp_p] in valids ) do_notice(c, a, valids[a, 0.0.0.0, c$id$resp_p]); else do_notice(c, a, NONE); - - append_addl(c, analyzer_name(a)); } delete conns[c$id]; @@ -218,20 +214,6 @@ event protocol_confirmation(c: connection, atype: count, aid: count) } } -# event connection_analyzer_disabled(c: connection, analyzer: count) -# { -# if ( c$id !in conns ) -# return; -# -# delete conns[c$id][analyzer]; -# } - -function append_proto_addl(c: connection) - { - for ( a in conns[c$id] ) - append_addl(c, fmt_protocol(get_protocol(c, a))); - } - function found_protocol(c: connection, analyzer: count, protocol: string) { # Don't report anything running on a well-known port. From 76a0b9ad3c06c45ca8f0eeafaf32562f65e8ad4f Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Sat, 10 Dec 2011 22:33:49 -0500 Subject: [PATCH 169/964] Fixed some DPD signatures for IRC. Fixes ticket #311. - The larger issue from ticket 313 still stands. --- scripts/base/frameworks/dpd/dpd.sig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/base/frameworks/dpd/dpd.sig b/scripts/base/frameworks/dpd/dpd.sig index 8e07095b41..adda0ce54e 100644 --- a/scripts/base/frameworks/dpd/dpd.sig +++ b/scripts/base/frameworks/dpd/dpd.sig @@ -80,15 +80,15 @@ signature irc_server_reply { tcp-state responder } -signature irc_sig3 { +signature irc_server_to_server1 { ip-proto == tcp - payload /(.*\x0a)*(\x20)*[Ss][Ee][Rr][Vv][Ee][Rr](\x20)+.+\x0a/ + payload /(|.*[\r\n]) *[Ss][Ee][Rr][Vv][Ee][Rr] +[^ ]+ +[0-9]+ +:.+[\r\n]/ } -signature irc_sig4 { +signature irc_server_to_server2 { ip-proto == tcp - payload /(.*\x0a)*(\x20)*[Ss][Ee][Rr][Vv][Ee][Rr](\x20)+.+\x0a/ - requires-reverse-signature irc_sig3 + payload /(|.*[\r\n]) *[Ss][Ee][Rr][Vv][Ee][Rr] +[^ ]+ +[0-9]+ +:.+[\r\n]/ + requires-reverse-signature irc_server_to_server1 enable "irc" } From 4a9a17292ff9c7dc6060d169804a17898ba98964 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Sat, 10 Dec 2011 22:14:48 -0800 Subject: [PATCH 170/964] Finish documenting bro.bif. --- src/bro.bif | 1079 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 1020 insertions(+), 59 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index 42cf7d6093..db72777b92 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1251,6 +1251,8 @@ function exit%(%): int ## ## Returns: True after successful termination and false when Bro is still in ## the process of shutting down. +## +## .. bro:see:: is_terminating function terminate%(%): bool %{ if ( terminating ) @@ -1318,10 +1320,12 @@ static int do_system(const char* s) ## ## .. bro:see:: system_env str_shell_escape piped_exec ## -## .. note:: Note that this corresponds to the status of backgrounding the -## given command, not to the exit status of the command itself. A -## value of 127 corresponds to a failure to execute ``sh``, and -1 -## to an internal system failure. +## .. note:: +## +## Note that this corresponds to the status of backgrounding the +## given command, not to the exit status of the command itself. A +## value of 127 corresponds to a failure to execute ``sh``, and -1 +## to an internal system failure. function system%(str: string%): int %{ int result = do_system(str->CheckString()); @@ -2083,13 +2087,13 @@ function skip_http_entity_data%(c: connection, is_orig: bool%): any ## ## Returns: The unescaped URI with all ``%xx`` groups decoded. ## -## .. note:: Unescaping reserved characters may cause loss of -## information (see below). +## .. note:: ## -## RFC 2396: A URI is always in an "escaped" form, since escaping or -## unescaping a completed URI might change its semantics. Normally, -## the only time escape encodings can safely be made is when the URI -## is being created from its component parts. +## Unescaping reserved characters may cause loss of information. RFC 2396: +## A URI is always in an "escaped" form, since escaping or unescaping a +## completed URI might change its semantics. Normally, the only time +## escape encodings can safely be made is when the URI is being created +## from its component parts. function unescape_URI%(URI: string%): string %{ const u_char* line = URI->Bytes(); @@ -2146,7 +2150,10 @@ extern const char* bro_version(); ## ## Returns: A record of packet statistics. ## -## .. bro:see:: resource_usage get_matcher_stats dump_rule_stats +## .. bro:see:: do_profiling +## resource_usage +## get_matcher_stats +## dump_rule_stats ## get_gap_summary function net_stats%(%): NetStats %{ @@ -2179,7 +2186,10 @@ function net_stats%(%): NetStats ## ## Returns: A record with resource usage statistics. ## -## .. bro:see:: net_stats get_matcher_stats dump_rule_stats +## .. bro:see:: do_profiling +## net_stats +## get_matcher_stats +## dump_rule_stats ## get_gap_summary function resource_usage%(%): bro_resources %{ @@ -2254,7 +2264,11 @@ function resource_usage%(%): bro_resources ## ## Returns: A record with matcher statistics. ## -## .. bro:see:: net_stats resource_usage dump_rule_stats get_gap_summary +## .. bro:see:: do_profiling +## net_stats +## resource_usage +## dump_rule_stats +## get_gap_summary function get_matcher_stats%(%): matcher_stats %{ RuleMatcher::Stats s; @@ -2279,7 +2293,11 @@ function get_matcher_stats%(%): matcher_stats ## ## Returns: A record with TCP gap statistics. ## -## .. bro:see:: net_stats resource_usage dump_rule_stats get_matcher_stats +## .. bro:see:: do_profiling +## net_stats +## resource_usage +## dump_rule_stats +## get_matcher_stats function get_gap_summary%(%): gap_info %{ RecordVal* r = new RecordVal(gap_info); @@ -2291,11 +2309,22 @@ function get_gap_summary%(%): gap_info return r; %} +## Returns the number bytes that a value occupies in memory. +## +## v: The value +## +## Returns: The number of bytes that *v* occupies. function val_size%(v: any%): count %{ return new Val(v->MemoryAllocation(), TYPE_COUNT); %} +## Generates a table of the size of all global variables. The table index is +## the variable name and the value the variable size in bytes. +## +## Returns: A table that maps variable names to their sizes. +## +## .. bro:see:: global_ids function global_sizes%(%): var_sizes %{ TableVal* sizes = new TableVal(var_sizes); @@ -2316,6 +2345,14 @@ function global_sizes%(%): var_sizes return sizes; %} +## Generates a table with information about all global identifiers. The table +## value is a record containing the type name of the identifier, whether it is +## exported, a constant, an enum constant, redefinable, and its value (if it +## has one). +## +## Returns: A table that maps identifier names to information about them. +## +## .. bro:see:: global_sizes function global_ids%(%): id_table %{ TableVal* ids = new TableVal(id_table); @@ -2350,6 +2387,13 @@ function global_ids%(%): id_table return ids; %} +## Generates meta data about a record fields. The returned information +## includes the field name, whether it is logged, its value (if it has one), +## and its default value (if specified). +## +## rec: The record to inspect. +## +## Returns: A table that describes the fields of a record. function record_fields%(rec: any%): record_field_table %{ TableVal* fields = new TableVal(record_field_table); @@ -2392,7 +2436,8 @@ function record_fields%(rec: any%): record_field_table #include "Anon.h" %%} -# Preserve prefix as original one in anonymization. +## Preserve prefix as original one in anonymization. +## .. todo: Currently broken. function preserve_prefix%(a: addr, width: count%): any %{ AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50]; @@ -2412,6 +2457,7 @@ function preserve_prefix%(a: addr, width: count%): any return 0; %} +## .. todo: Currently broken. function preserve_subnet%(a: subnet%): any %{ DEBUG_MSG("%s/%d\n", dotted_addr(a->AsAddr()), a->Width()); @@ -2431,7 +2477,8 @@ function preserve_subnet%(a: subnet%): any return 0; %} -# Anonymize given IP address. +## Anonymize an IP address. +## .. todo: Currently broken. function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr %{ int anon_class = cl->InternalInt(); @@ -2486,6 +2533,17 @@ static void hmac_md5_val(val_list& vlist, unsigned char digest[16]) } %%} +## Computes the MD5 hash value of the provided list of arguments. +## +## Returns: The MD5 hash value of the concatenated arguments. +## +## .. bro:see:: md5_hmac md5_hash_init md5_hash_update md5_hash_finish +## +## .. note:: +## +## This function performs a one-shot computation of its arguments. +## For incremental hash computation, see :bro:id:`md5_hash_init` and +## friends. function md5_hash%(...%): string %{ unsigned char digest[16]; @@ -2493,6 +2551,13 @@ function md5_hash%(...%): string return new StringVal(md5_digest_print(digest)); %} +## Computes an HMAC-MD5 hash value of the provided list of arguments. The HMAC +## secret key is generated from available entropy when Bro starts up, or it can +## be specified for repeatability using the ``-K`` command line flag. +## +## Returns: The HMAC-MD5 hash value of the concatenated arguments. +## +## .. bro:see:: md5_hash md5_hash_init md5_hash_update md5_hash_finish function md5_hmac%(...%): string %{ unsigned char digest[16]; @@ -2513,6 +2578,20 @@ BroString* convert_index_to_string(Val* index) } %%} +## Initializes MD5 state to enable incremental hash computation. After +## initializing the MD5 state with this function, you can feed data to +## :bro:id:`md5_hash_update` and finally need to call :bro:id:`md5_hash_finish` +## to finish the computation and get the final hash value. +## +## For example, when computing incremental MD5 values of transferred files in +## multiple concurrent HTTP connections, one would call ``md5_hash_init(c$id)`` +## once before invoking ``md5_hash_update(c$id, some_more_data)`` in the +## :bro:id:`http_entity_data` event handler. When all data has arrived, a call +## to :bro:id:`md5_hash_finish` returns the final hash value. +## +## index: The unique identifier to associate with this hash computation. +## +## .. bro:see:: md5_hash md5_hmac md5_hash_update md5_hash_finish function md5_hash_init%(index: any%): bool %{ BroString* s = convert_index_to_string(index); @@ -2530,6 +2609,15 @@ function md5_hash_init%(index: any%): bool return new Val(status, TYPE_BOOL); %} +## Update the MD5 value associated with a given index. It is required to +## call :bro:id:`md5_hash_init(index)` once before calling this +## function. +## +## index: The unique identifier to associate with this hash computation. +## +## data: The data to add to the hash computation. +## +## .. bro:see:: md5_hash md5_hmac md5_hash_init md5_hash_finish function md5_hash_update%(index: any, data: string%): bool %{ BroString* s = convert_index_to_string(index); @@ -2545,6 +2633,13 @@ function md5_hash_update%(index: any, data: string%): bool return new Val(status, TYPE_BOOL); %} +## Returns the final MD5 digest of an incremental hash computation. +## +## index: The unique identifier of this hash computation. +## +## Returns: The hash value associated with the computation at *index*. +## +## .. bro:see:: md5_hash md5_hmac md5_hash_init md5_hash_update function md5_hash_finish%(index: any%): string %{ BroString* s = convert_index_to_string(index); @@ -2564,7 +2659,18 @@ function md5_hash_finish%(index: any%): string return printable_digest; %} -# Wrappings for rand() and srand() +## Generates a random number. +## +## max: The maximum value the random number. +## +## Returns: a random positive integer in the interval *[0, max)*. +## +## .. bro:see:: srand +## +## .. note:: +## +## This function is a wrapper about the function ``rand`` provided by +## the OS. function rand%(max: count%): count %{ int result; @@ -2572,12 +2678,29 @@ function rand%(max: count%): count return new Val(result, TYPE_COUNT); %} +## Sets the seed for subsequent :bro:id:`rand` calls. +## +## seed: The seed for the PRNG. +## +## .. bro:see:: rand +## +## .. note:: +## +## This function is a wrapper about the function ``srand`` provided +## by the OS. function srand%(seed: count%): any %{ srand(seed); return 0; %} +## Decodes a Base64-encoded string. +## +## s: The Base64-encoded string. +## +## Returns: The decoded version of *s*. +## +## .. bro:see:: decode_base64_custom function decode_base64%(s: string%): string %{ BroString* t = decode_base64(s->AsString()); @@ -2590,6 +2713,17 @@ function decode_base64%(s: string%): string } %} +## Decodes a Base64-encoded string with a custom alphabet. +## +## s: The Base64-encoded string. +## +## a: The custom alphabet. The empty string indicates the default alphabet. The +## lengh of *a* must bt 64. For example, a custom alphabet could be +## ``"!#$%&/(),-.:;<>@[]^ `_{|}~abcdefghijklmnopqrstuvwxyz0123456789+?"``. +## +## Returns: The decoded version of *s*. +## +## .. bro:see:: decode_base64 function decode_base64_custom%(s: string, a: string%): string %{ BroString* t = decode_base64(s->AsString(), a->AsString()); @@ -2615,6 +2749,14 @@ typedef struct { } bro_uuid_t; %%} +## Converts a bytes representation of a UUID into its string form. For example, +## given a string of 16 bytes, it produces an output string in this format: +## ``550e8400-e29b-41d4-a716-446655440000``. +## See ``_. +## +## uuid: The 16 bytes of the UUID. +## +## Returns: The string representation of *uuid*. function uuid_to_string%(uuid: string%): string %{ if ( uuid->Len() != 16 ) @@ -2644,6 +2786,20 @@ function uuid_to_string%(uuid: string%): string # computed NFAs and DFAs cannot be cleanly deallocated (at least for now), # they can only be used at initialization time. +## Merges and compiles two regular expressions at initialization time. +## +## p1: The first pattern. +## +## p2: The second pattern. +## +## Returns: The compiled pattern of the concatentation of *p1* and *p2*. +## +## .. bro:see:: convert_for_pattern string_to_pattern +## +## .. note:: +## +## This function must be called at Bro startup time, e.g., in the event +## :bro:id:`bro_init`. function merge_pattern%(p1: pattern, p2: pattern%): pattern %{ if ( bro_start_network_time != 0.0 ) @@ -2682,6 +2838,17 @@ char* to_pat_str(int sn, const char* ss) } %%} +## Escapes a string so that it becomes a valid :bro:type:`pattern` and can be +## used with the :bro:id:`string_to_pattern`. Any character from the set +## ``^$-:"\/|*+?.(){}[]`` is prefixed with a ``\``. +## +## s: The string to escape. +## +## Returns: An escaped version of *s* that has the structure of a valid +## :bro:type:`pattern`. +## +## .. bro:see:: merge_pattern string_to_pattern +## function convert_for_pattern%(s: string%): string %{ char* t = to_pat_str(s->Len(), (const char*)(s->Bytes())); @@ -2690,6 +2857,22 @@ function convert_for_pattern%(s: string%): string return ret; %} +## Converts a :bro:type:`string` into a :bro:type:`pattern`. +## +## s: The string to convert. +## +## convert: If true, *s* is first passed through the function +## :bro:id:`convert_for_pattern` to escape special characters of +## patterns. +## +## Returns: *s* as :bro:type:`pattern`. +## +## .. bro:see:: convert_for_pattern merge_pattern +## +## .. note:: +## +## This function must be called at Bro startup time, e.g., in the event +## :bro:id:`bro_init`. function string_to_pattern%(s: string, convert: bool%): pattern %{ if ( bro_start_network_time != 0.0 ) @@ -2717,7 +2900,24 @@ function string_to_pattern%(s: string, convert: bool%): pattern return new PatternVal(re); %} -# Precompile a pcap filter. +## Precompiles a PCAP filter and binds it to a given identifier. +## +## id: The PCAP identifier to reference the filter *s* later on. +## +## s: The PCAP filter. See ``man tcpdump`` for valid expressions. +## +## Returns: True if *s* is valid and precompiles successfully. +## +## .. bro:see:: install_pcap_filter +## install_src_addr_filter +## install_src_net_filter +## uninstall_src_addr_filter +## uninstall_src_net_filter +## install_dst_addr_filter +## install_dst_net_filter +## uninstall_dst_addr_filter +## uninstall_dst_net_filter +## pcap_error function precompile_pcap_filter%(id: PcapFilterID, s: string%): bool %{ bool success = true; @@ -2738,7 +2938,24 @@ function precompile_pcap_filter%(id: PcapFilterID, s: string%): bool return new Val(success, TYPE_BOOL); %} -# Install precompiled pcap filter. +## Installs a PCAP filter that has been precompiled with +## :bro:id:`precompile_pcap_filter`. +## +## id: The PCAP filter id of a precompiled filter. +## +## Returns: True if the filter associated with *id* has been installed +## successfully. +## +## .. bro:see:: precompile_pcap_filter +## install_src_addr_filter +## install_src_net_filter +## uninstall_src_addr_filter +## uninstall_src_net_filter +## install_dst_addr_filter +## install_dst_net_filter +## uninstall_dst_addr_filter +## uninstall_dst_net_filter +## pcap_error function install_pcap_filter%(id: PcapFilterID%): bool %{ bool success = true; @@ -2754,7 +2971,20 @@ function install_pcap_filter%(id: PcapFilterID%): bool return new Val(success, TYPE_BOOL); %} -# If last pcap function failed, returns a descriptive error message +## Returns a string representation of the last PCAP error. +## +## Returns: A descriptive error message of the PCAP function that failed. +## +## .. bro:see:: precompile_pcap_filter +## install_pcap_filter +## install_src_addr_filter +## install_src_net_filter +## uninstall_src_addr_filter +## uninstall_src_net_filter +## install_dst_addr_filter +## install_dst_net_filter +## uninstall_dst_addr_filter +## uninstall_dst_net_filter function pcap_error%(%): string %{ loop_over_list(pkt_srcs, i) @@ -2767,69 +2997,248 @@ function pcap_error%(%): string return new StringVal("no error"); %} -# Install filter to drop packets from a source (addr/subnet) with a given -# probability (0.0-1.0) if none of the given TCP flags is set. +## Installs a filter to drop packets from a given IP source address with +## a certain probability if none of a given set of TCP flags are set. +## +## ip: The IP address to drop. +## +## tcp_flags: If none of these TCP flags are set, drop packets from *ip* with +## probability *prob*. +## +## prob: The probability [0.0, 1.0] used to drop packets from *ip*. +## +## Returns: True (unconditionally). +## +## .. bro:see:: precompile_pcap_filter +## install_pcap_filter +## install_src_net_filter +## uninstall_src_addr_filter +## uninstall_src_net_filter +## install_dst_addr_filter +## install_dst_net_filter +## uninstall_dst_addr_filter +## uninstall_dst_net_filter +## pcap_error +## +## .. todo:: The return value should be changed to any. function install_src_addr_filter%(ip: addr, tcp_flags: count, prob: double%) : bool %{ sessions->GetPacketFilter()->AddSrc(ip, tcp_flags, prob); return new Val(1, TYPE_BOOL); %} +## Installs a filter to drop packets originating from a given subnet with +## a certain probability if none of a given set of TCP flags are set. +## +## snet: The subnet to drop packets from. +## +## tcp_flags: If none of these TCP flags are set, drop packets from *snet* with +## probability *prob*. +## +## prob: The probability [0.0, 1.0] used to drop packets from *snet*. +## +## Returns: True (unconditionally). +## +## .. bro:see:: precompile_pcap_filter +## install_pcap_filter +## install_src_addr_filter +## uninstall_src_addr_filter +## uninstall_src_net_filter +## install_dst_addr_filter +## install_dst_net_filter +## uninstall_dst_addr_filter +## uninstall_dst_net_filter +## pcap_error +## +## .. todo:: The return value should be changed to any. function install_src_net_filter%(snet: subnet, tcp_flags: count, prob: double%) : bool %{ sessions->GetPacketFilter()->AddSrc(snet, tcp_flags, prob); return new Val(1, TYPE_BOOL); %} -# Remove the filter for the source. +## Removes a source address filter. +## +## ip: The IP address for which a source filter was previously installed. +## +## Returns: True on success. +## +## .. bro:see:: precompile_pcap_filter +## install_pcap_filter +## install_src_addr_filter +## install_src_net_filter +## uninstall_src_net_filter +## install_dst_addr_filter +## install_dst_net_filter +## uninstall_dst_addr_filter +## uninstall_dst_net_filter +## pcap_error function uninstall_src_addr_filter%(ip: addr%) : bool %{ return new Val(sessions->GetPacketFilter()->RemoveSrc(ip), TYPE_BOOL); %} +## Removes a source subnet filter. +## +## snet: The subnet for which a source filter was previously installed. +## +## Returns: True on success. +## +## .. bro:see:: precompile_pcap_filter +## install_pcap_filter +## install_src_addr_filter +## install_src_net_filter +## uninstall_src_addr_filter +## install_dst_addr_filter +## install_dst_net_filter +## uninstall_dst_addr_filter +## uninstall_dst_net_filter +## pcap_error function uninstall_src_net_filter%(snet: subnet%) : bool %{ return new Val(sessions->GetPacketFilter()->RemoveSrc(snet), TYPE_BOOL); %} -# Same for destination. +## Installs a filter to drop packets destined to a given IP address with +## a certain probability if none of a given set of TCP flags are set. +## +## ip: Drop packets to this IP address. +## +## tcp_flags: If none of these TCP flags are set, drop packets to *ip* with +## probability *prob*. +## +## prob: The probability [0.0, 1.0] used to drop packets to *ip*. +## +## Returns: True (unconditionally). +## +## .. bro:see:: precompile_pcap_filter +## install_pcap_filter +## install_src_addr_filter +## install_src_net_filter +## uninstall_src_addr_filter +## uninstall_src_net_filter +## install_dst_net_filter +## uninstall_dst_addr_filter +## uninstall_dst_net_filter +## pcap_error +## +## .. todo:: The return value should be changed to any. function install_dst_addr_filter%(ip: addr, tcp_flags: count, prob: double%) : bool %{ sessions->GetPacketFilter()->AddDst(ip, tcp_flags, prob); return new Val(1, TYPE_BOOL); %} +## Installs a filter to drop packets destined to a given subnet with +## a certain probability if none of a given set of TCP flags are set. +## +## snet: Drop packets to this subnet. +## +## tcp_flags: If none of these TCP flags are set, drop packets to *snet* with +## probability *prob*. +## +## prob: The probability [0.0, 1.0] used to drop packets to *snet*. +## +## Returns: True (unconditionally). +## +## .. bro:see:: precompile_pcap_filter +## install_pcap_filter +## install_src_addr_filter +## install_src_net_filter +## uninstall_src_addr_filter +## uninstall_src_net_filter +## install_dst_addr_filter +## uninstall_dst_addr_filter +## uninstall_dst_net_filter +## pcap_error +## +## .. todo:: The return value should be changed to any. function install_dst_net_filter%(snet: subnet, tcp_flags: count, prob: double%) : bool %{ sessions->GetPacketFilter()->AddDst(snet, tcp_flags, prob); return new Val(1, TYPE_BOOL); %} +## Removes a destination address filter. +## +## ip: The IP address for which a destination filter was previously installed. +## +## Returns: True on success. +## +## .. bro:see:: precompile_pcap_filter +## install_pcap_filter +## install_src_addr_filter +## install_src_net_filter +## uninstall_src_addr_filter +## uninstall_src_net_filter +## install_dst_addr_filter +## install_dst_net_filter +## uninstall_dst_net_filter +## pcap_error function uninstall_dst_addr_filter%(ip: addr%) : bool %{ return new Val(sessions->GetPacketFilter()->RemoveDst(ip), TYPE_BOOL); %} +## Removes a destination subnet filter. +## +## snet: The subnet for which a destination filter was previously installed. +## +## Returns: True on success. +## +## .. bro:see:: precompile_pcap_filter +## install_pcap_filter +## install_src_addr_filter +## install_src_net_filter +## uninstall_src_addr_filter +## uninstall_src_net_filter +## install_dst_addr_filter +## install_dst_net_filter +## uninstall_dst_addr_filter +## pcap_error function uninstall_dst_net_filter%(snet: subnet%) : bool %{ return new Val(sessions->GetPacketFilter()->RemoveDst(snet), TYPE_BOOL); %} +## Flushes in-memory state tagged with the :bro:attr:`&persistence` attribute +## to disk. The function writes the state to the file ``.state/state.bst`` in +## the directory where Bro was started. +## +## Returns: True on success. +## +## .. bro:see:: rescan_state function checkpoint_state%(%) : bool %{ return new Val(persistence_serializer->WriteState(true), TYPE_BOOL); %} +## Deprecated. Will be removed. function dump_config%(%) : bool %{ return new Val(persistence_serializer->WriteConfig(true), TYPE_BOOL); %} +## Reads persistent state from the \texttt{.state} directory and populates the +## in-memory data structures accordingly. This function is the dual to +## :bro:id:`checkpoint_state`. +## +## Returns: True on success. +## +## .. bro:see:: checkpoint_state function rescan_state%(%) : bool %{ return new Val(persistence_serializer->ReadAll(false, true), TYPE_BOOL); %} +## Writes the binary event stream generated by the core to a given file. +## Use the ``-x `` command line switch to replay saved events. +## +## filename: The name of the file which stores the events. +## +## Returns: True if opening the target file succeeds. +## +## .. bro:see:: capture_state_updates function capture_events%(filename: string%) : bool %{ if ( ! event_serializer ) @@ -2841,6 +3250,14 @@ function capture_events%(filename: string%) : bool (const char*) filename->CheckString()), TYPE_BOOL); %} +## Writes state updates generated by :bro:attr:`&synchronized` variables to a +## file. +## +## filename: The name of the file which stores the state updates. +## +## Returns: True if opening the target file succeeds. +## +## .. bro:see:: capture_events function capture_state_updates%(filename: string%) : bool %{ if ( ! state_serializer ) @@ -2852,6 +3269,33 @@ function capture_state_updates%(filename: string%) : bool (const char*) filename->CheckString()), TYPE_BOOL); %} +## Establishes a connection to a remote Bro or Broccoli instance. +## +## ip: The IP address of the remote peer. +## +## port: The port of the remote peer. +## +## our_class: If an non-empty string, the remote (listening) peer checks it +## against its class name in its peer table and terminates the +## connection if they don't match. +## +## retry: If the connection fails, try to reconnect with the peer after this +## time interval. +## +## ssl: If true, uses SSL to encrypt the session. +## +## Returns: A locally unique ID of the new peer. +## +## .. bro:see:: disconnect +## listen +## request_remote_events +## request_remote_sync +## request_remote_logs +## request_remote_events +## set_compression_accept_state +## set_compression_level +## send_state +## send_id function connect%(ip: addr, p: port, our_class: string, retry: interval, ssl: bool%) : count %{ return new Val(uint32(remote_serializer->Connect(ip, p->Port(), @@ -2859,12 +3303,31 @@ function connect%(ip: addr, p: port, our_class: string, retry: interval, ssl: bo TYPE_COUNT); %} +## Terminate the connection with a peer. +## +## p: The peer ID return from :bro:id:`connect`. +## +## Returns: True on success. +## +## .. bro:see:: connect listen function disconnect%(p: event_peer%) : bool %{ RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount(); return new Val(remote_serializer->CloseConnection(id), TYPE_BOOL); %} +## Subscribes to all events from a remote peer whose names match a given +## pattern. +## +## p: The peer ID return from :bro:id:`connect`. +## +## handlers: The pattern describing the events to request from peer *p*. +## +## Returns: True on success. +## +## .. bro:see:: request_remote_sync +## request_remote_logs +## set_accept_state function request_remote_events%(p: event_peer, handlers: pattern%) : bool %{ RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount(); @@ -2872,18 +3335,47 @@ function request_remote_events%(p: event_peer, handlers: pattern%) : bool TYPE_BOOL); %} +## Requests synchronization of IDs with a remote peer. +## +## p: The peer ID return from :bro:id:`connect`. +## +## auth: If true, the local instance considers its current state authoritative +## and sends it to *p* right after the handshake. +## +## Returns: True on success. +## +## .. bro:see:: request_remote_events +## request_remote_logs +## set_accept_state function request_remote_sync%(p: event_peer, auth: bool%) : bool %{ RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount(); return new Val(remote_serializer->RequestSync(id, auth), TYPE_BOOL); %} +## Requests logs from a remote peer. +## +## p: The peer ID return from :bro:id:`connect`. +## +## Returns: True on success. +## +## .. bro:see:: request_remote_events +## request_remote_sync function request_remote_logs%(p: event_peer%) : bool %{ RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount(); return new Val(remote_serializer->RequestLogs(id), TYPE_BOOL); %} +## Sets a boolean flag whether Bro accepts state from a remote peer. +## +## p: The peer ID return from :bro:id:`connect`. +## +## Returns: True on success. +## +## .. bro:see:: request_remote_events +## request_remote_sync +## set_compression_level function set_accept_state%(p: event_peer, accept: bool%) : bool %{ RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount(); @@ -2891,6 +3383,16 @@ function set_accept_state%(p: event_peer, accept: bool%) : bool TYPE_BOOL); %} +## Sets the compression level of the session with a remote peer. +## +## p: The peer ID return from :bro:id:`connect`. +## +## level: Allowed values are in the range *[0, 9]*, where 0 is the default and +## means no compression. +## +## Returns: True on success. +## +## .. bro:see:: set_accept_state function set_compression_level%(p: event_peer, level: count%) : bool %{ RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount(); @@ -2898,24 +3400,53 @@ function set_compression_level%(p: event_peer, level: count%) : bool TYPE_BOOL); %} +## Listens on address a given IP address and port for remote connections. +## +## ip: The IP address to bind to. +## +## p: The TCP port to listen to. +## +## ssl: If true, Bro uses SSL to encrypt the session. +## +## Returns: True on success. +## +## .. bro:see:: connect disconnect function listen%(ip: addr, p: port, ssl: bool %) : bool %{ return new Val(remote_serializer->Listen(ip, p->Port(), ssl), TYPE_BOOL); %} +## Checks whether the last raised event came from a remote peer. +## +## Returns: True if the last raised event came from a remote peer. function is_remote_event%(%) : bool %{ return new Val(mgr.CurrentSource() != SOURCE_LOCAL, TYPE_BOOL); %} +## Sends all persistent state to a remote peer. +## +## p: The peer ID return from :bro:id:`connect`. +## +## Returns: True on success. +## +## .. bro:see:: send_id send_ping send_current_packet send_capture_filter function send_state%(p: event_peer%) : bool %{ RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount(); return new Val(persistence_serializer->SendState(id, true), TYPE_BOOL); %} -# Send the value of the given global ID to the peer (which might -# then install it locally). +## Sends a global identifier to a remote peer, which them might install it +## locally. +## +## p: The peer ID return from :bro:id:`connect`. +## +## id: The identifier to send. +## +## Returns: True on success. +## +## .. bro:see:: send_state send_ping send_current_packet send_capture_filter function send_id%(p: event_peer, id: string%) : bool %{ RemoteSerializer::PeerID pid = p->AsRecordVal()->Lookup(0)->AsCount(); @@ -2931,24 +3462,52 @@ function send_id%(p: event_peer, id: string%) : bool return new Val(remote_serializer->SendID(&info, pid, *i), TYPE_BOOL); %} -# Gracely shut down communication. +## Gracefully finishes communication by first making sure that all remaining +## data from parent and child has been sent out. +## +## Returns: True if the termination process has been started successfully. function terminate_communication%(%) : bool %{ return new Val(remote_serializer->Terminate(), TYPE_BOOL); %} +## Signals a remote peer that the local Bro instance finished the initial +## handshake. +## +## p: The peer ID return from :bro:id:`connect`. +## +## Returns: True on success. function complete_handshake%(p: event_peer%) : bool %{ RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount(); return new Val(remote_serializer->CompleteHandshake(id), TYPE_BOOL); %} +## Sends a ping event to a remote peer. In combination with an event handler +## for :bro:id:`remote_pong`, this function can be used to measure latency +## between two peers. +## +## p: The peer ID return from :bro:id:`connect`. +## +## seq: A sequence number (also included by :bro:id:`remote_pong`). +## +## Returns: True if sending the ping succeeds. +## +## .. bro:see:: send_state send_id send_current_packet send_capture_filter function send_ping%(p: event_peer, seq: count%) : bool %{ RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount(); return new Val(remote_serializer->SendPing(id, seq), TYPE_BOOL); %} +## Sends the currently processed packet to a remote peer. +## +## p: The peer ID return from :bro:id:`connect`. +## +## Returns: True if sending the packet succeeds. +## +## .. bro:see:: send_id send_state send_ping send_capture_filter +## dump_packet dump_current_packet get_current_packet function send_current_packet%(p: event_peer%) : bool %{ Packet pkt(""); @@ -2967,6 +3526,16 @@ function send_current_packet%(p: event_peer%) : bool return new Val(remote_serializer->SendPacket(&info, id, pkt), TYPE_BOOL); %} +## Enables detailed collections of statistics about CPU/memory usage, +## connections, TCP states/reassembler, DNS lookups, timers, and script-level +## state. The script variable :bro:id:`profiling_file` holds the name of the +## file. +## +## .. bro:see:: net_stats +## resource_usage +## get_matcher_stats +## dump_rule_stats +## get_gap_summary function do_profiling%(%) : bool %{ if ( profiling_logger ) @@ -2975,6 +3544,11 @@ function do_profiling%(%) : bool return new Val(1, TYPE_BOOL); %} +## Returns the peer who generated the last event. +## +## Returns: The ID of the peer who genereated the last event. +## +## .. bro:see:: get_local_event_peer function get_event_peer%(%) : event_peer %{ SourceID src = mgr.CurrentSource(); @@ -3001,6 +3575,11 @@ function get_event_peer%(%) : event_peer return v; %} +## Returns the local peer ID. +## +## Returns: The peer ID of the local Bro instance. +## +## .. bro:see:: get_event_peer function get_local_event_peer%(%) : event_peer %{ RecordVal* p = mgr.GetLocalPeerVal(); @@ -3009,18 +3588,33 @@ function get_local_event_peer%(%) : event_peer %} +## Sends a capture filter to a remote peer. +## +## p: The peer ID return from :bro:id:`connect`. +## +## s: The capture filter. +## +## Returns: True if sending the packet succeeds. +## +## .. bro:see:: send_id send_state send_ping send_current_packet function send_capture_filter%(p: event_peer, s: string%) : bool %{ RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount(); return new Val(remote_serializer->SendCaptureFilter(id, s->CheckString()), TYPE_BOOL); %} +## Deprecated. Will be removed. function make_connection_persistent%(c: connection%) : bool %{ c->MakePersistent(); return new Val(1, TYPE_BOOL); %} +## Checks whether a given IP address belongs to a local interface. +## +## ip: The IP address to check. +## +## Returns: True if *ip* belongs to a local interface. function is_local_interface%(ip: addr%) : bool %{ static uint32* addrs; @@ -3065,6 +3659,13 @@ function is_local_interface%(ip: addr%) : bool return new Val(0, TYPE_BOOL); %} +## Writes the current packet to a file. +## +## file_name: The name of the file to write the packet to. +## +## Returns: True on success. +## +## .. bro:see:: dump_packet get_current_packet send_current_packet function dump_current_packet%(file_name: string%) : bool %{ const struct pcap_pkthdr* hdr; @@ -3083,6 +3684,12 @@ function dump_current_packet%(file_name: string%) : bool return new Val(! addl_pkt_dumper->IsError(), TYPE_BOOL); %} +## Returns the currently processed PCAP packet. +## +## Returns: The currently processed packet, which is as a record +## containing the timestamp, ``snaplen,'' and packet data. +## +## .. bro:see:: dump_current_packet dump_packet send_current_packet function get_current_packet%(%) : pcap_packet %{ const struct pcap_pkthdr* hdr; @@ -3109,6 +3716,15 @@ function get_current_packet%(%) : pcap_packet return pkt; %} +## Writes a given packet to a file. +## +## pkt: The PCAP packet. +## +## file_name: The name of the file to write *pkt* to. +## +## Returns: True on success +## +## .. bro:see:: get_current_packet dump_current_packet send_current_packet function dump_packet%(pkt: pcap_packet, file_name: string%) : bool %{ struct pcap_pkthdr hdr; @@ -3128,9 +3744,13 @@ function dump_packet%(pkt: pcap_packet, file_name: string%) : bool return new Val(addl_pkt_dumper->IsError(), TYPE_BOOL); %} -# The return value is the old size of the vector. -# ### Need better type checking on the argument here and in subsequent -# vector builtins -- probably need to update "bif" code generator. +## Resizes a vector. +## +## aggr: The vector instance. +## +## newsize: The new size of *aggr*. +## +## Returns: The old size of *aggr* and 0 if *aggr* is not a :bro:type:`vector`. function resize%(aggr: any, newsize: count%) : count %{ if ( aggr->Type()->Tag() != TYPE_VECTOR ) @@ -3142,7 +3762,14 @@ function resize%(aggr: any, newsize: count%) : count return new Val(aggr->AsVectorVal()->Resize(newsize), TYPE_COUNT); %} -# Returns true if any element is T. +## Tests whether a boolean vector (``vector of bool``) has *any* true +## element. +## +## v: The boolean vector instance. +## +## Returns: True if any element in *v* is true. +## +## :bro:see:: all_set function any_set%(v: any%) : bool %{ if ( v->Type()->Tag() != TYPE_VECTOR || @@ -3160,7 +3787,18 @@ function any_set%(v: any%) : bool return new Val(false, TYPE_BOOL); %} -# Returns true if all elements are T (missing counts as F). +## Tests whether *all* elements of a boolean vector (``vector of bool``) are +## true. +## +## v: The boolean vector instance. +## +## Returns: True iff all elements in *v* are true. +## +## :bro:see:: any_set +## +## .. note:: +## +## Missing elements count as false. function all_set%(v: any%) : bool %{ if ( v->Type()->Tag() != TYPE_VECTOR || @@ -3178,9 +3816,6 @@ function all_set%(v: any%) : bool return new Val(true, TYPE_BOOL); %} -# sort() takes a vector and comparison function on the elements and -# sorts the vector in place. It returns the original vector. - %%{ static Func* sort_function_comp = 0; static Val** index_map = 0; // used for indirect sorting to support order() @@ -3231,6 +3866,16 @@ bool indirect_int_sort_function(int a, int b) } %%} +## Sorts a vector in place. The second argument is a comparison function that +## takes two arguments: if the vector type is \verb|vector of T|, then the +## comparison function must be ``function(a: T, b: T): bool``, which returns +## ``a < b`` for some type-specific notion of the less-than operator. +## +## v: The vector instance to sort. +## +## Returns: The original vector. +## +## .. bro:see:: order function sort%(v: any, ...%) : any %{ v->Ref(); // we always return v @@ -3284,6 +3929,15 @@ function sort%(v: any, ...%) : any return v; %} +## Returns the order of the elements in a vector according to some +## comparison function. See :bro:id:`sort` for details about the comparison +## function. +## +## v: The vector whose order to compute. +## +## Returns: A ``vector of count`` with the indices of the ordered elements. +## +## .. bro:see:: sort function order%(v: any, ...%) : index_vec %{ VectorVal* result_v = @@ -3383,6 +4037,7 @@ char* port_to_string(PortVal* port) %%} +## Deprecated. Will be removed. function generate_idmef%(src_ip: addr, src_port: port, dst_ip: addr, dst_port: port%) : bool %{ @@ -3420,6 +4075,20 @@ function generate_idmef%(src_ip: addr, src_port: port, #endif %} +## Write rule matcher statistics (DFA states, transitions, memory usage, cache +## hits/misses) to a file. +## +## f: The file to write to. +## +## Returns: True (unconditionally). +## +## .. bro:see:: do_profiling +## resource_usage +## get_matcher_stats +## net_stats +## get_gap_summary +## +## .. todo:: The return value should be changed to any or check appropriately. function dump_rule_stats%(f: file%): bool %{ if ( rule_matcher ) @@ -3428,11 +4097,24 @@ function dump_rule_stats%(f: file%): bool return new Val(1, TYPE_BOOL); %} +## Checks wheter Bro is terminating. +## +## Returns: True if Bro is in the process of shutting down. +## +## .. bro:see: terminate function bro_is_terminating%(%): bool %{ return new Val(terminating, TYPE_BOOL); %} +## Rotates a file. +## +## f: An open file handle. +## +## Returns: Rotations statistics which include the original file name, the name +## after the rotation, and the time when *f* was opened/closed. +## +## .. bro:see:: rotate_file_by_name calc_next_rotate function rotate_file%(f: file%): rotate_info %{ RecordVal* info = f->Rotate(); @@ -3449,6 +4131,14 @@ function rotate_file%(f: file%): rotate_info return info; %} +## Rotates a file identified by its name. +## +## f: The name of the file to rotate +## +## Returns: Rotations statistics which include the original file name, the name +## after the rotation, and the time when *f* was opened/closed. +## +## .. bro:see:: rotate_file calc_next_rotate function rotate_file_by_name%(f: string%): rotate_info %{ RecordVal* info = new RecordVal(rotate_info); @@ -3495,6 +4185,14 @@ function rotate_file_by_name%(f: string%): rotate_info return info; %} +## Calculates the duration until the next time a file is to be rotated, based +## on a given rotate interval. +## +## i: The rotate interval to base the calculation on. +## +## Returns: The duration until the next file rotation time. +## +## .. bro:see:: rotate_file rotate_file_by_name function calc_next_rotate%(i: interval%) : interval %{ const char* base_time = log_rotate_base_time ? @@ -3502,6 +4200,11 @@ function calc_next_rotate%(i: interval%) : interval return new Val(calc_next_rotate(i, base_time), TYPE_INTERVAL); %} +## Returns the size of a given file. +## +## f: The name of the file whose size to lookup. +## +## Returns: The size of *f* in bytes. function file_size%(f: string%) : double %{ struct stat s; @@ -3512,6 +4215,13 @@ function file_size%(f: string%) : double return new Val(double(s.st_size), TYPE_DOUBLE); %} +## Formats a given time value according to a format string. +## +## fmt: The format string. See ``man strftime`` for the syntax. +## +## d: The time value. +## +## Returns: The time *d* formatted according to *fmt*. function strftime%(fmt: string, d: time%) : string %{ static char buffer[128]; @@ -3524,6 +4234,7 @@ function strftime%(fmt: string, d: time%) : string return new StringVal(buffer); %} +## .. todo:: document function match_signatures%(c: connection, pattern_type: int, s: string, bol: bool, eol: bool, from_orig: bool, clear: bool%) : bool @@ -3537,6 +4248,9 @@ function match_signatures%(c: connection, pattern_type: int, s: string, return new Val(1, TYPE_BOOL); %} +## Returns the hostname of the machine Bro runs on. +## +## Returns: The hostname of the machine Bro runs on. function gethostname%(%) : string %{ char buffer[MAXHOSTNAMELEN]; @@ -3612,8 +4326,15 @@ private: }; %%} -# These two functions issue DNS lookups asynchronously and delay the -# function result. Therefore, they can only be called inside a when-condition. +## Issues an asynchronous reverse DNS lookup and delays the function result. +## This function can therefore only be called inside a ``when`` condition, +## e.g., ``when ( local host = lookup_addr(10.0.0.1) ) { f(host); }``. +## +## host: The IP address to lookup. +## +## Returns: The DNS name of *host*. +## +## .. bro:see:: lookup_hostname function lookup_addr%(host: addr%) : string %{ // FIXME: It should be easy to adapt the function to synchronous @@ -3655,6 +4376,15 @@ function lookup_addr%(host: addr%) : string return 0; %} +## Issues an asynchronous DNS lookup and delays the function result. +## This function can therefore only be called inside a ``when`` condition, +## e.g., ``when ( local h = lookup_hostname("www.bro-ids.org") ) { f(h); }``. +## +## host: The hostname to lookup. +## +## Returns: A set of DNS A records associated with *host*. +## +## .. bro:see:: lookup_addr function lookup_hostname%(host: string%) : addr_set %{ // FIXME: Is should be easy to adapt the function to synchronous @@ -3675,14 +4405,20 @@ function lookup_hostname%(host: string%) : addr_set return 0; %} -# Stop Bro's packet processing. +## Stops Bro's packet processing. This function is used to synchronize +## distributed trace processing with communication enabled +## (*pseudo-realtime* mode). +## +## .. bro:see: continue_processing suspend_state_updates resume_state_updates function suspend_processing%(%) : any %{ net_suspend_processing(); return 0; %} -# Resume Bro's packet processing. +## Resumes Bro's packet processing. +## +## .. bro:see: suspend_processing suspend_state_updates resume_state_updates function continue_processing%(%) : any %{ net_continue_processing(); @@ -3693,7 +4429,25 @@ function continue_processing%(%) : any #include "DPM.h" %%} -# Schedule analyzer for a future connection. +## Schedules an analyzer for a future connection from a given IP address and +## port. The function ignores the scheduling request if the connection did +## not occur within the specified time interval. +## +## orig: The IP address originating a connection in the future. +## +## resp: The IP address responding to a connection from *orig*. +## +## resp_p: The destination port at *resp*. +## +## analyzer: The analyzer ID. +## +## tout: The timeout interval after which to ignore the scheduling request. +## +## Returns: True (unconditionally). +## +## .. bro:see:: disable_analyzer analyzer_name +## +## .. todo:: The return value should be changed to any. function expect_connection%(orig: addr, resp: addr, resp_p: port, analyzer: count, tout: interval%) : bool %{ @@ -3702,8 +4456,17 @@ function expect_connection%(orig: addr, resp: addr, resp_p: port, return new Val(1, TYPE_BOOL); %} -# Disables the analyzer which raised the current event (if the analyzer -# belongs to the given connection). +## Disables the analyzer which raised the current event (if the analyzer +## belongs to the given connection). +## +## cid: The connection identifier. +## +## aid: The analyzer ID. +## +## Returns: True if the connection identified by *cid* exists and has analyzer +## *aid*. +## +## .. bro:see:: expect_connection analyzer_name function disable_analyzer%(cid: conn_id, aid: count%) : bool %{ Connection* c = sessions->FindConnection(cid); @@ -3724,12 +4487,24 @@ function disable_analyzer%(cid: conn_id, aid: count%) : bool return new Val(1, TYPE_BOOL); %} -# Translate analyzer type to an ASCII string. +## Translate an analyzer type to an ASCII string. +## +## aid: The analyzer ID. +## +## Returns: The analyzer *aid* as string. +## +## .. bro:see:: expect_connection disable_analyzer current_analyzer function analyzer_name%(aid: count%) : string %{ return new StringVal(Analyzer::GetTagName((AnalyzerTag::Tag) aid)); %} +## Returns the value of a global identifier. +## +## id: The global identifier. +## +## Returns the value of *id*. If *id* does not describe a valid identifier, the +## function returns the string ``""`` or ``""``. function lookup_ID%(id: string%) : any %{ ID* i = global_scope()->Lookup(id->CheckString()); @@ -3742,7 +4517,9 @@ function lookup_ID%(id: string%) : any return i->ID_Val()->Ref(); %} -# Stop propagating &synchronized accesses. +## Stops propagating :bro:attr:`&synchronized` accesses. +## +## .. bro:see: suspend_processing continue_processing resume_state_updates function suspend_state_updates%(%) : any %{ if ( remote_serializer ) @@ -3750,7 +4527,9 @@ function suspend_state_updates%(%) : any return 0; %} -# Resume propagating &synchronized accesses. +## Resumes propagating :bro:attr:`&synchronized` accesses. +## +## .. bro:see: suspend_processing continue_processing suspend_state_updates function resume_state_updates%(%) : any %{ if ( remote_serializer ) @@ -3758,13 +4537,18 @@ function resume_state_updates%(%) : any return 0; %} -# Return ID of analyzer which raised current event, or 0 if none. +## Returns the ID of the analyzer which raised the current event. +## +## Returns: The ID of the analyzer which raised hte current event, or 0 if +## none. function current_analyzer%(%) : count %{ return new Val(mgr.CurrentAnalyzer(), TYPE_COUNT); %} -# Returns Bro's process id. +## Returns Bro's process ID. +## +## Returns: Bro's process ID. function getpid%(%) : count %{ return new Val(getpid(), TYPE_COUNT); @@ -3773,6 +4557,10 @@ function getpid%(%) : count #include %%} + +## Send a string to syslog. +## +## s: The string to log via syslog function syslog%(s: string%): any %{ reporter->Syslog("%s", s->CheckString()); @@ -3801,7 +4589,14 @@ static GeoIP* open_geoip_db(GeoIPDBTypes type) #endif %%} -# Return a record with the city, region, and country of an IPv4 address. +## Performs a geo-lookup of an IP address. +## Requires Bro to be built with ``libgeoip``. +## +## a: The IP address to lookup. +## +## Returns: A record with country, region, city, latitude, and longitude. +## +## .. bro:see:: lookup_asn function lookup_location%(a: addr%) : geo_location %{ RecordVal* location = new RecordVal(geo_location); @@ -3929,6 +4724,13 @@ function lookup_location%(a: addr%) : geo_location return location; %} +## Performs an AS lookup of an IP address. +## +## a: The IP address to lookup. +## +## Returns: The number of the AS that contains *a*. +## +## .. bro:see:: lookup_location function lookup_asn%(a: addr%) : count %{ #ifdef USE_GEOIP @@ -3993,20 +4795,41 @@ function lookup_asn%(a: addr%) : count return new Val(0, TYPE_COUNT); %} -# Returns true if connection has been received externally. +## Determines whether *c* has been received externally. For example, +## Broccoli or the Time Machine can send packets to Bro via a mechanism that +## one step lower than sending events. This function checks whether the packets +## of a connection stem from one of these external *packet sources*. +## +## c: The connection to test. +## +## Returns: True if *c* has been received externally. function is_external_connection%(c: connection%) : bool %{ return new Val(c && c->IsExternal(), TYPE_BOOL); %} -# Function equivalent of the &disable_print_hook attribute. +## Disables sending :bro:id:`print_hook` events to remote peers for a given +## file. This function is equivalent to :bro:attr:`&disable_print_hook`. In a +## distributed setup, communicating Bro instances generate the event +## :bro:id:`print_hook` for each print statement and send it to the remote +## side. When disabled for a particular file, these events will not be +## propagated to other peers. +## +## f: The file to disable :bro:id:`print_hook` events for. +## +## .. bro:see:: enable_raw_output function disable_print_hook%(f: file%): any %{ f->DisablePrintHook(); return 0; %} -# Function equivalent of the &raw_output attribute. +## Prevents escaping of non-ASCII character when writing to a file. +## This function is equivalent to :bro:attr:`&disable_print_hook`. +## +## f: The file to disable raw output for. +## +## .. bro:see:: disable_print_hook function enable_raw_output%(f: file%): any %{ f->EnableRawOutput(); @@ -4019,6 +4842,14 @@ extern "C" { } %%} +## Determines the MIME type of a piece of data using ``libmagic``. +## +## data: The data to find the MIME type for. +## +## return_mime: If true, the function returns a short MIME type string (e.g., +## ``text/plain`` instead of a more elaborate textual description. +## +## Returns: The MIME type of *data*. function identify_data%(data: string, return_mime: bool%): string %{ const char* descr = ""; @@ -4052,12 +4883,25 @@ function identify_data%(data: string, return_mime: bool%): string return new StringVal(descr); %} +## Enables all event handlers in a given group. One can tag event handlers with +## the :bro:attr:`&group` attribute to logically group them together, e.g, +## `` event foo() &group="bar"``. This function enables all event handlers that +## belong to such a group. +## +## group: The group. +## +## .. bro:see:: disable_event_group function enable_event_group%(group: string%) : any %{ event_registry->EnableGroup(group->CheckString(), true); return 0; %} +## Disables all event handlers in a given group. +## +## group: The group. +## +## .. bro:see:: enable_event_group function disable_event_group%(group: string%) : any %{ event_registry->EnableGroup(group->CheckString(), false); @@ -4070,6 +4914,46 @@ function disable_event_group%(group: string%) : any static map entropy_states; %%} +## Performs an `entropy `_ test on the given +## data. +## +## data: The data to compute the entropy for. +## +## Returns: The result of the entropy test, which contains the following +## fields. +## +## - ``entropy``: The information density expressed as a number of +## bits per character. +## +## - ``chi_square``: The chi-square test value expressed as an +## absolute number and a percentage which indicates how +## frequently a truly random sequence would exceed the value +## calculated, i.e., the degree to which the sequence tested is +## suspected of being non-random. +## +## If the percentage is greater than 99% or less than 1%, the +## sequence is almost certainly not random. If the percentage is +## between 99% and 95% or between 1% and 5%, the sequence is +## suspect. Percentages between 90\% and 95\% and 5\% and 10\% +## indicate the sequence is "almost suspect." +## +## - ``mean``: The arithmetic mean of all the bytes. If the data +## are close to random, it should be around 127.5. +## +## - ``monte_carlo_pi``: Each successive sequence of six bytes is +## used as 24-bit *x* and *y* coordinates within a square. If +## the distance of the randomly-generated point is less than the +## radius of a circle inscribed within the square, the six-byte +## sequence is considered a "hit." The percentage of hits can +## be used to calculate the value of pi. For very large streams +## the value will approach the correct value of pi if the +## sequence is close to random. +## +## - ``serial_correlation``: This quantity measures the extent to +## which each byte in the file depends upon the previous byte. +## For random sequences this value will be close to zero. +## +## .. bro:see:: entropy_test_init entropy_test_add entropy_test_finish function find_entropy%(data: string%): entropy_test_result %{ double montepi, scc, ent, mean, chisq; @@ -4089,6 +4973,13 @@ function find_entropy%(data: string%): entropy_test_result return ent_result; %} +## Initializes data structures for incremental entropy calculation. +## +## index: An arbitrary unique value per distinct computation. +## +## Returns: True on success. +## +## .. bro:see:: find_entropy entropy_test_add entropy_test_finish function entropy_test_init%(index: any%): bool %{ BroString* s = convert_index_to_string(index); @@ -4104,6 +4995,17 @@ function entropy_test_init%(index: any%): bool return new Val(status, TYPE_BOOL); %} +## Adds data to an incremental entropy calculation. Before using this function, +## one needs to invoke :bro:id:`entropy_test_init`. +## +## data: The data to add to the entropy calculation. +## +## index: An arbitrary unique value that identifies a particular entropy +## computation. +## +## Returns: True on success. +## +## .. bro:see:: find_entropy entropy_test_add entropy_test_finish function entropy_test_add%(index: any, data: string%): bool %{ BroString* s = convert_index_to_string(index); @@ -4119,6 +5021,17 @@ function entropy_test_add%(index: any, data: string%): bool return new Val(status, TYPE_BOOL); %} +## Finishes an incremental entropy calculation. Before using this function, +## one needs to initialize the computation with :bro:id:`entropy_test_init` and +## add data to it via :bro:id:`entropy_test_add`. +## +## index: An arbitrary unique value that identifies a particular entropy +## computation. +## +## Returns: The result of the entropy test. See :bro:id:`find_entropy` for a +## description of the individual components. +## +## .. bro:see:: find_entropy entropy_test_init entropy_test_add function entropy_test_finish%(index: any%): entropy_test_result %{ BroString* s = convert_index_to_string(index); @@ -4144,6 +5057,7 @@ function entropy_test_finish%(index: any%): entropy_test_result return ent_result; %} +## Depreacated. Will be removed. function bro_has_ipv6%(%) : bool %{ #ifdef BROv6 @@ -4153,6 +5067,11 @@ function bro_has_ipv6%(%) : bool #endif %} +## Creates an identifier that is unique with high probability. +## +## prefix: A custom string prepended to the result. +## +## .. bro:see:: unique_id_from function unique_id%(prefix: string%) : string %{ char tmp[20]; @@ -4160,6 +5079,13 @@ function unique_id%(prefix: string%) : string return new StringVal(uitoa_n(uid, tmp, sizeof(tmp), 62, prefix->CheckString())); %} +## Creates an identifier that is unique with high probability. +## +## pool: A seed for determinism. +## +## prefix: A custom string prepended to the result. +## +## .. bro:see:: unique_id function unique_id_from%(pool: int, prefix: string%) : string %{ pool += UID_POOL_CUSTOM_SCRIPT; // Make sure we don't conflict with internal pool. @@ -4193,6 +5119,21 @@ X509* d2i_X509_(X509** px, const u_char** in, int len) %%} +## Verifies a X.509 certificate. +## +## der_cert: The X.509 certificate in DER format. +## +## cert_stack: Specifies a certificate chain to validate against, with index 0 +## typically being the root CA. Bro uses the Mozilla root CA list +## by default. +## +## root_certs: A list of additional root certificates that extends +## *cert_stack*. +## +## Returns: A status code of the verification which can be converted into an +## ASCII string via :bro:id:`x509_err2str`. +## +## .. bro:see:: x509_err2str function x509_verify%(der_cert: string, cert_stack: string_vec, root_certs: table_string_of_string%): count %{ X509_STORE* ctx = 0; @@ -4273,11 +5214,24 @@ function x509_verify%(der_cert: string, cert_stack: string_vec, root_certs: tabl return new Val((uint64) csc.error, TYPE_COUNT); %} +## Converts a X.509 certificate verification error code into an ASCII string. +## +## err_num: The error code. +## +## Returns: A string representation of *err_num*. +## +## .. bro:see:: x509_verify function x509_err2str%(err_num: count%): string %{ return new StringVal(X509_verify_cert_error_string(err_num)); %} +## Converts UNIX file permissions given by a mode to an ASCII string. +## +## mode: The permisssions, e.g., 644 or 755. +## +## Returns: A string representation of *mode* in the format +## ``rw[xsS]rw[xsS]rw[xtT]``. function NFS3::mode2string%(mode: count%): string %{ char str[12]; @@ -4364,11 +5318,16 @@ function NFS3::mode2string%(mode: count%): string return new StringVal(str); %} -## Opens a program with popen() and writes a given string to the returned +## Opens a program with ``popen`` and writes a given string to the returned ## stream to send it to the opened process's stdin. -## program: a string naming the program to execute -## to_write: a string to pipe to the opened program's process over stdin -## Returns: F if popen'ing the program failed, else T +## +## program: The program to execute. +## +## to_write: Data to pipe to the opened program's process via ``stdin``. +## +## Returns: True on success. +## +## .. bro:see:: system system_env function piped_exec%(program: string, to_write: string%): bool %{ const char* prog = program->CheckString(); @@ -4396,9 +5355,9 @@ function piped_exec%(program: string, to_write: string%): bool return new Val(1, TYPE_BOOL); %} -## Enables the communication system. Note that by default, -## communication is off until explicitly enabled, and all other calls -## to communication-related BiFs' will be ignored until done so. +## Enables the communication system. By default, the communication is off until +## explicitly enabled, and all other calls to communication-related functions +## will be ignored until done so. function enable_communication%(%): any %{ if ( bro_start_network_time != 0.0 ) @@ -4416,7 +5375,9 @@ function enable_communication%(%): any return 0; %} -## Returns the Bro version string +## Returns the Bro version string. +## +## Returns: Bro's version, e.g., 2.0-beta-47-debug. function bro_version%(%): string %{ return new StringVal(bro_version()); From 1b646c91193c71f1c6b54e80c77efd47676abb8a Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Sat, 10 Dec 2011 23:13:04 -0800 Subject: [PATCH 171/964] Reorder and group BiFs. --- src/bro.bif | 7520 ++++++++++++++++++++++++++------------------------- 1 file changed, 3790 insertions(+), 3730 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index db72777b92..9ae9fb94ca 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -317,812 +317,15 @@ static int next_fmt(const char*& fmt, val_list* args, ODesc* d, int& n) } %%} -## Returns the number of elements in a container. This function works with all -## container types, i.e., sets, tables, and vectors. -## -## v: The container whose elements are counted. -## -## Returns: The number of elements in *v*. -function length%(v: any%): count - %{ - TableVal* tv = v->Type()->Tag() == TYPE_TABLE ? v->AsTableVal() : 0; - - if ( tv ) - return new Val(tv->Size(), TYPE_COUNT); - - else if ( v->Type()->Tag() == TYPE_VECTOR ) - return new Val(v->AsVectorVal()->Size(), TYPE_COUNT); - - else - { - builtin_error("length() requires a table/set/vector argument"); - return new Val(0, TYPE_COUNT); - } - %} - -## Checks whether two objects reference the same internal object. This function -## uses equality comparison of C++ raw pointer values to determine if the two -## objects are the same. -## -## o1: The first object. -## -## o2: The second object. -## -## Returns: True if *o1* and *o2* are equal. -function same_object%(o1: any, o2: any%): bool - %{ - return new Val(o1 == o2, TYPE_BOOL); - %} - -## Removes all elements from a set or table. -## -## v: The set or table -## -## Returns: The cleared set/table or 0 if *v* is not a set/table type. -function clear_table%(v: any%): any - %{ - if ( v->Type()->Tag() == TYPE_TABLE ) - v->AsTableVal()->RemoveAll(); - else - builtin_error("clear_table() requires a table/set argument"); - - return 0; - %} - -## Returns the concatenation of the string representation of its arguments. The -## arguments can be of any type. For example, ``cat("foo", 3, T)`` returns -## ``"foo3T"``. -## -## Returns: A string concatentation of all arguments. -function cat%(...%): string - %{ - ODesc d; - loop_over_list(@ARG@, i) - @ARG@[i]->Describe(&d); - - BroString* s = new BroString(1, d.TakeBytes(), d.Len()); - s->SetUseFreeToDelete(true); - - return new StringVal(s); - %} - -## Converts a record type name to a vector of strings, where each element is -## the name of a record field. Nested records are flattened. -## -## rt: The name of the record type. -## -## Returns: A string vector with the field names of *rt*. -function record_type_to_vector%(rt: string%): string_vec - %{ - VectorVal* result = - new VectorVal(internal_type("string_vec")->AsVectorType()); - - RecordType *type = internal_type(rt->CheckString())->AsRecordType(); - - if ( type ) - { - for ( int i = 0; i < type->NumFields(); ++i ) - { - StringVal* val = new StringVal(type->FieldName(i)); - result->Assign(i+1, val, 0); - } - } - - return result; - %} - -## Concatenates all arguments, with a separator placed between each one. This -## function is similar to :bro:id:`cat`, but places a separator between each -## given argument. If any of the variable arguments is an empty string it is -## replaced by a given default string instead. -## -## sep: The separator to place betwen each argument. -## -## def: The default string to use when an argument is the empty string. -## -## Returns: A concatenation of all arguments with *sep* between each one and -## empty strings replaced with *def*. -## -## .. bro:see:: cat string_cat cat_string_array cat_string_array_n -function cat_sep%(sep: string, def: string, ...%): string - %{ - ODesc d; - int pre_size = 0; - - loop_over_list(@ARG@, i) - { - // Skip named parameters. - if ( i < 2 ) - continue; - - if ( i > 2 ) - d.Add(sep->CheckString(), 0); - - Val* v = @ARG@[i]; - if ( v->Type()->Tag() == TYPE_STRING && ! v->AsString()->Len() ) - v = def; - - v->Describe(&d); - } - - BroString* s = new BroString(1, d.TakeBytes(), d.Len()); - s->SetUseFreeToDelete(true); - - return new StringVal(s); - %} - -## Produces a formatted string à la ``printf``. The first argument is the -## *format string* and specifies how subsequent arguments are converted for -## output. It is composed of zero or more directives: ordinary characters (not -## ``%``), which are copied unchanged to the output, and conversion -## specifications, each of which fetches zero or more subsequent arguments. -## Conversion specifications begin with ``%`` and the arguments must properly -## correspond to the specifier. After the ``%``, the following characters -## may appear in sequence: -## -## - ``%``: Literal ``%`` -## -## - ``-``: Left-align field -## -## - ``[0-9]+``: The field width (< 128) -## -## - ``.``: Precision of floating point specifiers ``[efg]`` (< 128) -## -## - ``A``: Escape NUL bytes, i.e., replace ``0`` with ``\0`` -## -## - ``[DTdxsefg]``: Format specifier -## -## - ``[DT]``: ISO timestamp with microsecond precision -## -## - ``d``: Signed/Unsigned integer (using C-style ``%lld|``/``%llu`` -## for ``int``/``count``) -## -## - ``x``: Unsigned hexadecimal (using C-style ``%llx``); -## addresses/ports are converted to host-byte order -## -## - ``s``: Escaped string -## -## - ``[efg]``: Double -## -## Returns: Given no arguments, :bro:id:`fmt` returns an empty string. Given a -## non-string first argument, :bro:id:`fmt` returns the concatenation -## of all its arguments, per :bro:id:`cat`. Finally, given the wrong -## number of additional arguments for the given format specifier, -## :bro:id:`fmt` generates a run-time error. -## -## .. bro:see:: cat cat_sep string_cat cat_string_array cat_string_array_n -function fmt%(...%): string - %{ - if ( @ARGC@ == 0 ) - return new StringVal(""); - - Val* fmt_v = @ARG@[0]; - if ( fmt_v->Type()->Tag() != TYPE_STRING ) - return bro_cat(frame, @ARGS@); - - const char* fmt = fmt_v->AsString()->CheckString(); - ODesc d; - int n = 0; - - while ( next_fmt(fmt, @ARGS@, &d, n) ) - ; - - if ( n < @ARGC@ - 1 ) - builtin_error("too many arguments for format", fmt_v); - - else if ( n >= @ARGC@ ) - builtin_error("too few arguments for format", fmt_v); - - BroString* s = new BroString(1, d.TakeBytes(), d.Len()); - s->SetUseFreeToDelete(true); - - return new StringVal(s); - %} - - -## Returns the type name of an arbitrary Bro variable. -## -## t: An arbitrary object. -## -## Returns: The type name of *t*. -function type_name%(t: any%): string - %{ - ODesc d; - t->Type()->Describe(&d); - - BroString* s = new BroString(1, d.TakeBytes(), d.Len()); - s->SetUseFreeToDelete(true); - - return new StringVal(s); - %} - -## Converts a :bro:type:`string` to a :bro:type:`int`. -## -## str: The :bro:type:`string` to convert. -## -## Returns: The :bro:type:`string` *str* as :bro:type:`int`. -## -## .. bro:see:: to_addr to_port -function to_int%(str: string%): int - %{ - const char* s = str->CheckString(); - char* end_s; - - long l = strtol(s, &end_s, 10); - int i = int(l); - -#if 0 - // Not clear we should complain. For example, is " 205 " - // a legal conversion? - if ( s[0] == '\0' || end_s[0] != '\0' ) - builtin_error("bad conversion to integer", @ARG@[0]); -#endif - - return new Val(i, TYPE_INT); - %} - - -## Converts a (positive) :bro:type:`int` to a :bro:type:`count`. -## -## n: The :bro:type:`int` to convert. -## -## Returns: The :bro:type:`int` *n* as unsigned integer or 0 if *n* < 0. -function int_to_count%(n: int%): count - %{ - if ( n < 0 ) - { - builtin_error("bad conversion to count", @ARG@[0]); - n = 0; - } - return new Val(n, TYPE_COUNT); - %} - -## Converts a :bro:type:`double` to a :bro:type:`count`. -## -## d: The :bro:type:`double` to convert. -## -## Returns: The :bro:type:`double` *d* as unsigned integer or 0 if *d* < 0.0. -## -## .. bro:see:: double_to_time -function double_to_count%(d: double%): count - %{ - if ( d < 0.0 ) - builtin_error("bad conversion to count", @ARG@[0]); - - return new Val(bro_uint_t(rint(d)), TYPE_COUNT); - %} - -## Converts a :bro:type:`string` to a :bro:type:`count`. -## -## str: The :bro:type:`string` to convert. -## -## Returns: The :bro:type:`string` *str* as unsigned integer or if in invalid -## format. -## -## .. bro:see:: to_addr to_int to_port -function to_count%(str: string%): count - %{ - const char* s = str->CheckString(); - char* end_s; - - uint64 u = (uint64) strtoll(s, &end_s, 10); - - if ( s[0] == '\0' || end_s[0] != '\0' ) - { - builtin_error("bad conversion to count", @ARG@[0]); - u = 0; - } - - return new Val(u, TYPE_COUNT); - %} - -## Converts an :bro:type:`interval` to a :bro:type:`double`. -## -## i: The :bro:type:`interval` to convert. -## -## Returns: The :bro:type:`interval` *i* as :bro:type:`double`. -## -## .. bro:see:: double_to_interval -function interval_to_double%(i: interval%): double - %{ - return new Val(i, TYPE_DOUBLE); - %} - -## Converts a :bro:type:`time` value to a :bro:type:`double`. -## -## t: The :bro:type:`interval` to convert. -## -## Returns: The :bro:type:`time` value *t* as :bro:type:`double`. -## -## .. bro:see:: double_to_time -function time_to_double%(t: time%): double - %{ - return new Val(t, TYPE_DOUBLE); - %} - -## Converts a :bro:type:`time` value to a :bro:type:`double`. -## -## t: The :bro:type:`interval` to convert. -## -## Returns: The :bro:type:`time` value *t* as :bro:type:`double`. -## -## .. bro:see:: time_to_double double_to_count -function double_to_time%(d: double%): time - %{ - return new Val(d, TYPE_TIME); - %} - -## Converts a :bro:type:`double` to an :bro:type:`interval`. -## -## d: The :bro:type:`double` to convert. -## -## Returns: The :bro:type:`double` *d* as :bro:type:`interval`. -## -## .. bro:see:: interval_to_double -function double_to_interval%(d: double%): interval - %{ - return new Val(d, TYPE_INTERVAL); - %} - -## Converts a :bro:type:`addr` to a :bro:type:`count`. -## -## a: The :bro:type:`addr` to convert. -## -## Returns: The :bro:type:`addr` *a* as :bro:type:`count`. -## -## .. bro:see:: addr_to_ptr_name -function addr_to_count%(a: addr%): count - %{ -#ifdef BROv6 - if ( ! is_v4_addr(a) ) - { - builtin_error("conversion of non-IPv4 address to count", @ARG@[0]); - return new Val(0, TYPE_COUNT); - } - - uint32 addr = to_v4_addr(a); -#else - uint32 addr = a; -#endif - return new Val(ntohl(addr), TYPE_COUNT); - %} - -## Converts a :bro:type:`addr` to a :bro:type:`count`. -## -## a: The :bro:type:`addr` to convert. -## -## Returns: The :bro:type:`addr` *a* as :bro:type:`count`. -## -## .. bro:see:: count_to_port -function port_to_count%(p: port%): count - %{ - return new Val(p->Port(), TYPE_COUNT); - %} - -## Converts a :bro:type:`count` and ``transport_proto`` to a :bro:type:`port`. -## -## a: The :bro:type:`addr` to convert. -## -## Returns: The :bro:type:`addr` *a* as :bro:type:`count`. -## -## .. bro:see:: port_to_count -function count_to_port%(c: count, t: transport_proto%): port - %{ - return new PortVal(c, (TransportProto)(t->InternalInt())); - %} - -## Chops off any decimal digits of the given double, i.e., computes the -## "floor" of it. For example, ``floor(3.14)`` returns ``3.0``. -## -## d: The :bro:type:`double` to manipulate. -## -## Returns: The next lowest integer of *d* as :bro:type:`double`. -## -## .. bro:see:: sqrt exp ln log10 -function floor%(d: double%): double - %{ - return new Val(floor(d), TYPE_DOUBLE); - %} - -## Converts a :bro:type:`string` to an :bro:type:`addr`. -## -## ip: The :bro:type:`string` to convert. -## -## Returns: The :bro:type:`string` *ip* as :bro:type:`addr`. -## -## .. bro:see:: to_count to_int to_port count_to_v4_addr raw_bytes_to_v4_addr -function to_addr%(ip: string%): addr - %{ - char* s = ip->AsString()->Render(); - Val* ret = new AddrVal(s); - delete [] s; - return ret; - %} - -## Converts a :bro:type:`count` to an :bro:type:`addr`. -## -## ip: The :bro:type:`count` to convert. -## -## Returns: The :bro:type:`count` *ip* as :bro:type:`addr`. -## -## .. bro:see:: raw_bytes_to_v4_addr to_addr -function count_to_v4_addr%(ip: count%): addr - %{ - if ( ip > 4294967295LU ) - { - builtin_error("conversion of non-IPv4 count to addr", @ARG@[0]); - return new AddrVal(uint32(0)); - } - - return new AddrVal(htonl(uint32(ip))); - %} - -## Converts a :bro:type:`string` of bytes into an IP address. In particular, -## this function interprets the first 4 bytes of the string as an IPv4 address -## in network order. -## -## b: The raw bytes (:bro:type:`string`) to convert. -## -## Returns: The byte :bro:type:`string` *ip* as :bro:type:`addr`. -## -## .. bro:see:: raw_bytes_to_v4_addr to_addr -function raw_bytes_to_v4_addr%(b: string%): addr - %{ - uint32 a = 0; - - if ( b->Len() < 4 ) - builtin_error("too short a string as input to raw_bytes_to_v4_addr()"); - - else - { - const u_char* bp = b->Bytes(); - a = (bp[0] << 24) | (bp[1] << 16) | (bp[2] << 8) | bp[3]; - } - - return new AddrVal(htonl(a)); - %} - -## Creates a :bro:type:`port` from a given number and transport protocol. -## -## num: The port number. -## -## proto: THe transport protocol of the port. -## -## Returns: A :bro:type:`port` with number *num* and transport protocol -## *proto*. -## -## .. bro:see:: to_addr to_count to_int -function to_port%(num: count, proto: transport_proto%): port - %{ - return new PortVal(num, (TransportProto)proto->AsEnum()); - %} - -## Masks an address down to the number of given upper bits. For example, -## ``mask_addr(1.2.3.4, 18)`` returns ``1.2.0.0``. -## -## a: The address to mask. -## -## top_bits_to_keep: The number of top bits to keep in *a*; must be greater -## than 0 and less than 33. -## -## Returns: The address *a* masked down to *top_bits_to_keep* bits. -## -## .. bro:see:: remask_addr -function mask_addr%(a: addr, top_bits_to_keep: count%): subnet - %{ - return new SubNetVal(mask_addr(a, top_bits_to_keep), top_bits_to_keep); - %} - -## Takes some top bits (e.g., subnet address) from one address and the other -## bits (intra-subnet part) from a second address and merges them to get a new -## address. This is useful for anonymizing at subnet level while preserving -## serial scans. -## -## a1: The address to mask with *top_bits_from_a1*. -## -## a2: The address to take the remaining bits from. -## -## top_bits_from_a1: The number of top bits to keep in *a1*; must be greater -## than 0 and less than 33. -## -## Returns: The address *a* masked down to *top_bits_to_keep* bits. -## -## .. bro:see:: mask_addr -function remask_addr%(a1: addr, a2: addr, top_bits_from_a1: count%): addr - %{ -#ifdef BROv6 - if ( ! is_v4_addr(a1) || ! is_v4_addr(a2) ) - { - builtin_error("cannot use remask_addr on IPv6 addresses"); - return new AddrVal(a1); - } - - uint32 x1 = to_v4_addr(a1); - uint32 x2 = to_v4_addr(a2); -#else - uint32 x1 = a1; - uint32 x2 = a2; -#endif - return new AddrVal( - mask_addr(x1, top_bits_from_a1) | - (x2 ^ mask_addr(x2, top_bits_from_a1)) ); - %} - -## Checks whether a given :bro:type:`port` has TCP as transport protocol. -## -## p: The :bro:type:`port` to check. -## -## Returns: True iff *p* is a TCP port. -## -## .. bro:see:: is_udp_port is_icmp_port -function is_tcp_port%(p: port%): bool - %{ - return new Val(p->IsTCP(), TYPE_BOOL); - %} - -## Checks whether a given :bro:type:`port` has UDP as transport protocol. -## -## p: The :bro:type:`port` to check. -## -## Returns: True iff *p* is a UDP port. -## -## .. bro:see:: is_icmp_port is_tcp_port -function is_udp_port%(p: port%): bool - %{ - return new Val(p->IsUDP(), TYPE_BOOL); - %} - -## Checks whether a given :bro:type:`port` has ICMP as transport protocol. -## -## p: The :bro:type:`port` to check. -## -## Returns: True iff *p* is a ICMP port. -## -## .. bro:see:: is_tcp_port is_udp_port -function is_icmp_port%(p: port%): bool - %{ - return new Val(p->IsICMP(), TYPE_BOOL); - %} - -## Checks whether Bro reads traffic from one or more network interfaces (as -## opposed to from a network trace in a file). Note that this function returns -## true even after Bro has stopped reading network traffic, for example due to -## receiving a termination signal. -## -## Returns: True if reading traffic from a network interface. -## -## .. bro:see:: reading_traces -function reading_live_traffic%(%): bool - %{ - return new Val(reading_live, TYPE_BOOL); - %} - -## Checks whether Bro reads traffic from a trace file (as opposed to from a -## network interface). -## -## Returns: True if reading traffic from a network trace. -## -## .. bro:see:: reading_live_traffic -function reading_traces%(%): bool - %{ - return new Val(reading_traces, TYPE_BOOL); - %} - -## Opens a file for writing. If a file with the same name already exists, this -## function overwrites it (as opposed to :bro:id:`open_for_append`). -## -## f: The path to the file. -## -## Returns: A :bro:type:`file` handle for subsequent operations. -## -## .. bro:see;: active_file open_for_append close write_file -## get_file_name set_buf flush_all mkdir enable_raw_output -function open%(f: string%): file - %{ - const char* file = f->CheckString(); - - if ( streq(file, "-") ) - return new Val(new BroFile(stdout, "-", "w")); - else - return new Val(new BroFile(file, "w")); - %} - -## Opens a file for writing or appending. If a file with the same name already -## exists, this function appends to it (as opposed to :bro:id:`open`). -## -## f: The path to the file. -## -## Returns: A :bro:type:`file` handle for subsequent operations. -## -## .. bro:see;: active_file open close write_file -## get_file_name set_buf flush_all mkdir enable_raw_output -function open_for_append%(f: string%): file - %{ - return new Val(new BroFile(f->CheckString(), "a")); - %} - -## Closes an open file and flushes any buffered content. -## exists, this function appends to it (as opposed to :bro:id:`open`). -## -## f: A :bro:type:`file` handle to an open file. -## -## Returns: True on success. -## -## .. bro:see;: active_file open open_for_append write_file -## get_file_name set_buf flush_all mkdir enable_raw_output -function close%(f: file%): bool - %{ - return new Val(f->Close(), TYPE_BOOL); - %} - -## Writes data to an open file. -## -## f: A :bro:type:`file` handle to an open file. -## -## data: The data to write to *f*. -## -## Returns: True on success. -## -## .. bro:see;: active_file open open_for_append close -## get_file_name set_buf flush_all mkdir enable_raw_output -function write_file%(f: file, data: string%): bool - %{ - if ( ! f ) - return new Val(0, TYPE_BOOL); - - return new Val(f->Write((const char*) data->Bytes(), data->Len()), - TYPE_BOOL); - %} - -## Alters the buffering behavior of a file. -## -## f: A :bro:type:`file` handle to an open file. -## -## buffered: When true, *f* is fully buffered, i.e., bytes are saved in a -## buffered until the block size has been reached. When -## false, *f* is line buffered, i.e., bytes are saved up until a -## newline occurs. -## -## .. bro:see;: active_file open open_for_append close -## get_file_name write_file flush_all mkdir enable_raw_output -function set_buf%(f: file, buffered: bool%): any - %{ - f->SetBuf(buffered); - return new Val(0, TYPE_VOID); - %} - -## Flushes all open files to disk. -## -## Returns: True on success. -## -## .. bro:see;: active_file open open_for_append close -## get_file_name write_file set_buf mkdir enable_raw_output -function flush_all%(%): bool - %{ - return new Val(fflush(0) == 0, TYPE_BOOL); - %} - -## Creates a new directory. -## -## f: The directory name. -## -## Returns: Returns true if the operation succeeds and false if the -## creation fails or if *f* exists already. -## -## .. bro:see;: active_file open_for_append close write_file -## get_file_name set_buf flush_all enable_raw_output -function mkdir%(f: string%): bool - %{ - const char* filename = f->CheckString(); - if ( mkdir(filename, 0777) < 0 && errno != EEXIST ) - { - builtin_error("cannot create directory", @ARG@[0]); - return new Val(0, TYPE_BOOL); - } - else - return new Val(1, TYPE_BOOL); - %} - -## Checks whether a given file is open. -## -## f: The file to check. -## -## Returns: True if *f* is an open :bro:type:`file`. -## -## .. TODO:: Rename to ``is_open``. -function active_file%(f: file%): bool - %{ - return new Val(f->IsOpen(), TYPE_BOOL); - %} - -## Deprecated. Will be removed. -function active_connection%(id: conn_id%): bool - %{ - Connection* c = sessions->FindConnection(id); - return new Val(c ? 1 : 0, TYPE_BOOL); - %} - -## Deprecated. Will be removed. -function connection_record%(cid: conn_id%): connection - %{ - Connection* c = sessions->FindConnection(cid); - if ( c ) - return c->BuildConnVal(); - else - { - // Hard to recover from this until we have union types ... - builtin_error("connection ID not a known connection (fatal)", cid); - exit(0); - return 0; - } - %} - -%%{ -EnumVal* map_conn_type(TransportProto tp) - { - switch ( tp ) { - case TRANSPORT_UNKNOWN: - return new EnumVal(0, transport_proto); - break; - - case TRANSPORT_TCP: - return new EnumVal(1, transport_proto); - break; - - case TRANSPORT_UDP: - return new EnumVal(2, transport_proto); - break; - - case TRANSPORT_ICMP: - return new EnumVal(3, transport_proto); - break; - - default: - reporter->InternalError("bad connection type in map_conn_type()"); - } - - // Cannot be reached; - assert(false); - return 0; // Make compiler happy. - } -%%} - -## Extracts the transport protocol from a connection. -## -## cid: The connection identifier. -## -## Returns: The transport protocol of the connection identified by *id*. -## -## .. bro:see:: get_port_transport_proto -## get_orig_seq get_resp_seq -function get_conn_transport_proto%(cid: conn_id%): transport_proto - %{ - Connection* c = sessions->FindConnection(cid); - if ( ! c ) - { - builtin_error("unknown connection id in get_conn_transport_proto()", cid); - return new EnumVal(0, transport_proto); - } - - return map_conn_type(c->ConnTransport()); - %} - -## Extracts the transport protocol from a :bro:type:`port`. -## -## p: The port. -## -## Returns: The transport protocol of the port *p*. -## -## .. bro:see:: get_conn_transport_proto -## get_orig_seq get_resp_seq -function get_port_transport_proto%(p: port%): transport_proto - %{ - return map_conn_type(p->PortType()); - %} +# =========================================================================== +# +# Core +# +# =========================================================================== ## Returns the current wall-clock time. ## -## In general, you should use :bro:id:`network_time` instead +## In general, you should use :bro:id:`network_time` instead ## unless you are using Bro for non-networking uses (such as general ## scripting; not particularly recommended), because otherwise your script ## may behave very differently on live traffic versus played-back traffic @@ -1138,8 +341,8 @@ function current_time%(%): time ## Returns the timestamp of the last packet processed. This function returns ## the timestamp of the most recently read packet, whether read from a -## live network interface or from a save file. -## +## live network interface or from a save file. +## ## Returns: The timestamp of the packet processed. ## ## .. bro:see:: current_time @@ -1183,61 +386,6 @@ function setenv%(var: string, val: string%): bool return new Val(1, TYPE_BOOL); %} - -## Computes the square root of a :bro:type:`double`. -## -## x: The number to compute the square root of. -## -## Returns: The square root of *x*. -## -## .. bro:see:: floor exp ln log10 -function sqrt%(x: double%): double - %{ - if ( x < 0 ) - { - reporter->Error("negative sqrt argument"); - return new Val(-1.0, TYPE_DOUBLE); - } - - return new Val(sqrt(x), TYPE_DOUBLE); - %} - -## Computes the exponential function. -## -## d: The argument to the exponential function. -## -## Returns: *e* to the power of *d*. -## -## .. bro:see:: floor sqrt ln log10 -function exp%(d: double%): double - %{ - return new Val(exp(d), TYPE_DOUBLE); - %} - -## Computes the natural logarithm of a number. -## -## d: The argument to the logarithm. -## -## Returns: The natural logarithm of *d*. -## -## .. bro:see:: exp floor sqrt log10 -function ln%(d: double%): double - %{ - return new Val(log(d), TYPE_DOUBLE); - %} - -## Computes the common logarithm of a number. -## -## d: The argument to the logarithm. -## -## Returns: The common logarithm of *d*. -## -## .. bro:see:: exp floor sqrt ln -function log10%(d: double%): double - %{ - return new Val(log10(d), TYPE_DOUBLE); - %} - ## Shuts down the Bro process immediately. ## ## .. todo: Change function signature to ``exit(code: int): any``. @@ -1319,10 +467,10 @@ static int do_system(const char* s) ## Returns: The return value from the OS ``system`` function. ## ## .. bro:see:: system_env str_shell_escape piped_exec -## +## ## .. note:: ## -## Note that this corresponds to the status of backgrounding the +## Note that this corresponds to the status of backgrounding the ## given command, not to the exit status of the command itself. A ## value of 127 corresponds to a failure to execute ``sh``, and -1 ## to an internal system failure. @@ -1337,12 +485,12 @@ function system%(str: string%): int ## but changes the environment before invoking the command. ## ## str: The command to execute. -## +## ## env: A :bro:type:`set` or :bro:type:`table` with the environment variables ## in the form of key-value pairs (where the value is optional). -## +## ## Returns: The return value from the OS ``system`` function. -## +## ## .. bro:see:: system str_shell_escape piped_exec function system_env%(str: string, env: any%): int %{ @@ -1362,1144 +510,43 @@ function system_env%(str: string, env: any%): int return new Val(result, TYPE_INT); %} - -%%{ -static Val* parse_eftp(const char* line) - { - RecordVal* r = new RecordVal(ftp_port); - - int net_proto = 0; // currently not used - uint32 addr = 0; - int port = 0; - int good = 0; - - if ( line ) - { - while ( isspace(*line) ) // skip whitespace - ++line; - - char delimiter = *line; - good = 1; - char* next_delim; - - ++line; // cut off delimiter - net_proto = strtol(line, &next_delim, 10); // currently ignored - if ( *next_delim != delimiter ) - good = 0; - - line = next_delim + 1; - if ( *line != delimiter ) // default of 0 is ok - { - addr = dotted_to_addr(line); - if ( addr == 0 ) - good = 0; - } - - // FIXME: check for garbage between IP and delimiter. - line = strchr(line, delimiter); - - ++line; // now the port - port = strtol(line, &next_delim, 10); - if ( *next_delim != delimiter ) - good = 0; - } - - r->Assign(0, new AddrVal(addr)); - r->Assign(1, new PortVal(port, TRANSPORT_TCP)); - r->Assign(2, new Val(good, TYPE_BOOL)); - - return r; - } -%%} - -%%{ -static Val* parse_port(const char* line) - { - RecordVal* r = new RecordVal(ftp_port); - - int bytes[6]; - if ( line && sscanf(line, "%d,%d,%d,%d,%d,%d", - &bytes[0], &bytes[1], &bytes[2], - &bytes[3], &bytes[4], &bytes[5]) == 6 ) - { - int good = 1; - - for ( int i = 0; i < 6; ++i ) - if ( bytes[i] < 0 || bytes[i] > 255 ) - { - good = 0; - break; - } - - uint32 addr = (bytes[0] << 24) | (bytes[1] << 16) | - (bytes[2] << 8) | bytes[3]; - uint32 port = (bytes[4] << 8) | bytes[5]; - - // Since port is unsigned, no need to check for < 0. - if ( port > 65535 ) - { - port = 0; - good = 0; - } - - r->Assign(0, new AddrVal(htonl(addr))); - r->Assign(1, new PortVal(port, TRANSPORT_TCP)); - r->Assign(2, new Val(good, TYPE_BOOL)); - } - else - { - r->Assign(0, new AddrVal(uint32(0))); - r->Assign(1, new PortVal(0, TRANSPORT_TCP)); - r->Assign(2, new Val(0, TYPE_BOOL)); - } - - return r; - } -%%} - -## Checks whether a connection is (still) active. +## Opens a program with ``popen`` and writes a given string to the returned +## stream to send it to the opened process's stdin. ## -## c: The connection id to check. +## program: The program to execute. ## -## Returns: True if the connection identified by *c* exists. +## to_write: Data to pipe to the opened program's process via ``stdin``. ## -## .. bro:see:: lookup_connection -function connection_exists%(c: conn_id%): bool +## Returns: True on success. +## +## .. bro:see:: system system_env +function piped_exec%(program: string, to_write: string%): bool %{ - if ( sessions->FindConnection(c) ) - return new Val(1, TYPE_BOOL); - else - return new Val(0, TYPE_BOOL); - %} + const char* prog = program->CheckString(); -## Returns the :bro:id:`connection` record for a given connection identifier. -## -## cid: The connection ID. -## -## Returns: The :bro:id:`connection` record for *cid*. If *cid* does not point -## to an existing connection, the function generates a run-time error -## and returns a dummy value. -## -## .. bro:see:: connection_exists -function lookup_connection%(cid: conn_id%): connection - %{ - Connection* conn = sessions->FindConnection(cid); - if ( conn ) - return conn->BuildConnVal(); - - builtin_error("connection ID not a known connection", cid); - - // Return a dummy connection record. - RecordVal* c = new RecordVal(connection_type); - - RecordVal* id_val = new RecordVal(conn_id); - id_val->Assign(0, new AddrVal((unsigned int) 0)); - id_val->Assign(1, new PortVal(ntohs(0), TRANSPORT_UDP)); - id_val->Assign(2, new AddrVal((unsigned int) 0)); - id_val->Assign(3, new PortVal(ntohs(0), TRANSPORT_UDP)); - c->Assign(0, id_val); - - RecordVal* orig_endp = new RecordVal(endpoint); - orig_endp->Assign(0, new Val(0, TYPE_COUNT)); - orig_endp->Assign(1, new Val(int(0), TYPE_COUNT)); - - RecordVal* resp_endp = new RecordVal(endpoint); - resp_endp->Assign(0, new Val(0, TYPE_COUNT)); - resp_endp->Assign(1, new Val(int(0), TYPE_COUNT)); - - c->Assign(1, orig_endp); - c->Assign(2, resp_endp); - - c->Assign(3, new Val(network_time, TYPE_TIME)); - c->Assign(4, new Val(0.0, TYPE_INTERVAL)); - c->Assign(5, new TableVal(string_set)); // service - c->Assign(6, new StringVal("")); // addl - c->Assign(7, new Val(0, TYPE_COUNT)); // hot - c->Assign(8, new StringVal("")); // history - - return c; - %} - -## Informs Bro that it should skip any further processing of the contents of -## a given connection. In particular, Bro will refrain from reassembling the -## TCP byte stream and from generating events relating to any analyzers that -## have been processing the connection. -## -## cid: The connection ID. -## -## Returns: False if *id* does not point to an active connection and true -## otherwise. -## -## .. note:: -## -## Bro will still generate connection-oriented events such as -## :bro:id:`connection_finished`. -function skip_further_processing%(cid: conn_id%): bool - %{ - Connection* c = sessions->FindConnection(cid); - if ( ! c ) - return new Val(0, TYPE_BOOL); - - c->SetSkip(1); - return new Val(1, TYPE_BOOL); - %} - -## Controls whether packet contents belonging to a connection should be -## recorded (when ``-w`` option is provided on the command line). -## -## cid: The connection identifier. -## -## do_record: True to enable packet contens and false to disable for the -## connection identified by *cid*. -## -## Returns: False if *id* does not point to an active connection and true -## otherwise. -## -## .. bro:see:: skip_further_processing -## -## .. note:: -## -## This is independent of whether Bro processes the packets of this -## connection, which is controlled separately by -## :bro:id:`skip_further_processing`. -## -## .. bro:see: get_contents_file set_contents_file -function set_record_packets%(cid: conn_id, do_record: bool%): bool - %{ - Connection* c = sessions->FindConnection(cid); - if ( ! c ) - return new Val(0, TYPE_BOOL); - - c->SetRecordPackets(do_record); - return new Val(1, TYPE_BOOL); - %} - -## Associates a file handle with a connection for writing TCP byte stream -## contents. -## -## cid: The connection ID. -## -## direction: Controls what sides of the connection to record. The argument can -## take one the four values: -## -## - ``CONTENTS_NONE``: Stop recording the connection's content. -## - ``CONTENTS_ORIG``: Record the data sent by the connection -## originator (often the client). -## - ``CONTENTS_RESP``: Record the data sent by the connection -## responder (often the server). -## - ``CONTENTS_BOTH``: Record the data sent in both directions. -## Results in the two directions being -## intermixed in the file, in the order the -## data was seen by Bro. -## -## f: The file handle of the file to write the contents to. -## -## Returns: Returns false if *id* does not point to an active connection and -## true otherwise. -## -## .. note:: -## -## The data recorded to the file reflects the byte stream, not the -## contents of individual packets. Reordering and duplicates are -## removed. If any data is missing, the recording stops at the -## missing data; this can happen, e.g., due to an -## :bro:id:`ack_above_hole` event. -## -## .. bro:see: get_contents_file set_record_packets -function set_contents_file%(cid: conn_id, direction: count, f: file%): bool - %{ - Connection* c = sessions->FindConnection(cid); - if ( ! c ) - return new Val(0, TYPE_BOOL); - - c->GetRootAnalyzer()->SetContentsFile(direction, f); - return new Val(1, TYPE_BOOL); - %} - -## Returns the file handle of the contents file of a connection. -## -## cid: The connection ID. -## -## direction: Controls what sides of the connection to record. SEe -## :bro:id:`set_contents_file` for possible values. -## -## Returns: The :bro:type:`file` handle for the contentents file of the -## connection identified by *cid*. If the connection exists -## but no contents file for *direction*, the function generates a -## error and returns a file handle to ``stderr``. -## -## .. bro:see: set_contents_file set_record_packets -function get_contents_file%(cid: conn_id, direction: count%): file - %{ - Connection* c = sessions->FindConnection(cid); - BroFile* f = c ? c->GetRootAnalyzer()->GetContentsFile(direction) : 0; - - if ( f ) - { - Ref(f); - return new Val(f); - } - - // Return some sort of error value. - if ( ! c ) - builtin_error("unknown connection id in get_contents_file()", cid); - else - builtin_error("no contents file for given direction"); - - return new Val(new BroFile(stderr, "-", "w")); - %} - -## Gets the filename associated with a file handle. -## -## f: The file handle to inquire the name for. -## -## Returns: The filename associated with *f*. -## -## .. bro:see:: open -function get_file_name%(f: file%): string - %{ + FILE* f = popen(prog, "w"); if ( ! f ) - return new StringVal(""); - - return new StringVal(f->Name()); - %} - -## Sets an individual inactivity timeout for a connection and thus -## overrides the global inactivity timeout. -## -## cid: The connection ID. -## -## t: The new inactivity timeout for the connection identified by *cid*. -## -## Returns: The previous timeout interval. -function set_inactivity_timeout%(cid: conn_id, t: interval%): interval - %{ - Connection* c = sessions->FindConnection(cid); - if ( ! c ) - return new Val(0, TYPE_INTERVAL); - - double old_timeout = c->InactivityTimeout(); - c->SetInactivityTimeout(t); - - return new Val(old_timeout, TYPE_INTERVAL); - %} - -## Returns the state of the given login (Telnet or Rlogin) connection. -## -## cid: The connection ID. -## -## Returns: False if the connection is not active or is not tagged as a -## login analyzer. Otherwise the function returns the state, which can -## be one of: -## -## - ``LOGIN_STATE_AUTHENTICATE``: The connection is in its -## initial authentication dialog. -## - ``OGIN_STATE_LOGGED_IN``: The analyzer believes the user has -## successfully authenticated. -## - ``LOGIN_STATE_SKIP``: The analyzer has skipped any further -## processing of the connection. -## - ``LOGIN_STATE_CONFUSED``: The analyzer has concluded that it -## does not correctly know the state of the connection, and/or -## the username associated with it. -## -## .. bro:see: set_login_state -function get_login_state%(cid: conn_id%): count - %{ - Connection* c = sessions->FindConnection(cid); - if ( ! c ) + { + reporter->Error("Failed to popen %s", prog); return new Val(0, TYPE_BOOL); + } - Analyzer* la = c->FindAnalyzer(AnalyzerTag::Login); - if ( ! la ) + const u_char* input_data = to_write->Bytes(); + int input_data_len = to_write->Len(); + + int bytes_written = fwrite(input_data, 1, input_data_len, f); + + pclose(f); + + if ( bytes_written != input_data_len ) + { + reporter->Error("Failed to write all given data to %s", prog); return new Val(0, TYPE_BOOL); + } - return new Val(int(static_cast(la)->LoginState()), - TYPE_COUNT); - %} - -## Sets the login state of a connection with a login analyzer. -## -## cid: The connection ID. -## -## new_state: The new state of the login analyzer. See -## :bro:id:`get_login_state` for possible values. -## -## Returns: Returns false if *cid* is not an active connection -## or does not tagged as login analyzer, and true otherwise. -## -## .. bro:see: get_login_state -function set_login_state%(cid: conn_id, new_state: count%): bool - %{ - Connection* c = sessions->FindConnection(cid); - if ( ! c ) - return new Val(0, TYPE_BOOL); - - Analyzer* la = c->FindAnalyzer(AnalyzerTag::Login); - if ( ! la ) - return new Val(0, TYPE_BOOL); - - static_cast(la)->SetLoginState(login_state(new_state)); return new Val(1, TYPE_BOOL); %} -%%{ -#include "TCP.h" -%%} - -## Get the originator sequence number of a TCP connection. Sequence numbers -## are absolute (i.e., they reflect the values seen directly in packet headers; -## they are not relative to the beginning of the connection). -## -## cid: The connection ID. -## -## Returns: The highest sequence number sent by a connection's originator, or 0 -## if *cid* does not point to an active TCP connection. -## -## .. bro:see:: get_resp_seq -function get_orig_seq%(cid: conn_id%): count - %{ - Connection* c = sessions->FindConnection(cid); - if ( ! c ) - return new Val(0, TYPE_COUNT); - - if ( c->ConnTransport() != TRANSPORT_TCP ) - return new Val(0, TYPE_COUNT); - - Analyzer* tc = c->FindAnalyzer(AnalyzerTag::TCP); - if ( tc ) - return new Val(static_cast(tc)->OrigSeq(), - TYPE_COUNT); - else - { - reporter->Error("connection does not have TCP analyzer"); - return new Val(0, TYPE_COUNT); - } - %} - -## Get the responder sequence number of a TCP connection. Sequence numbers -## are absolute (i.e., they reflect the values seen directly in packet headers; -## they are not relative to the beginning of the connection). -## -## cid: The connection ID. -## -## Returns: The highest sequence number sent by a connection's responder, or 0 -## if *cid* does not point to an active TCP connection. -## -## .. bro:see:: get_orig_seq -function get_resp_seq%(cid: conn_id%): count - %{ - Connection* c = sessions->FindConnection(cid); - if ( ! c ) - return new Val(0, TYPE_COUNT); - - if ( c->ConnTransport() != TRANSPORT_TCP ) - return new Val(0, TYPE_COUNT); - - Analyzer* tc = c->FindAnalyzer(AnalyzerTag::TCP); - if ( tc ) - return new Val(static_cast(tc)->RespSeq(), - TYPE_COUNT); - else - { - reporter->Error("connection does not have TCP analyzer"); - return new Val(0, TYPE_COUNT); - } - %} - -## Converts a reverse pointer name to an address. For example, -## ``1.0.168.192.in-addr.arpa`` to ``192.168.0.1``. -## -## s: The string with the reverse pointer name. -## -## Returns: The IP address corresponding to *s*. -## -## .. bro:see:: addr_to_ptr_name parse_dotted_addr -function ptr_name_to_addr%(s: string%): addr - %{ - int a[4]; - uint32 addr; - - if ( sscanf(s->CheckString(), - "%d.%d.%d.%d.in-addr.arpa", - a, a+1, a+2, a+3) != 4 ) - { - builtin_error("bad PTR name", @ARG@[0]); - addr = 0; - } - else - addr = (a[3] << 24) | (a[2] << 16) | (a[1] << 8) | a[0]; - - return new AddrVal(htonl(addr)); - %} - -## Converts an IP address to a reverse pointer name. For example, -## ``192.168.0.1`` to ``1.0.168.192.in-addr.arpa``. -## -## a: The IP address to convert to a reverse pointer name. -## -## Returns: The reverse pointer representation of *a*. -## -## .. bro:see:: addr_to_count ptr_name_to_addr parse_dotted_addr -function addr_to_ptr_name%(a: addr%): string - %{ - // ## Question: - // uint32 addr = ntohl((*args)[0]->InternalUnsigned()); - uint32 addr; -#ifdef BROv6 - if ( is_v4_addr(a) ) - addr = a[3]; - else - { - builtin_error("conversion of non-IPv4 address to net", @ARG@[0]); - addr = 0; - } -#else - addr = a; -#endif - - addr = ntohl(addr); - uint32 a3 = (addr >> 24) & 0xff; - uint32 a2 = (addr >> 16) & 0xff; - uint32 a1 = (addr >> 8) & 0xff; - uint32 a0 = addr & 0xff; - - char buf[256]; - sprintf(buf, "%u.%u.%u.%u.in-addr.arpa", a0, a1, a2, a3); - - return new StringVal(buf); - %} - -# Transforms n0.n1.n2.n3 -> addr. - -## Converts a decimal dotted IP address in a :bro:type:`string` to an -## :bro:type:`addr` type. -## -## s: The IP address in the form ``n0.n1.n2.n3``. -## -## Returns: The IP address as type :bro:type:`addr`. -## -## .. bro:see:: addr_to_ptr_name parse_dotted_addr addr_to_count -function parse_dotted_addr%(s: string%): addr - %{ - return new AddrVal(dotted_to_addr(s->CheckString())); - %} - -## Converts a string representation of the FTP PORT command to an ``ftp_port``. -## -## s: The string of the FTP PORT command, e.g., ``"10,0,0,1,4,31"``. -## -## Returns: The FTP PORT, e.g., ``[h=10.0.0.1, p=1055/tcp, valid=T]`` -## -## .. bro:see:: parse_eftp_port parse_ftp_pasv parse_ftp_epsv fmt_ftp_port -function parse_ftp_port%(s: string%): ftp_port - %{ - return parse_port(s->CheckString()); - %} - -## Converts a string representation of the FTP EPRT command to an ``ftp_port``. -## (see `RFC 2428 `_). -## The format is ``EPRT``, -## where ```` is a delimiter in the ASCII range 33-126 (usually ``|``). -## -## s: The string of the FTP PORT command, e.g., ``"10,0,0,1,4,31"``. -## -## Returns: The FTP PORT, e.g., ``[h=10.0.0.1, p=1055/tcp, valid=T]`` -## -## .. bro:see:: parse_ftp_port parse_ftp_pasv parse_ftp_epsv fmt_ftp_port -function parse_eftp_port%(s: string%): ftp_port - %{ - return parse_eftp(s->CheckString()); - %} - -## Converts the result of the FTP PASV command to an ``ftp_port``. -## -## str: The string containing the result of the FTP PASV command. -## -## Returns: The FTP PORT, e.g., ``[h=10.0.0.1, p=1055/tcp, valid=T]`` -## -## .. bro:see:: parse_ftp_port parse_eftp_port parse_ftp_epsv fmt_ftp_port -function parse_ftp_pasv%(str: string%): ftp_port - %{ - const char* s = str->CheckString(); - const char* line = strchr(s, '('); - if ( line ) - ++line; // move past '(' - else if ( (line = strstr(s, "PORT")) ) - line += 5; // Skip over - else if ( (line = strchr(s, ',')) ) - { // Look for comma-separated list. - while ( --line >= s && isdigit(*line) ) - ; // Back up over preceding digits. - ++line; // now points to first digit, or beginning of s - } - - return parse_port(line); - %} - -## Converts the result of the FTP EPSV command to an ``ftp_port``. -## See `RFC 2428 `_. -## The format is `` ()``, where ```` is a -## delimiter in the ASCII range 33-126 (usually ``|``). -## -## str: The string containing the result of the FTP PASV command. -## -## Returns: The FTP PORT, e.g., ``[h=10.0.0.1, p=1055/tcp, valid=T]`` -## -## .. bro:see:: parse_ftp_port parse_eftp_port parse_ftp_pasv fmt_ftp_port -function parse_ftp_epsv%(str: string%): ftp_port - %{ - const char* s = str->CheckString(); - const char* line = strchr(s, '('); - if ( line ) - ++line; // move past '(' - return parse_eftp(line); - %} - -## Formats an IP address and TCP port as an FTP PORT command. For example, -## ``10.0.0.1`` and ``1055/tcp`` yields ``"10,0,0,1,4,31"``. -## -## a: The IP address. -## -## p: The TCP port. -## -## Returns: The FTP PORT string. -## -## .. bro:see:: parse_ftp_port parse_eftp_port parse_ftp_pasv parse_ftp_epsv -function fmt_ftp_port%(a: addr, p: port%): string - %{ -#ifdef BROv6 - if ( ! is_v4_addr(a) ) - builtin_error("conversion of non-IPv4 address to net", @ARG@[0]); - - uint32 addr = to_v4_addr(a); -#else - uint32 addr = a; -#endif - addr = ntohl(addr); - uint32 pn = p->Port(); - return new StringVal(fmt("%d,%d,%d,%d,%d,%d", - addr >> 24, (addr >> 16) & 0xff, - (addr >> 8) & 0xff, addr & 0xff, - pn >> 8, pn & 0xff)); - %} - -## Decode a NetBIOS name. See http://support.microsoft.com/kb/194203. -## -## name: The encoded NetBIOS name, e.g., ``"FEEIEFCAEOEFFEECEJEPFDCAEOEBENEF:``. -## -## Returns: The decoded NetBIOS name, e.g., ``"THE NETBIOS NAME"``. -## -## .. bro:see:: decode_netbios_name_type -function decode_netbios_name%(name: string%): string - %{ - char buf[16]; - char result[16]; - const u_char* s = name->Bytes(); - int i, j; - - for ( i = 0, j = 0; i < 16; ++i ) - { - char c0 = (j < name->Len()) ? toupper(s[j++]) : 'A'; - char c1 = (j < name->Len()) ? toupper(s[j++]) : 'A'; - buf[i] = ((c0 - 'A') << 4) + (c1 - 'A'); - } - - for ( i = 0; i < 15; ++i ) - { - if ( isalnum(buf[i]) || ispunct(buf[i]) || - // \x01\x02 is seen in at least one case as the first two bytes. - // I think that any \x01 and \x02 should always be passed through. - buf[i] < 3 ) - result[i] = buf[i]; - else - break; - } - - return new StringVal(i, result); - %} - -## Converts a NetBIOS name type to its corresonding numeric value. -## See http://support.microsoft.com/kb/163409. -## -## name: The NetBIOS name type. -## -## Returns: The numeric value of *name*. -## -## .. bro:see:: decode_netbios_name -function decode_netbios_name_type%(name: string%): count - %{ - const u_char* s = name->Bytes(); - char return_val = ((toupper(s[30]) - 'A') << 4) + (toupper(s[31]) - 'A'); - return new Val(return_val, TYPE_COUNT); - %} - -%%{ -#include "HTTP.h" - -const char* conn_id_string(Val* c) - { - Val* id = (*(c->AsRecord()))[0]; - const val_list* vl = id->AsRecord(); - - addr_type orig_h = (*vl)[0]->AsAddr(); - uint32 orig_p = (*vl)[1]->AsPortVal()->Port(); - addr_type resp_h = (*vl)[2]->AsAddr(); - uint32 resp_p = (*vl)[3]->AsPortVal()->Port(); - - return fmt("%s/%u -> %s/%u\n", dotted_addr(orig_h), orig_p, dotted_addr(resp_h), resp_p); - } -%%} - -## Skips the data of the HTTP entity. -## -## c: The HTTP connection. -## -## is_orig: If true, the client data is skipped and the server data otherwise. -## -## .. bro:see:: skip_smtp_data -function skip_http_entity_data%(c: connection, is_orig: bool%): any - %{ - AnalyzerID id = mgr.CurrentAnalyzer(); - if ( id ) - { - Analyzer* ha = c->FindAnalyzer(id); - - if ( ha ) - { - if ( ha->GetTag() == AnalyzerTag::HTTP ) - static_cast(ha)->SkipEntityData(is_orig); - else - reporter->Error("non-HTTP analyzer associated with connection record"); - } - else - reporter->Error("could not find analyzer for skip_http_entity_data"); - - } - else - reporter->Error("no analyzer associated with connection record"); - - return 0; - %} - -## Unescapes all characters in a URI, i.e., decodes every ``%xx`` group. -## -## URI: The URI to unescape. -## -## Returns: The unescaped URI with all ``%xx`` groups decoded. -## -## .. note:: -## -## Unescaping reserved characters may cause loss of information. RFC 2396: -## A URI is always in an "escaped" form, since escaping or unescaping a -## completed URI might change its semantics. Normally, the only time -## escape encodings can safely be made is when the URI is being created -## from its component parts. -function unescape_URI%(URI: string%): string - %{ - const u_char* line = URI->Bytes(); - const u_char* const line_end = line + URI->Len(); - - return new StringVal(unescape_URI(line, line_end, 0)); - %} - -%%{ -#include "SMTP.h" -%%} - -## Skips SMTP data until the next email in a connection. -## -## c: The SMTP connection. -## -## .. bro:see:: skip_http_entity_data -function skip_smtp_data%(c: connection%): any - %{ - Analyzer* sa = c->FindAnalyzer(AnalyzerTag::SMTP); - if ( sa ) - static_cast(sa)->SkipData(); - return 0; - %} - - -## Converts a string of bytes into its hexadecimal representation, e.g., -## ``"04"`` to ``"3034"``. -## -## bytestring: The string of bytes. -## -## Returns: The hexadecimal reprsentation of *bytestring*. -## -## .. bro:see:: hexdump -function bytestring_to_hexstr%(bytestring: string%): string - %{ - bro_uint_t len = bytestring->AsString()->Len(); - const u_char* bytes = bytestring->AsString()->Bytes(); - char hexstr[(2 * len) + 1]; - - hexstr[0] = 0; - for ( bro_uint_t i = 0; i < len; ++i ) - snprintf(hexstr + (2 * i), 3, "%.2hhx", bytes[i]); - - return new StringVal(hexstr); - %} - -%%{ -extern const char* bro_version(); -%%} - -## Returns statistics about the number of packets *(i)* received by Bro, -## *(ii)* dropped, and *(iii)* seen on the link (not always available). -## -## Returns: A record of packet statistics. -## -## .. bro:see:: do_profiling -## resource_usage -## get_matcher_stats -## dump_rule_stats -## get_gap_summary -function net_stats%(%): NetStats - %{ - unsigned int recv = 0; - unsigned int drop = 0; - unsigned int link = 0; - - loop_over_list(pkt_srcs, i) - { - PktSrc* ps = pkt_srcs[i]; - - struct PktSrc::Stats stat; - ps->Statistics(&stat); - recv += stat.received; - drop += stat.dropped; - link += stat.link; - } - - RecordVal* ns = new RecordVal(net_stats); - ns->Assign(0, new Val(recv, TYPE_COUNT)); - ns->Assign(1, new Val(drop, TYPE_COUNT)); - ns->Assign(2, new Val(link, TYPE_COUNT)); - - return ns; - %} - -## Returns Bro process statistics, such as real/user/sys CPU time, memory -## usage, page faults, number of TCP/UDP/ICMP connections, timers, and events -## queued/dispatched. -## -## Returns: A record with resource usage statistics. -## -## .. bro:see:: do_profiling -## net_stats -## get_matcher_stats -## dump_rule_stats -## get_gap_summary -function resource_usage%(%): bro_resources - %{ - struct rusage r; - - if ( getrusage(RUSAGE_SELF, &r) < 0 ) - reporter->InternalError("getrusage() failed in bro_resource_usage()"); - - double elapsed_time = current_time() - bro_start_time; - - double user_time = - double(r.ru_utime.tv_sec) + double(r.ru_utime.tv_usec) / 1e6; - double system_time = - double(r.ru_stime.tv_sec) + double(r.ru_stime.tv_usec) / 1e6; - - RecordVal* res = new RecordVal(bro_resources); - int n = 0; - - res->Assign(n++, new StringVal(bro_version())); - -#ifdef DEBUG - res->Assign(n++, new Val(1, TYPE_COUNT)); -#else - res->Assign(n++, new Val(0, TYPE_COUNT)); -#endif - - res->Assign(n++, new Val(bro_start_time, TYPE_TIME)); - - res->Assign(n++, new IntervalVal(elapsed_time, Seconds)); - res->Assign(n++, new IntervalVal(user_time, Seconds)); - res->Assign(n++, new IntervalVal(system_time, Seconds)); - - unsigned int total_mem; - get_memory_usage(&total_mem, 0); - res->Assign(n++, new Val(unsigned(total_mem), TYPE_COUNT)); - - res->Assign(n++, new Val(unsigned(r.ru_minflt), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_majflt), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_nswap), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_inblock), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_oublock), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_nivcsw), TYPE_COUNT)); - - SessionStats s; - if ( sessions ) - sessions->GetStats(s); - -#define ADD_STAT(x) \ - res->Assign(n++, new Val(unsigned(sessions ? x : 0), TYPE_COUNT)); - - ADD_STAT(s.num_TCP_conns); - ADD_STAT(s.num_UDP_conns); - ADD_STAT(s.num_ICMP_conns); - ADD_STAT(s.num_fragments); - ADD_STAT(s.num_packets); - ADD_STAT(s.num_timers); - ADD_STAT(s.num_events_queued); - ADD_STAT(s.num_events_dispatched); - ADD_STAT(s.max_TCP_conns); - ADD_STAT(s.max_UDP_conns); - ADD_STAT(s.max_ICMP_conns); - ADD_STAT(s.max_fragments); - ADD_STAT(s.max_timers); - - return res; - %} - -## Returns statistics about the regular expression engine, such as the number -## of distinct matchers, DFA states, DFA state transitions, memory usage of -## DFA states, cache hits/misses, and average number of NFA states across all -## matchers. -## -## Returns: A record with matcher statistics. -## -## .. bro:see:: do_profiling -## net_stats -## resource_usage -## dump_rule_stats -## get_gap_summary -function get_matcher_stats%(%): matcher_stats - %{ - RuleMatcher::Stats s; - memset(&s, 0, sizeof(s)); - - if ( rule_matcher ) - rule_matcher->GetStats(&s); - - RecordVal* r = new RecordVal(matcher_stats); - r->Assign(0, new Val(s.matchers, TYPE_COUNT)); - r->Assign(1, new Val(s.dfa_states, TYPE_COUNT)); - r->Assign(2, new Val(s.computed, TYPE_COUNT)); - r->Assign(3, new Val(s.mem, TYPE_COUNT)); - r->Assign(4, new Val(s.hits, TYPE_COUNT)); - r->Assign(5, new Val(s.misses, TYPE_COUNT)); - r->Assign(6, new Val(s.avg_nfa_states, TYPE_COUNT)); - - return r; - %} - -## Returns statistics about TCP gaps. -## -## Returns: A record with TCP gap statistics. -## -## .. bro:see:: do_profiling -## net_stats -## resource_usage -## dump_rule_stats -## get_matcher_stats -function get_gap_summary%(%): gap_info - %{ - RecordVal* r = new RecordVal(gap_info); - r->Assign(0, new Val(tot_ack_events, TYPE_COUNT)); - r->Assign(1, new Val(tot_ack_bytes, TYPE_COUNT)); - r->Assign(2, new Val(tot_gap_events, TYPE_COUNT)); - r->Assign(3, new Val(tot_gap_bytes, TYPE_COUNT)); - - return r; - %} - -## Returns the number bytes that a value occupies in memory. -## -## v: The value -## -## Returns: The number of bytes that *v* occupies. -function val_size%(v: any%): count - %{ - return new Val(v->MemoryAllocation(), TYPE_COUNT); - %} - -## Generates a table of the size of all global variables. The table index is -## the variable name and the value the variable size in bytes. -## -## Returns: A table that maps variable names to their sizes. -## -## .. bro:see:: global_ids -function global_sizes%(%): var_sizes - %{ - TableVal* sizes = new TableVal(var_sizes); - PDict(ID)* globals = global_scope()->Vars(); - IterCookie* c = globals->InitForIteration(); - - ID* id; - while ( (id = globals->NextEntry(c)) ) - if ( id->HasVal() && ! id->IsInternalGlobal() ) - { - Val* id_name = new StringVal(id->Name()); - Val* id_size = new Val(id->ID_Val()->MemoryAllocation(), - TYPE_COUNT); - sizes->Assign(id_name, id_size); - Unref(id_name); - } - - return sizes; - %} - -## Generates a table with information about all global identifiers. The table -## value is a record containing the type name of the identifier, whether it is -## exported, a constant, an enum constant, redefinable, and its value (if it -## has one). -## -## Returns: A table that maps identifier names to information about them. -## -## .. bro:see:: global_sizes -function global_ids%(%): id_table - %{ - TableVal* ids = new TableVal(id_table); - PDict(ID)* globals = global_scope()->Vars(); - IterCookie* c = globals->InitForIteration(); - - ID* id; - while ( (id = globals->NextEntry(c)) ) - { - if ( id->IsInternalGlobal() ) - continue; - - RecordVal* rec = new RecordVal(script_id); - rec->Assign(0, new StringVal(type_name(id->Type()->Tag()))); - rec->Assign(1, new Val(id->IsExport(), TYPE_BOOL)); - rec->Assign(2, new Val(id->IsConst(), TYPE_BOOL)); - rec->Assign(3, new Val(id->IsEnumConst(), TYPE_BOOL)); - rec->Assign(4, new Val(id->IsRedefinable(), TYPE_BOOL)); - - if ( id->HasVal() ) - { - Val* val = id->ID_Val(); - Ref(val); - rec->Assign(5, val); - } - - Val* id_name = new StringVal(id->Name()); - ids->Assign(id_name, rec); - Unref(id_name); - } - - return ids; - %} - -## Generates meta data about a record fields. The returned information -## includes the field name, whether it is logged, its value (if it has one), -## and its default value (if specified). -## -## rec: The record to inspect. -## -## Returns: A table that describes the fields of a record. -function record_fields%(rec: any%): record_field_table - %{ - TableVal* fields = new TableVal(record_field_table); - - RecordVal* rv = rec->AsRecordVal(); - RecordType* rt = rv->Type()->AsRecordType(); - - if ( rt->Tag() != TYPE_RECORD ) - { - reporter->Error("non-record passed to record_fields"); - return fields; - } - - for ( int i = 0; i < rt->NumFields(); ++i ) - { - BroType* ft = rt->FieldType(i); - TypeDecl* fd = rt->FieldDecl(i); - Val* fv = rv->Lookup(i); - - if ( fv ) - Ref(fv); - - bool logged = (fd->attrs && fd->FindAttr(ATTR_LOG) != 0); - - RecordVal* nr = new RecordVal(record_field); - nr->Assign(0, new StringVal(type_name(rt->Tag()))); - nr->Assign(1, new Val(logged, TYPE_BOOL)); - nr->Assign(2, fv); - nr->Assign(3, rt->FieldDefault(i)); - - Val* field_name = new StringVal(rt->FieldName(i)); - fields->Assign(field_name, nr); - Unref(field_name); - } - - return fields; - %} - -%%{ -#include "Anon.h" -%%} - -## Preserve prefix as original one in anonymization. -## .. todo: Currently broken. -function preserve_prefix%(a: addr, width: count%): any - %{ - AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50]; - if ( ip_anon ) - { -#ifdef BROv6 - if ( ! is_v4_addr(a) ) - builtin_error("preserve_prefix() not supported for IPv6 addresses"); - else - ip_anon->PreservePrefix(a[3], width); -#else - ip_anon->PreservePrefix(a, width); -#endif - } - - - return 0; - %} - -## .. todo: Currently broken. -function preserve_subnet%(a: subnet%): any - %{ - DEBUG_MSG("%s/%d\n", dotted_addr(a->AsAddr()), a->Width()); - AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50]; - if ( ip_anon ) - { -#ifdef BROv6 - if ( ! is_v4_addr(a->AsAddr()) ) - builtin_error("preserve_subnet() not supported for IPv6 addresses"); - else - ip_anon->PreservePrefix(a->AsAddr()[3], a->Width()); -#else - ip_anon->PreservePrefix(a->AsAddr(), a->Width()); -#endif - } - - return 0; - %} - -## Anonymize an IP address. -## .. todo: Currently broken. -function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr - %{ - int anon_class = cl->InternalInt(); - if ( anon_class < 0 || anon_class >= NUM_ADDR_ANONYMIZATION_CLASSES ) - builtin_error("anonymize_addr(): invalid ip addr anonymization class"); - -#ifdef BROv6 - if ( ! is_v4_addr(a) ) - { - builtin_error("anonymize_addr() not supported for IPv6 addresses"); - return 0; - } - else - return new AddrVal(anonymize_ip(a[3], - (enum ip_addr_anonymization_class_t) anon_class)); -#else - return new AddrVal(anonymize_ip(a, - (enum ip_addr_anonymization_class_t) anon_class)); -#endif - %} - %%{ static void hash_md5_val(val_list& vlist, unsigned char digest[16]) { @@ -2694,6 +741,1899 @@ function srand%(seed: count%): any return 0; %} +%%{ +#include +%%} + +## Send a string to syslog. +## +## s: The string to log via syslog +function syslog%(s: string%): any + %{ + reporter->Syslog("%s", s->CheckString()); + return 0; + %} + +%%{ +extern "C" { +#include +} +%%} + +## Determines the MIME type of a piece of data using ``libmagic``. +## +## data: The data to find the MIME type for. +## +## return_mime: If true, the function returns a short MIME type string (e.g., +## ``text/plain`` instead of a more elaborate textual description. +## +## Returns: The MIME type of *data*. +function identify_data%(data: string, return_mime: bool%): string + %{ + const char* descr = ""; + + static magic_t magic_mime = 0; + static magic_t magic_descr = 0; + + magic_t* magic = return_mime ? &magic_mime : &magic_descr; + + if( ! *magic ) + { + *magic = magic_open(return_mime ? MAGIC_MIME : MAGIC_NONE); + + if ( ! *magic ) + { + reporter->Error("can't init libmagic: %s", magic_error(*magic)); + return new StringVal(""); + } + + if ( magic_load(*magic, 0) < 0 ) + { + reporter->Error("can't load magic file: %s", magic_error(*magic)); + magic_close(*magic); + *magic = 0; + return new StringVal(""); + } + } + + descr = magic_buffer(*magic, data->Bytes(), data->Len()); + + return new StringVal(descr); + %} + +%%{ +#include +static map entropy_states; +%%} + +## Performs an `entropy `_ test on the given +## data. +## +## data: The data to compute the entropy for. +## +## Returns: The result of the entropy test, which contains the following +## fields. +## +## - ``entropy``: The information density expressed as a number of +## bits per character. +## +## - ``chi_square``: The chi-square test value expressed as an +## absolute number and a percentage which indicates how +## frequently a truly random sequence would exceed the value +## calculated, i.e., the degree to which the sequence tested is +## suspected of being non-random. +## +## If the percentage is greater than 99% or less than 1%, the +## sequence is almost certainly not random. If the percentage is +## between 99% and 95% or between 1% and 5%, the sequence is +## suspect. Percentages between 90\% and 95\% and 5\% and 10\% +## indicate the sequence is "almost suspect." +## +## - ``mean``: The arithmetic mean of all the bytes. If the data +## are close to random, it should be around 127.5. +## +## - ``monte_carlo_pi``: Each successive sequence of six bytes is +## used as 24-bit *x* and *y* coordinates within a square. If +## the distance of the randomly-generated point is less than the +## radius of a circle inscribed within the square, the six-byte +## sequence is considered a "hit." The percentage of hits can +## be used to calculate the value of pi. For very large streams +## the value will approach the correct value of pi if the +## sequence is close to random. +## +## - ``serial_correlation``: This quantity measures the extent to +## which each byte in the file depends upon the previous byte. +## For random sequences this value will be close to zero. +## +## .. bro:see:: entropy_test_init entropy_test_add entropy_test_finish +function find_entropy%(data: string%): entropy_test_result + %{ + double montepi, scc, ent, mean, chisq; + montepi = scc = ent = mean = chisq = 0.0; + RecordVal* ent_result = new RecordVal(entropy_test_result); + RandTest *rt = new RandTest(); + + rt->add((char*) data->Bytes(), data->Len()); + rt->end(&ent, &chisq, &mean, &montepi, &scc); + delete rt; + + ent_result->Assign(0, new Val(ent, TYPE_DOUBLE)); + ent_result->Assign(1, new Val(chisq, TYPE_DOUBLE)); + ent_result->Assign(2, new Val(mean, TYPE_DOUBLE)); + ent_result->Assign(3, new Val(montepi, TYPE_DOUBLE)); + ent_result->Assign(4, new Val(scc, TYPE_DOUBLE)); + return ent_result; + %} + +## Initializes data structures for incremental entropy calculation. +## +## index: An arbitrary unique value per distinct computation. +## +## Returns: True on success. +## +## .. bro:see:: find_entropy entropy_test_add entropy_test_finish +function entropy_test_init%(index: any%): bool + %{ + BroString* s = convert_index_to_string(index); + int status = 0; + + if ( entropy_states.count(*s) < 1 ) + { + entropy_states[*s] = new RandTest(); + status = 1; + } + + delete s; + return new Val(status, TYPE_BOOL); + %} + +## Adds data to an incremental entropy calculation. Before using this function, +## one needs to invoke :bro:id:`entropy_test_init`. +## +## data: The data to add to the entropy calculation. +## +## index: An arbitrary unique value that identifies a particular entropy +## computation. +## +## Returns: True on success. +## +## .. bro:see:: find_entropy entropy_test_add entropy_test_finish +function entropy_test_add%(index: any, data: string%): bool + %{ + BroString* s = convert_index_to_string(index); + int status = 0; + + if ( entropy_states.count(*s) > 0 ) + { + entropy_states[*s]->add((char*) data->Bytes(), data->Len()); + status = 1; + } + + delete s; + return new Val(status, TYPE_BOOL); + %} + +## Finishes an incremental entropy calculation. Before using this function, +## one needs to initialize the computation with :bro:id:`entropy_test_init` and +## add data to it via :bro:id:`entropy_test_add`. +## +## index: An arbitrary unique value that identifies a particular entropy +## computation. +## +## Returns: The result of the entropy test. See :bro:id:`find_entropy` for a +## description of the individual components. +## +## .. bro:see:: find_entropy entropy_test_init entropy_test_add +function entropy_test_finish%(index: any%): entropy_test_result + %{ + BroString* s = convert_index_to_string(index); + double montepi, scc, ent, mean, chisq; + montepi = scc = ent = mean = chisq = 0.0; + RecordVal* ent_result = new RecordVal(entropy_test_result); + + if ( entropy_states.count(*s) > 0 ) + { + RandTest *rt = entropy_states[*s]; + rt->end(&ent, &chisq, &mean, &montepi, &scc); + entropy_states.erase(*s); + delete rt; + } + + ent_result->Assign(0, new Val(ent, TYPE_DOUBLE)); + ent_result->Assign(1, new Val(chisq, TYPE_DOUBLE)); + ent_result->Assign(2, new Val(mean, TYPE_DOUBLE)); + ent_result->Assign(3, new Val(montepi, TYPE_DOUBLE)); + ent_result->Assign(4, new Val(scc, TYPE_DOUBLE)); + + delete s; + return ent_result; + %} + +## Creates an identifier that is unique with high probability. +## +## prefix: A custom string prepended to the result. +## +## .. bro:see:: unique_id_from +function unique_id%(prefix: string%) : string + %{ + char tmp[20]; + uint64 uid = calculate_unique_id(UID_POOL_DEFAULT_SCRIPT); + return new StringVal(uitoa_n(uid, tmp, sizeof(tmp), 62, prefix->CheckString())); + %} + +## Creates an identifier that is unique with high probability. +## +## pool: A seed for determinism. +## +## prefix: A custom string prepended to the result. +## +## .. bro:see:: unique_id +function unique_id_from%(pool: int, prefix: string%) : string + %{ + pool += UID_POOL_CUSTOM_SCRIPT; // Make sure we don't conflict with internal pool. + + char tmp[20]; + uint64 uid = calculate_unique_id(pool); + return new StringVal(uitoa_n(uid, tmp, sizeof(tmp), 62, prefix->CheckString())); + %} + +# =========================================================================== +# +# Generic Programming +# +# =========================================================================== + +## Removes all elements from a set or table. +## +## v: The set or table +## +## Returns: The cleared set/table or 0 if *v* is not a set/table type. +function clear_table%(v: any%): any + %{ + if ( v->Type()->Tag() == TYPE_TABLE ) + v->AsTableVal()->RemoveAll(); + else + builtin_error("clear_table() requires a table/set argument"); + + return 0; + %} + +## Returns the number of elements in a container. This function works with all +## container types, i.e., sets, tables, and vectors. +## +## v: The container whose elements are counted. +## +## Returns: The number of elements in *v*. +function length%(v: any%): count + %{ + TableVal* tv = v->Type()->Tag() == TYPE_TABLE ? v->AsTableVal() : 0; + + if ( tv ) + return new Val(tv->Size(), TYPE_COUNT); + + else if ( v->Type()->Tag() == TYPE_VECTOR ) + return new Val(v->AsVectorVal()->Size(), TYPE_COUNT); + + else + { + builtin_error("length() requires a table/set/vector argument"); + return new Val(0, TYPE_COUNT); + } + %} + +## Checks whether two objects reference the same internal object. This function +## uses equality comparison of C++ raw pointer values to determine if the two +## objects are the same. +## +## o1: The first object. +## +## o2: The second object. +## +## Returns: True if *o1* and *o2* are equal. +function same_object%(o1: any, o2: any%): bool + %{ + return new Val(o1 == o2, TYPE_BOOL); + %} + +## Returns the number bytes that a value occupies in memory. +## +## v: The value +## +## Returns: The number of bytes that *v* occupies. +function val_size%(v: any%): count + %{ + return new Val(v->MemoryAllocation(), TYPE_COUNT); + %} + +## Resizes a vector. +## +## aggr: The vector instance. +## +## newsize: The new size of *aggr*. +## +## Returns: The old size of *aggr* and 0 if *aggr* is not a :bro:type:`vector`. +function resize%(aggr: any, newsize: count%) : count + %{ + if ( aggr->Type()->Tag() != TYPE_VECTOR ) + { + builtin_error("resize() operates on vectors"); + return 0; + } + + return new Val(aggr->AsVectorVal()->Resize(newsize), TYPE_COUNT); + %} + +## Tests whether a boolean vector (``vector of bool``) has *any* true +## element. +## +## v: The boolean vector instance. +## +## Returns: True if any element in *v* is true. +## +## :bro:see:: all_set +function any_set%(v: any%) : bool + %{ + if ( v->Type()->Tag() != TYPE_VECTOR || + v->Type()->YieldType()->Tag() != TYPE_BOOL ) + { + builtin_error("any_set() requires vector of bool"); + return new Val(false, TYPE_BOOL); + } + + VectorVal* vv = v->AsVectorVal(); + for ( unsigned int i = 0; i < vv->Size(); ++i ) + if ( vv->Lookup(i) && vv->Lookup(i)->AsBool() ) + return new Val(true, TYPE_BOOL); + + return new Val(false, TYPE_BOOL); + %} + +## Tests whether *all* elements of a boolean vector (``vector of bool``) are +## true. +## +## v: The boolean vector instance. +## +## Returns: True iff all elements in *v* are true. +## +## :bro:see:: any_set +## +## .. note:: +## +## Missing elements count as false. +function all_set%(v: any%) : bool + %{ + if ( v->Type()->Tag() != TYPE_VECTOR || + v->Type()->YieldType()->Tag() != TYPE_BOOL ) + { + builtin_error("all_set() requires vector of bool"); + return new Val(false, TYPE_BOOL); + } + + VectorVal* vv = v->AsVectorVal(); + for ( unsigned int i = 0; i < vv->Size(); ++i ) + if ( ! vv->Lookup(i) || ! vv->Lookup(i)->AsBool() ) + return new Val(false, TYPE_BOOL); + + return new Val(true, TYPE_BOOL); + %} + +%%{ +static Func* sort_function_comp = 0; +static Val** index_map = 0; // used for indirect sorting to support order() + +bool sort_function(Val* a, Val* b) + { + // Sort missing values as "high". + if ( ! a ) + return 0; + if ( ! b ) + return 1; + + val_list sort_func_args; + sort_func_args.append(a->Ref()); + sort_func_args.append(b->Ref()); + + Val* result = sort_function_comp->Call(&sort_func_args); + int int_result = result->CoerceToInt(); + Unref(result); + + sort_func_args.remove_nth(1); + sort_func_args.remove_nth(0); + + return int_result < 0; + } + +bool indirect_sort_function(int a, int b) + { + return sort_function(index_map[a], index_map[b]); + } + +bool int_sort_function (Val* a, Val* b) + { + if ( ! a ) + return 0; + if ( ! b ) + return 1; + + int ia = a->CoerceToInt(); + int ib = b->CoerceToInt(); + + return ia < ib; + } + +bool indirect_int_sort_function(int a, int b) + { + return int_sort_function(index_map[a], index_map[b]); + } +%%} + +## Sorts a vector in place. The second argument is a comparison function that +## takes two arguments: if the vector type is \verb|vector of T|, then the +## comparison function must be ``function(a: T, b: T): bool``, which returns +## ``a < b`` for some type-specific notion of the less-than operator. +## +## v: The vector instance to sort. +## +## Returns: The original vector. +## +## .. bro:see:: order +function sort%(v: any, ...%) : any + %{ + v->Ref(); // we always return v + + if ( v->Type()->Tag() != TYPE_VECTOR ) + { + builtin_error("sort() requires vector"); + return v; + } + + BroType* elt_type = v->Type()->YieldType(); + Func* comp = 0; + + if ( @ARG@.length() > 2 ) + builtin_error("sort() called with extraneous argument"); + + if ( @ARG@.length() == 2 ) + { + Val* comp_val = @ARG@[1]; + if ( ! IsFunc(comp_val->Type()->Tag()) ) + { + builtin_error("second argument to sort() needs to be comparison function"); + return v; + } + + comp = comp_val->AsFunc(); + } + + if ( ! comp && ! IsIntegral(elt_type->Tag()) ) + builtin_error("comparison function required for sort() with non-integral types"); + + vector& vv = *v->AsVector(); + + if ( comp ) + { + FuncType* comp_type = comp->FType()->AsFuncType(); + if ( comp_type->YieldType()->Tag() != TYPE_INT || + ! comp_type->ArgTypes()->AllMatch(elt_type, 0) ) + { + builtin_error("invalid comparison function in call to sort()"); + return v; + } + + sort_function_comp = comp; + + sort(vv.begin(), vv.end(), sort_function); + } + else + sort(vv.begin(), vv.end(), int_sort_function); + + return v; + %} + +## Returns the order of the elements in a vector according to some +## comparison function. See :bro:id:`sort` for details about the comparison +## function. +## +## v: The vector whose order to compute. +## +## Returns: A ``vector of count`` with the indices of the ordered elements. +## +## .. bro:see:: sort +function order%(v: any, ...%) : index_vec + %{ + VectorVal* result_v = + new VectorVal(new VectorType(base_type(TYPE_COUNT))); + + if ( v->Type()->Tag() != TYPE_VECTOR ) + { + builtin_error("order() requires vector"); + return result_v; + } + + BroType* elt_type = v->Type()->YieldType(); + Func* comp = 0; + + if ( @ARG@.length() > 2 ) + builtin_error("order() called with extraneous argument"); + + if ( @ARG@.length() == 2 ) + { + Val* comp_val = @ARG@[1]; + if ( ! IsFunc(comp_val->Type()->Tag()) ) + { + builtin_error("second argument to order() needs to be comparison function"); + return v; + } + + comp = comp_val->AsFunc(); + } + + if ( ! comp && ! IsIntegral(elt_type->Tag()) ) + builtin_error("comparison function required for sort() with non-integral types"); + + vector& vv = *v->AsVector(); + int n = vv.size(); + + // Set up initial mapping of indices directly to corresponding + // elements. We stay zero-based until after the sorting. + vector ind_vv(n); + index_map = new Val*[n]; + int i; + for ( i = 0; i < n; ++i ) + { + ind_vv[i] = i; + index_map[i] = vv[i]; + } + + if ( comp ) + { + FuncType* comp_type = comp->FType()->AsFuncType(); + if ( comp_type->YieldType()->Tag() != TYPE_INT || + ! comp_type->ArgTypes()->AllMatch(elt_type, 0) ) + { + builtin_error("invalid comparison function in call to sort()"); + return v; + } + + sort_function_comp = comp; + + sort(ind_vv.begin(), ind_vv.end(), indirect_sort_function); + } + else + sort(ind_vv.begin(), ind_vv.end(), indirect_int_sort_function); + + delete [] index_map; + index_map = 0; + + // Now spin through ind_vv to read out the rearrangement, + // adjusting indices as we do so. + for ( i = 0; i < n; ++i ) + { + int ind = ind_vv[i]; + result_v->Assign(i, new Val(ind, TYPE_COUNT), 0); + } + + return result_v; + %} + +# =========================================================================== +# +# String Processing +# +# =========================================================================== + +## Returns the concatenation of the string representation of its arguments. The +## arguments can be of any type. For example, ``cat("foo", 3, T)`` returns +## ``"foo3T"``. +## +## Returns: A string concatentation of all arguments. +function cat%(...%): string + %{ + ODesc d; + loop_over_list(@ARG@, i) + @ARG@[i]->Describe(&d); + + BroString* s = new BroString(1, d.TakeBytes(), d.Len()); + s->SetUseFreeToDelete(true); + + return new StringVal(s); + %} + +## Concatenates all arguments, with a separator placed between each one. This +## function is similar to :bro:id:`cat`, but places a separator between each +## given argument. If any of the variable arguments is an empty string it is +## replaced by a given default string instead. +## +## sep: The separator to place betwen each argument. +## +## def: The default string to use when an argument is the empty string. +## +## Returns: A concatenation of all arguments with *sep* between each one and +## empty strings replaced with *def*. +## +## .. bro:see:: cat string_cat cat_string_array cat_string_array_n +function cat_sep%(sep: string, def: string, ...%): string + %{ + ODesc d; + int pre_size = 0; + + loop_over_list(@ARG@, i) + { + // Skip named parameters. + if ( i < 2 ) + continue; + + if ( i > 2 ) + d.Add(sep->CheckString(), 0); + + Val* v = @ARG@[i]; + if ( v->Type()->Tag() == TYPE_STRING && ! v->AsString()->Len() ) + v = def; + + v->Describe(&d); + } + + BroString* s = new BroString(1, d.TakeBytes(), d.Len()); + s->SetUseFreeToDelete(true); + + return new StringVal(s); + %} + +## Produces a formatted string à la ``printf``. The first argument is the +## *format string* and specifies how subsequent arguments are converted for +## output. It is composed of zero or more directives: ordinary characters (not +## ``%``), which are copied unchanged to the output, and conversion +## specifications, each of which fetches zero or more subsequent arguments. +## Conversion specifications begin with ``%`` and the arguments must properly +## correspond to the specifier. After the ``%``, the following characters +## may appear in sequence: +## +## - ``%``: Literal ``%`` +## +## - ``-``: Left-align field +## +## - ``[0-9]+``: The field width (< 128) +## +## - ``.``: Precision of floating point specifiers ``[efg]`` (< 128) +## +## - ``A``: Escape NUL bytes, i.e., replace ``0`` with ``\0`` +## +## - ``[DTdxsefg]``: Format specifier +## +## - ``[DT]``: ISO timestamp with microsecond precision +## +## - ``d``: Signed/Unsigned integer (using C-style ``%lld|``/``%llu`` +## for ``int``/``count``) +## +## - ``x``: Unsigned hexadecimal (using C-style ``%llx``); +## addresses/ports are converted to host-byte order +## +## - ``s``: Escaped string +## +## - ``[efg]``: Double +## +## Returns: Given no arguments, :bro:id:`fmt` returns an empty string. Given a +## non-string first argument, :bro:id:`fmt` returns the concatenation +## of all its arguments, per :bro:id:`cat`. Finally, given the wrong +## number of additional arguments for the given format specifier, +## :bro:id:`fmt` generates a run-time error. +## +## .. bro:see:: cat cat_sep string_cat cat_string_array cat_string_array_n +function fmt%(...%): string + %{ + if ( @ARGC@ == 0 ) + return new StringVal(""); + + Val* fmt_v = @ARG@[0]; + if ( fmt_v->Type()->Tag() != TYPE_STRING ) + return bro_cat(frame, @ARGS@); + + const char* fmt = fmt_v->AsString()->CheckString(); + ODesc d; + int n = 0; + + while ( next_fmt(fmt, @ARGS@, &d, n) ) + ; + + if ( n < @ARGC@ - 1 ) + builtin_error("too many arguments for format", fmt_v); + + else if ( n >= @ARGC@ ) + builtin_error("too few arguments for format", fmt_v); + + BroString* s = new BroString(1, d.TakeBytes(), d.Len()); + s->SetUseFreeToDelete(true); + + return new StringVal(s); + %} + +# =========================================================================== +# +# Math +# +# =========================================================================== + +## Chops off any decimal digits of the given double, i.e., computes the +## "floor" of it. For example, ``floor(3.14)`` returns ``3.0``. +## +## d: The :bro:type:`double` to manipulate. +## +## Returns: The next lowest integer of *d* as :bro:type:`double`. +## +## .. bro:see:: sqrt exp ln log10 +function floor%(d: double%): double + %{ + return new Val(floor(d), TYPE_DOUBLE); + %} + +## Computes the square root of a :bro:type:`double`. +## +## x: The number to compute the square root of. +## +## Returns: The square root of *x*. +## +## .. bro:see:: floor exp ln log10 +function sqrt%(x: double%): double + %{ + if ( x < 0 ) + { + reporter->Error("negative sqrt argument"); + return new Val(-1.0, TYPE_DOUBLE); + } + + return new Val(sqrt(x), TYPE_DOUBLE); + %} + +## Computes the exponential function. +## +## d: The argument to the exponential function. +## +## Returns: *e* to the power of *d*. +## +## .. bro:see:: floor sqrt ln log10 +function exp%(d: double%): double + %{ + return new Val(exp(d), TYPE_DOUBLE); + %} + +## Computes the natural logarithm of a number. +## +## d: The argument to the logarithm. +## +## Returns: The natural logarithm of *d*. +## +## .. bro:see:: exp floor sqrt log10 +function ln%(d: double%): double + %{ + return new Val(log(d), TYPE_DOUBLE); + %} + +## Computes the common logarithm of a number. +## +## d: The argument to the logarithm. +## +## Returns: The common logarithm of *d*. +## +## .. bro:see:: exp floor sqrt ln +function log10%(d: double%): double + %{ + return new Val(log10(d), TYPE_DOUBLE); + %} + +# =========================================================================== +# +# Introspection +# +# =========================================================================== + +## Determines whether *c* has been received externally. For example, +## Broccoli or the Time Machine can send packets to Bro via a mechanism that +## one step lower than sending events. This function checks whether the packets +## of a connection stem from one of these external *packet sources*. +## +## c: The connection to test. +## +## Returns: True if *c* has been received externally. +function is_external_connection%(c: connection%) : bool + %{ + return new Val(c && c->IsExternal(), TYPE_BOOL); + %} + +## Returns the ID of the analyzer which raised the current event. +## +## Returns: The ID of the analyzer which raised hte current event, or 0 if +## none. +function current_analyzer%(%) : count + %{ + return new Val(mgr.CurrentAnalyzer(), TYPE_COUNT); + %} + +## Returns Bro's process ID. +## +## Returns: Bro's process ID. +function getpid%(%) : count + %{ + return new Val(getpid(), TYPE_COUNT); + %} + +%%{ +extern const char* bro_version(); +%%} + +## Returns the Bro version string. +## +## Returns: Bro's version, e.g., 2.0-beta-47-debug. +function bro_version%(%): string + %{ + return new StringVal(bro_version()); + %} + +## Converts a record type name to a vector of strings, where each element is +## the name of a record field. Nested records are flattened. +## +## rt: The name of the record type. +## +## Returns: A string vector with the field names of *rt*. +function record_type_to_vector%(rt: string%): string_vec + %{ + VectorVal* result = + new VectorVal(internal_type("string_vec")->AsVectorType()); + + RecordType *type = internal_type(rt->CheckString())->AsRecordType(); + + if ( type ) + { + for ( int i = 0; i < type->NumFields(); ++i ) + { + StringVal* val = new StringVal(type->FieldName(i)); + result->Assign(i+1, val, 0); + } + } + + return result; + %} + +## Returns the type name of an arbitrary Bro variable. +## +## t: An arbitrary object. +## +## Returns: The type name of *t*. +function type_name%(t: any%): string + %{ + ODesc d; + t->Type()->Describe(&d); + + BroString* s = new BroString(1, d.TakeBytes(), d.Len()); + s->SetUseFreeToDelete(true); + + return new StringVal(s); + %} + +## Checks whether Bro reads traffic from one or more network interfaces (as +## opposed to from a network trace in a file). Note that this function returns +## true even after Bro has stopped reading network traffic, for example due to +## receiving a termination signal. +## +## Returns: True if reading traffic from a network interface. +## +## .. bro:see:: reading_traces +function reading_live_traffic%(%): bool + %{ + return new Val(reading_live, TYPE_BOOL); + %} + +## Checks whether Bro reads traffic from a trace file (as opposed to from a +## network interface). +## +## Returns: True if reading traffic from a network trace. +## +## .. bro:see:: reading_live_traffic +function reading_traces%(%): bool + %{ + return new Val(reading_traces, TYPE_BOOL); + %} + +## Returns statistics about the number of packets *(i)* received by Bro, +## *(ii)* dropped, and *(iii)* seen on the link (not always available). +## +## Returns: A record of packet statistics. +## +## .. bro:see:: do_profiling +## resource_usage +## get_matcher_stats +## dump_rule_stats +## get_gap_summary +function net_stats%(%): NetStats + %{ + unsigned int recv = 0; + unsigned int drop = 0; + unsigned int link = 0; + + loop_over_list(pkt_srcs, i) + { + PktSrc* ps = pkt_srcs[i]; + + struct PktSrc::Stats stat; + ps->Statistics(&stat); + recv += stat.received; + drop += stat.dropped; + link += stat.link; + } + + RecordVal* ns = new RecordVal(net_stats); + ns->Assign(0, new Val(recv, TYPE_COUNT)); + ns->Assign(1, new Val(drop, TYPE_COUNT)); + ns->Assign(2, new Val(link, TYPE_COUNT)); + + return ns; + %} + +## Returns Bro process statistics, such as real/user/sys CPU time, memory +## usage, page faults, number of TCP/UDP/ICMP connections, timers, and events +## queued/dispatched. +## +## Returns: A record with resource usage statistics. +## +## .. bro:see:: do_profiling +## net_stats +## get_matcher_stats +## dump_rule_stats +## get_gap_summary +function resource_usage%(%): bro_resources + %{ + struct rusage r; + + if ( getrusage(RUSAGE_SELF, &r) < 0 ) + reporter->InternalError("getrusage() failed in bro_resource_usage()"); + + double elapsed_time = current_time() - bro_start_time; + + double user_time = + double(r.ru_utime.tv_sec) + double(r.ru_utime.tv_usec) / 1e6; + double system_time = + double(r.ru_stime.tv_sec) + double(r.ru_stime.tv_usec) / 1e6; + + RecordVal* res = new RecordVal(bro_resources); + int n = 0; + + res->Assign(n++, new StringVal(bro_version())); + +#ifdef DEBUG + res->Assign(n++, new Val(1, TYPE_COUNT)); +#else + res->Assign(n++, new Val(0, TYPE_COUNT)); +#endif + + res->Assign(n++, new Val(bro_start_time, TYPE_TIME)); + + res->Assign(n++, new IntervalVal(elapsed_time, Seconds)); + res->Assign(n++, new IntervalVal(user_time, Seconds)); + res->Assign(n++, new IntervalVal(system_time, Seconds)); + + unsigned int total_mem; + get_memory_usage(&total_mem, 0); + res->Assign(n++, new Val(unsigned(total_mem), TYPE_COUNT)); + + res->Assign(n++, new Val(unsigned(r.ru_minflt), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(r.ru_majflt), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(r.ru_nswap), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(r.ru_inblock), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(r.ru_oublock), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(r.ru_nivcsw), TYPE_COUNT)); + + SessionStats s; + if ( sessions ) + sessions->GetStats(s); + +#define ADD_STAT(x) \ + res->Assign(n++, new Val(unsigned(sessions ? x : 0), TYPE_COUNT)); + + ADD_STAT(s.num_TCP_conns); + ADD_STAT(s.num_UDP_conns); + ADD_STAT(s.num_ICMP_conns); + ADD_STAT(s.num_fragments); + ADD_STAT(s.num_packets); + ADD_STAT(s.num_timers); + ADD_STAT(s.num_events_queued); + ADD_STAT(s.num_events_dispatched); + ADD_STAT(s.max_TCP_conns); + ADD_STAT(s.max_UDP_conns); + ADD_STAT(s.max_ICMP_conns); + ADD_STAT(s.max_fragments); + ADD_STAT(s.max_timers); + + return res; + %} + +## Returns statistics about the regular expression engine, such as the number +## of distinct matchers, DFA states, DFA state transitions, memory usage of +## DFA states, cache hits/misses, and average number of NFA states across all +## matchers. +## +## Returns: A record with matcher statistics. +## +## .. bro:see:: do_profiling +## net_stats +## resource_usage +## dump_rule_stats +## get_gap_summary +function get_matcher_stats%(%): matcher_stats + %{ + RuleMatcher::Stats s; + memset(&s, 0, sizeof(s)); + + if ( rule_matcher ) + rule_matcher->GetStats(&s); + + RecordVal* r = new RecordVal(matcher_stats); + r->Assign(0, new Val(s.matchers, TYPE_COUNT)); + r->Assign(1, new Val(s.dfa_states, TYPE_COUNT)); + r->Assign(2, new Val(s.computed, TYPE_COUNT)); + r->Assign(3, new Val(s.mem, TYPE_COUNT)); + r->Assign(4, new Val(s.hits, TYPE_COUNT)); + r->Assign(5, new Val(s.misses, TYPE_COUNT)); + r->Assign(6, new Val(s.avg_nfa_states, TYPE_COUNT)); + + return r; + %} + +## Returns statistics about TCP gaps. +## +## Returns: A record with TCP gap statistics. +## +## .. bro:see:: do_profiling +## net_stats +## resource_usage +## dump_rule_stats +## get_matcher_stats +function get_gap_summary%(%): gap_info + %{ + RecordVal* r = new RecordVal(gap_info); + r->Assign(0, new Val(tot_ack_events, TYPE_COUNT)); + r->Assign(1, new Val(tot_ack_bytes, TYPE_COUNT)); + r->Assign(2, new Val(tot_gap_events, TYPE_COUNT)); + r->Assign(3, new Val(tot_gap_bytes, TYPE_COUNT)); + + return r; + %} + +## Generates a table of the size of all global variables. The table index is +## the variable name and the value the variable size in bytes. +## +## Returns: A table that maps variable names to their sizes. +## +## .. bro:see:: global_ids +function global_sizes%(%): var_sizes + %{ + TableVal* sizes = new TableVal(var_sizes); + PDict(ID)* globals = global_scope()->Vars(); + IterCookie* c = globals->InitForIteration(); + + ID* id; + while ( (id = globals->NextEntry(c)) ) + if ( id->HasVal() && ! id->IsInternalGlobal() ) + { + Val* id_name = new StringVal(id->Name()); + Val* id_size = new Val(id->ID_Val()->MemoryAllocation(), + TYPE_COUNT); + sizes->Assign(id_name, id_size); + Unref(id_name); + } + + return sizes; + %} + +## Generates a table with information about all global identifiers. The table +## value is a record containing the type name of the identifier, whether it is +## exported, a constant, an enum constant, redefinable, and its value (if it +## has one). +## +## Returns: A table that maps identifier names to information about them. +## +## .. bro:see:: global_sizes +function global_ids%(%): id_table + %{ + TableVal* ids = new TableVal(id_table); + PDict(ID)* globals = global_scope()->Vars(); + IterCookie* c = globals->InitForIteration(); + + ID* id; + while ( (id = globals->NextEntry(c)) ) + { + if ( id->IsInternalGlobal() ) + continue; + + RecordVal* rec = new RecordVal(script_id); + rec->Assign(0, new StringVal(type_name(id->Type()->Tag()))); + rec->Assign(1, new Val(id->IsExport(), TYPE_BOOL)); + rec->Assign(2, new Val(id->IsConst(), TYPE_BOOL)); + rec->Assign(3, new Val(id->IsEnumConst(), TYPE_BOOL)); + rec->Assign(4, new Val(id->IsRedefinable(), TYPE_BOOL)); + + if ( id->HasVal() ) + { + Val* val = id->ID_Val(); + Ref(val); + rec->Assign(5, val); + } + + Val* id_name = new StringVal(id->Name()); + ids->Assign(id_name, rec); + Unref(id_name); + } + + return ids; + %} + +## Returns the value of a global identifier. +## +## id: The global identifier. +## +## Returns the value of *id*. If *id* does not describe a valid identifier, the +## function returns the string ``""`` or ``""``. +function lookup_ID%(id: string%) : any + %{ + ID* i = global_scope()->Lookup(id->CheckString()); + if ( ! i ) + return new StringVal(""); + + if ( ! i->ID_Val() ) + return new StringVal(""); + + return i->ID_Val()->Ref(); + %} + +## Generates meta data about a record fields. The returned information +## includes the field name, whether it is logged, its value (if it has one), +## and its default value (if specified). +## +## rec: The record to inspect. +## +## Returns: A table that describes the fields of a record. +function record_fields%(rec: any%): record_field_table + %{ + TableVal* fields = new TableVal(record_field_table); + + RecordVal* rv = rec->AsRecordVal(); + RecordType* rt = rv->Type()->AsRecordType(); + + if ( rt->Tag() != TYPE_RECORD ) + { + reporter->Error("non-record passed to record_fields"); + return fields; + } + + for ( int i = 0; i < rt->NumFields(); ++i ) + { + BroType* ft = rt->FieldType(i); + TypeDecl* fd = rt->FieldDecl(i); + Val* fv = rv->Lookup(i); + + if ( fv ) + Ref(fv); + + bool logged = (fd->attrs && fd->FindAttr(ATTR_LOG) != 0); + + RecordVal* nr = new RecordVal(record_field); + nr->Assign(0, new StringVal(type_name(rt->Tag()))); + nr->Assign(1, new Val(logged, TYPE_BOOL)); + nr->Assign(2, fv); + nr->Assign(3, rt->FieldDefault(i)); + + Val* field_name = new StringVal(rt->FieldName(i)); + fields->Assign(field_name, nr); + Unref(field_name); + } + + return fields; + %} + +## Enables detailed collections of statistics about CPU/memory usage, +## connections, TCP states/reassembler, DNS lookups, timers, and script-level +## state. The script variable :bro:id:`profiling_file` holds the name of the +## file. +## +## .. bro:see:: net_stats +## resource_usage +## get_matcher_stats +## dump_rule_stats +## get_gap_summary +function do_profiling%(%) : bool + %{ + if ( profiling_logger ) + profiling_logger->Log(); + + return new Val(1, TYPE_BOOL); + %} + +## Checks whether a given IP address belongs to a local interface. +## +## ip: The IP address to check. +## +## Returns: True if *ip* belongs to a local interface. +function is_local_interface%(ip: addr%) : bool + %{ + static uint32* addrs; + static int len = -1; + + if ( len < 0 ) + { + char host[MAXHOSTNAMELEN]; + + strcpy(host, "localhost"); + gethostname(host, MAXHOSTNAMELEN); + host[MAXHOSTNAMELEN-1] = '\0'; + + struct hostent* ent = gethostbyname(host); + + for ( len = 0; ent->h_addr_list[len]; ++len ) + ; + + addrs = new uint32[len + 1]; + for ( int i = 0; i < len; i++ ) + addrs[i] = *(uint32*) ent->h_addr_list[i]; + + addrs[len++] = 0x0100007f; // 127.0.0.1 + } + +#ifdef BROv6 + if ( ! is_v4_addr(ip) ) + { + builtin_error("is_local_interface() only supports IPv4 addresses"); + return new Val(0, TYPE_BOOL); + } + + uint32 ip4 = to_v4_addr(ip); +#else + uint32 ip4 = ip; +#endif + + for ( int i = 0; i < len; i++ ) + if ( addrs[i] == ip4 ) + return new Val(1, TYPE_BOOL); + + return new Val(0, TYPE_BOOL); + %} + +## Write rule matcher statistics (DFA states, transitions, memory usage, cache +## hits/misses) to a file. +## +## f: The file to write to. +## +## Returns: True (unconditionally). +## +## .. bro:see:: do_profiling +## resource_usage +## get_matcher_stats +## net_stats +## get_gap_summary +## +## .. todo:: The return value should be changed to any or check appropriately. +function dump_rule_stats%(f: file%): bool + %{ + if ( rule_matcher ) + rule_matcher->DumpStats(f); + + return new Val(1, TYPE_BOOL); + %} + +## Checks wheter Bro is terminating. +## +## Returns: True if Bro is in the process of shutting down. +## +## .. bro:see: terminate +function bro_is_terminating%(%): bool + %{ + return new Val(terminating, TYPE_BOOL); + %} + +## Returns the hostname of the machine Bro runs on. +## +## Returns: The hostname of the machine Bro runs on. +function gethostname%(%) : string + %{ + char buffer[MAXHOSTNAMELEN]; + if ( gethostname(buffer, MAXHOSTNAMELEN) < 0 ) + strcpy(buffer, ""); + + buffer[MAXHOSTNAMELEN-1] = '\0'; + return new StringVal(buffer); + %} + +# =========================================================================== +# +# Conversion +# +# =========================================================================== + +## Converts a :bro:type:`string` to a :bro:type:`int`. +## +## str: The :bro:type:`string` to convert. +## +## Returns: The :bro:type:`string` *str* as :bro:type:`int`. +## +## .. bro:see:: to_addr to_port +function to_int%(str: string%): int + %{ + const char* s = str->CheckString(); + char* end_s; + + long l = strtol(s, &end_s, 10); + int i = int(l); + +#if 0 + // Not clear we should complain. For example, is " 205 " + // a legal conversion? + if ( s[0] == '\0' || end_s[0] != '\0' ) + builtin_error("bad conversion to integer", @ARG@[0]); +#endif + + return new Val(i, TYPE_INT); + %} + + +## Converts a (positive) :bro:type:`int` to a :bro:type:`count`. +## +## n: The :bro:type:`int` to convert. +## +## Returns: The :bro:type:`int` *n* as unsigned integer or 0 if *n* < 0. +function int_to_count%(n: int%): count + %{ + if ( n < 0 ) + { + builtin_error("bad conversion to count", @ARG@[0]); + n = 0; + } + return new Val(n, TYPE_COUNT); + %} + +## Converts a :bro:type:`double` to a :bro:type:`count`. +## +## d: The :bro:type:`double` to convert. +## +## Returns: The :bro:type:`double` *d* as unsigned integer or 0 if *d* < 0.0. +## +## .. bro:see:: double_to_time +function double_to_count%(d: double%): count + %{ + if ( d < 0.0 ) + builtin_error("bad conversion to count", @ARG@[0]); + + return new Val(bro_uint_t(rint(d)), TYPE_COUNT); + %} + +## Converts a :bro:type:`string` to a :bro:type:`count`. +## +## str: The :bro:type:`string` to convert. +## +## Returns: The :bro:type:`string` *str* as unsigned integer or if in invalid +## format. +## +## .. bro:see:: to_addr to_int to_port +function to_count%(str: string%): count + %{ + const char* s = str->CheckString(); + char* end_s; + + uint64 u = (uint64) strtoll(s, &end_s, 10); + + if ( s[0] == '\0' || end_s[0] != '\0' ) + { + builtin_error("bad conversion to count", @ARG@[0]); + u = 0; + } + + return new Val(u, TYPE_COUNT); + %} + +## Converts an :bro:type:`interval` to a :bro:type:`double`. +## +## i: The :bro:type:`interval` to convert. +## +## Returns: The :bro:type:`interval` *i* as :bro:type:`double`. +## +## .. bro:see:: double_to_interval +function interval_to_double%(i: interval%): double + %{ + return new Val(i, TYPE_DOUBLE); + %} + +## Converts a :bro:type:`time` value to a :bro:type:`double`. +## +## t: The :bro:type:`interval` to convert. +## +## Returns: The :bro:type:`time` value *t* as :bro:type:`double`. +## +## .. bro:see:: double_to_time +function time_to_double%(t: time%): double + %{ + return new Val(t, TYPE_DOUBLE); + %} + +## Converts a :bro:type:`time` value to a :bro:type:`double`. +## +## t: The :bro:type:`interval` to convert. +## +## Returns: The :bro:type:`time` value *t* as :bro:type:`double`. +## +## .. bro:see:: time_to_double double_to_count +function double_to_time%(d: double%): time + %{ + return new Val(d, TYPE_TIME); + %} + +## Converts a :bro:type:`double` to an :bro:type:`interval`. +## +## d: The :bro:type:`double` to convert. +## +## Returns: The :bro:type:`double` *d* as :bro:type:`interval`. +## +## .. bro:see:: interval_to_double +function double_to_interval%(d: double%): interval + %{ + return new Val(d, TYPE_INTERVAL); + %} + +## Converts a :bro:type:`addr` to a :bro:type:`count`. +## +## a: The :bro:type:`addr` to convert. +## +## Returns: The :bro:type:`addr` *a* as :bro:type:`count`. +## +## .. bro:see:: addr_to_ptr_name +function addr_to_count%(a: addr%): count + %{ +#ifdef BROv6 + if ( ! is_v4_addr(a) ) + { + builtin_error("conversion of non-IPv4 address to count", @ARG@[0]); + return new Val(0, TYPE_COUNT); + } + + uint32 addr = to_v4_addr(a); +#else + uint32 addr = a; +#endif + return new Val(ntohl(addr), TYPE_COUNT); + %} + +## Converts a :bro:type:`addr` to a :bro:type:`count`. +## +## a: The :bro:type:`addr` to convert. +## +## Returns: The :bro:type:`addr` *a* as :bro:type:`count`. +## +## .. bro:see:: count_to_port +function port_to_count%(p: port%): count + %{ + return new Val(p->Port(), TYPE_COUNT); + %} + +## Converts a :bro:type:`count` and ``transport_proto`` to a :bro:type:`port`. +## +## a: The :bro:type:`addr` to convert. +## +## Returns: The :bro:type:`addr` *a* as :bro:type:`count`. +## +## .. bro:see:: port_to_count +function count_to_port%(c: count, t: transport_proto%): port + %{ + return new PortVal(c, (TransportProto)(t->InternalInt())); + %} + +## Converts a :bro:type:`string` to an :bro:type:`addr`. +## +## ip: The :bro:type:`string` to convert. +## +## Returns: The :bro:type:`string` *ip* as :bro:type:`addr`. +## +## .. bro:see:: to_count to_int to_port count_to_v4_addr raw_bytes_to_v4_addr +function to_addr%(ip: string%): addr + %{ + char* s = ip->AsString()->Render(); + Val* ret = new AddrVal(s); + delete [] s; + return ret; + %} + +## Converts a :bro:type:`count` to an :bro:type:`addr`. +## +## ip: The :bro:type:`count` to convert. +## +## Returns: The :bro:type:`count` *ip* as :bro:type:`addr`. +## +## .. bro:see:: raw_bytes_to_v4_addr to_addr +function count_to_v4_addr%(ip: count%): addr + %{ + if ( ip > 4294967295LU ) + { + builtin_error("conversion of non-IPv4 count to addr", @ARG@[0]); + return new AddrVal(uint32(0)); + } + + return new AddrVal(htonl(uint32(ip))); + %} + +## Converts a :bro:type:`string` of bytes into an IP address. In particular, +## this function interprets the first 4 bytes of the string as an IPv4 address +## in network order. +## +## b: The raw bytes (:bro:type:`string`) to convert. +## +## Returns: The byte :bro:type:`string` *ip* as :bro:type:`addr`. +## +## .. bro:see:: raw_bytes_to_v4_addr to_addr +function raw_bytes_to_v4_addr%(b: string%): addr + %{ + uint32 a = 0; + + if ( b->Len() < 4 ) + builtin_error("too short a string as input to raw_bytes_to_v4_addr()"); + + else + { + const u_char* bp = b->Bytes(); + a = (bp[0] << 24) | (bp[1] << 16) | (bp[2] << 8) | bp[3]; + } + + return new AddrVal(htonl(a)); + %} + +## Creates a :bro:type:`port` from a given number and transport protocol. +## +## num: The port number. +## +## proto: THe transport protocol of the port. +## +## Returns: A :bro:type:`port` with number *num* and transport protocol +## *proto*. +## +## .. bro:see:: to_addr to_count to_int +function to_port%(num: count, proto: transport_proto%): port + %{ + return new PortVal(num, (TransportProto)proto->AsEnum()); + %} + +## Converts a reverse pointer name to an address. For example, +## ``1.0.168.192.in-addr.arpa`` to ``192.168.0.1``. +## +## s: The string with the reverse pointer name. +## +## Returns: The IP address corresponding to *s*. +## +## .. bro:see:: addr_to_ptr_name parse_dotted_addr +function ptr_name_to_addr%(s: string%): addr + %{ + int a[4]; + uint32 addr; + + if ( sscanf(s->CheckString(), + "%d.%d.%d.%d.in-addr.arpa", + a, a+1, a+2, a+3) != 4 ) + { + builtin_error("bad PTR name", @ARG@[0]); + addr = 0; + } + else + addr = (a[3] << 24) | (a[2] << 16) | (a[1] << 8) | a[0]; + + return new AddrVal(htonl(addr)); + %} + +## Converts an IP address to a reverse pointer name. For example, +## ``192.168.0.1`` to ``1.0.168.192.in-addr.arpa``. +## +## a: The IP address to convert to a reverse pointer name. +## +## Returns: The reverse pointer representation of *a*. +## +## .. bro:see:: addr_to_count ptr_name_to_addr parse_dotted_addr +function addr_to_ptr_name%(a: addr%): string + %{ + // ## Question: + // uint32 addr = ntohl((*args)[0]->InternalUnsigned()); + uint32 addr; +#ifdef BROv6 + if ( is_v4_addr(a) ) + addr = a[3]; + else + { + builtin_error("conversion of non-IPv4 address to net", @ARG@[0]); + addr = 0; + } +#else + addr = a; +#endif + + addr = ntohl(addr); + uint32 a3 = (addr >> 24) & 0xff; + uint32 a2 = (addr >> 16) & 0xff; + uint32 a1 = (addr >> 8) & 0xff; + uint32 a0 = addr & 0xff; + + char buf[256]; + sprintf(buf, "%u.%u.%u.%u.in-addr.arpa", a0, a1, a2, a3); + + return new StringVal(buf); + %} + +# Transforms n0.n1.n2.n3 -> addr. + +## Converts a decimal dotted IP address in a :bro:type:`string` to an +## :bro:type:`addr` type. +## +## s: The IP address in the form ``n0.n1.n2.n3``. +## +## Returns: The IP address as type :bro:type:`addr`. +## +## .. bro:see:: addr_to_ptr_name parse_dotted_addr addr_to_count +function parse_dotted_addr%(s: string%): addr + %{ + return new AddrVal(dotted_to_addr(s->CheckString())); + %} + +%%{ +static Val* parse_port(const char* line) + { + RecordVal* r = new RecordVal(ftp_port); + + int bytes[6]; + if ( line && sscanf(line, "%d,%d,%d,%d,%d,%d", + &bytes[0], &bytes[1], &bytes[2], + &bytes[3], &bytes[4], &bytes[5]) == 6 ) + { + int good = 1; + + for ( int i = 0; i < 6; ++i ) + if ( bytes[i] < 0 || bytes[i] > 255 ) + { + good = 0; + break; + } + + uint32 addr = (bytes[0] << 24) | (bytes[1] << 16) | + (bytes[2] << 8) | bytes[3]; + uint32 port = (bytes[4] << 8) | bytes[5]; + + // Since port is unsigned, no need to check for < 0. + if ( port > 65535 ) + { + port = 0; + good = 0; + } + + r->Assign(0, new AddrVal(htonl(addr))); + r->Assign(1, new PortVal(port, TRANSPORT_TCP)); + r->Assign(2, new Val(good, TYPE_BOOL)); + } + else + { + r->Assign(0, new AddrVal(uint32(0))); + r->Assign(1, new PortVal(0, TRANSPORT_TCP)); + r->Assign(2, new Val(0, TYPE_BOOL)); + } + + return r; + } + +static Val* parse_eftp(const char* line) + { + RecordVal* r = new RecordVal(ftp_port); + + int net_proto = 0; // currently not used + uint32 addr = 0; + int port = 0; + int good = 0; + + if ( line ) + { + while ( isspace(*line) ) // skip whitespace + ++line; + + char delimiter = *line; + good = 1; + char* next_delim; + + ++line; // cut off delimiter + net_proto = strtol(line, &next_delim, 10); // currently ignored + if ( *next_delim != delimiter ) + good = 0; + + line = next_delim + 1; + if ( *line != delimiter ) // default of 0 is ok + { + addr = dotted_to_addr(line); + if ( addr == 0 ) + good = 0; + } + + // FIXME: check for garbage between IP and delimiter. + line = strchr(line, delimiter); + + ++line; // now the port + port = strtol(line, &next_delim, 10); + if ( *next_delim != delimiter ) + good = 0; + } + + r->Assign(0, new AddrVal(addr)); + r->Assign(1, new PortVal(port, TRANSPORT_TCP)); + r->Assign(2, new Val(good, TYPE_BOOL)); + + return r; + } +%%} + +## Converts a string representation of the FTP PORT command to an ``ftp_port``. +## +## s: The string of the FTP PORT command, e.g., ``"10,0,0,1,4,31"``. +## +## Returns: The FTP PORT, e.g., ``[h=10.0.0.1, p=1055/tcp, valid=T]`` +## +## .. bro:see:: parse_eftp_port parse_ftp_pasv parse_ftp_epsv fmt_ftp_port +function parse_ftp_port%(s: string%): ftp_port + %{ + return parse_port(s->CheckString()); + %} + +## Converts a string representation of the FTP EPRT command to an ``ftp_port``. +## (see `RFC 2428 `_). +## The format is ``EPRT``, +## where ```` is a delimiter in the ASCII range 33-126 (usually ``|``). +## +## s: The string of the FTP PORT command, e.g., ``"10,0,0,1,4,31"``. +## +## Returns: The FTP PORT, e.g., ``[h=10.0.0.1, p=1055/tcp, valid=T]`` +## +## .. bro:see:: parse_ftp_port parse_ftp_pasv parse_ftp_epsv fmt_ftp_port +function parse_eftp_port%(s: string%): ftp_port + %{ + return parse_eftp(s->CheckString()); + %} + +## Converts the result of the FTP PASV command to an ``ftp_port``. +## +## str: The string containing the result of the FTP PASV command. +## +## Returns: The FTP PORT, e.g., ``[h=10.0.0.1, p=1055/tcp, valid=T]`` +## +## .. bro:see:: parse_ftp_port parse_eftp_port parse_ftp_epsv fmt_ftp_port +function parse_ftp_pasv%(str: string%): ftp_port + %{ + const char* s = str->CheckString(); + const char* line = strchr(s, '('); + if ( line ) + ++line; // move past '(' + else if ( (line = strstr(s, "PORT")) ) + line += 5; // Skip over + else if ( (line = strchr(s, ',')) ) + { // Look for comma-separated list. + while ( --line >= s && isdigit(*line) ) + ; // Back up over preceding digits. + ++line; // now points to first digit, or beginning of s + } + + return parse_port(line); + %} + +## Converts the result of the FTP EPSV command to an ``ftp_port``. +## See `RFC 2428 `_. +## The format is `` ()``, where ```` is a +## delimiter in the ASCII range 33-126 (usually ``|``). +## +## str: The string containing the result of the FTP PASV command. +## +## Returns: The FTP PORT, e.g., ``[h=10.0.0.1, p=1055/tcp, valid=T]`` +## +## .. bro:see:: parse_ftp_port parse_eftp_port parse_ftp_pasv fmt_ftp_port +function parse_ftp_epsv%(str: string%): ftp_port + %{ + const char* s = str->CheckString(); + const char* line = strchr(s, '('); + if ( line ) + ++line; // move past '(' + return parse_eftp(line); + %} + +## Formats an IP address and TCP port as an FTP PORT command. For example, +## ``10.0.0.1`` and ``1055/tcp`` yields ``"10,0,0,1,4,31"``. +## +## a: The IP address. +## +## p: The TCP port. +## +## Returns: The FTP PORT string. +## +## .. bro:see:: parse_ftp_port parse_eftp_port parse_ftp_pasv parse_ftp_epsv +function fmt_ftp_port%(a: addr, p: port%): string + %{ +#ifdef BROv6 + if ( ! is_v4_addr(a) ) + builtin_error("conversion of non-IPv4 address to net", @ARG@[0]); + + uint32 addr = to_v4_addr(a); +#else + uint32 addr = a; +#endif + addr = ntohl(addr); + uint32 pn = p->Port(); + return new StringVal(fmt("%d,%d,%d,%d,%d,%d", + addr >> 24, (addr >> 16) & 0xff, + (addr >> 8) & 0xff, addr & 0xff, + pn >> 8, pn & 0xff)); + %} + +## Decode a NetBIOS name. See http://support.microsoft.com/kb/194203. +## +## name: The encoded NetBIOS name, e.g., ``"FEEIEFCAEOEFFEECEJEPFDCAEOEBENEF:``. +## +## Returns: The decoded NetBIOS name, e.g., ``"THE NETBIOS NAME"``. +## +## .. bro:see:: decode_netbios_name_type +function decode_netbios_name%(name: string%): string + %{ + char buf[16]; + char result[16]; + const u_char* s = name->Bytes(); + int i, j; + + for ( i = 0, j = 0; i < 16; ++i ) + { + char c0 = (j < name->Len()) ? toupper(s[j++]) : 'A'; + char c1 = (j < name->Len()) ? toupper(s[j++]) : 'A'; + buf[i] = ((c0 - 'A') << 4) + (c1 - 'A'); + } + + for ( i = 0; i < 15; ++i ) + { + if ( isalnum(buf[i]) || ispunct(buf[i]) || + // \x01\x02 is seen in at least one case as the first two bytes. + // I think that any \x01 and \x02 should always be passed through. + buf[i] < 3 ) + result[i] = buf[i]; + else + break; + } + + return new StringVal(i, result); + %} + +## Converts a NetBIOS name type to its corresonding numeric value. +## See http://support.microsoft.com/kb/163409. +## +## name: The NetBIOS name type. +## +## Returns: The numeric value of *name*. +## +## .. bro:see:: decode_netbios_name +function decode_netbios_name_type%(name: string%): count + %{ + const u_char* s = name->Bytes(); + char return_val = ((toupper(s[30]) - 'A') << 4) + (toupper(s[31]) - 'A'); + return new Val(return_val, TYPE_COUNT); + %} + +## Converts a string of bytes into its hexadecimal representation, e.g., +## ``"04"`` to ``"3034"``. +## +## bytestring: The string of bytes. +## +## Returns: The hexadecimal reprsentation of *bytestring*. +## +## .. bro:see:: hexdump +function bytestring_to_hexstr%(bytestring: string%): string + %{ + bro_uint_t len = bytestring->AsString()->Len(); + const u_char* bytes = bytestring->AsString()->Bytes(); + char hexstr[(2 * len) + 1]; + + hexstr[0] = 0; + for ( bro_uint_t i = 0; i < len; ++i ) + snprintf(hexstr + (2 * i), 3, "%.2hhx", bytes[i]); + + return new StringVal(hexstr); + %} + ## Decodes a Base64-encoded string. ## ## s: The Base64-encoded string. @@ -2781,11 +2721,6 @@ function uuid_to_string%(uuid: string%): string return new StringVal(s); %} - -# The following functions convert strings into patterns at run-time. As the -# computed NFAs and DFAs cannot be cleanly deallocated (at least for now), -# they can only be used at initialization time. - ## Merges and compiles two regular expressions at initialization time. ## ## p1: The first pattern. @@ -2900,12 +2835,1683 @@ function string_to_pattern%(s: string, convert: bool%): pattern return new PatternVal(re); %} +## Formats a given time value according to a format string. +## +## fmt: The format string. See ``man strftime`` for the syntax. +## +## d: The time value. +## +## Returns: The time *d* formatted according to *fmt*. +function strftime%(fmt: string, d: time%) : string + %{ + static char buffer[128]; + + time_t t = time_t(d); + + if ( strftime(buffer, 128, fmt->CheckString(), localtime(&t)) == 0 ) + return new StringVal(""); + + return new StringVal(buffer); + %} + +# =========================================================================== +# +# Network Type Processing +# +# =========================================================================== + +## Masks an address down to the number of given upper bits. For example, +## ``mask_addr(1.2.3.4, 18)`` returns ``1.2.0.0``. +## +## a: The address to mask. +## +## top_bits_to_keep: The number of top bits to keep in *a*; must be greater +## than 0 and less than 33. +## +## Returns: The address *a* masked down to *top_bits_to_keep* bits. +## +## .. bro:see:: remask_addr +function mask_addr%(a: addr, top_bits_to_keep: count%): subnet + %{ + return new SubNetVal(mask_addr(a, top_bits_to_keep), top_bits_to_keep); + %} + +## Takes some top bits (e.g., subnet address) from one address and the other +## bits (intra-subnet part) from a second address and merges them to get a new +## address. This is useful for anonymizing at subnet level while preserving +## serial scans. +## +## a1: The address to mask with *top_bits_from_a1*. +## +## a2: The address to take the remaining bits from. +## +## top_bits_from_a1: The number of top bits to keep in *a1*; must be greater +## than 0 and less than 33. +## +## Returns: The address *a* masked down to *top_bits_to_keep* bits. +## +## .. bro:see:: mask_addr +function remask_addr%(a1: addr, a2: addr, top_bits_from_a1: count%): addr + %{ +#ifdef BROv6 + if ( ! is_v4_addr(a1) || ! is_v4_addr(a2) ) + { + builtin_error("cannot use remask_addr on IPv6 addresses"); + return new AddrVal(a1); + } + + uint32 x1 = to_v4_addr(a1); + uint32 x2 = to_v4_addr(a2); +#else + uint32 x1 = a1; + uint32 x2 = a2; +#endif + return new AddrVal( + mask_addr(x1, top_bits_from_a1) | + (x2 ^ mask_addr(x2, top_bits_from_a1)) ); + %} + +## Checks whether a given :bro:type:`port` has TCP as transport protocol. +## +## p: The :bro:type:`port` to check. +## +## Returns: True iff *p* is a TCP port. +## +## .. bro:see:: is_udp_port is_icmp_port +function is_tcp_port%(p: port%): bool + %{ + return new Val(p->IsTCP(), TYPE_BOOL); + %} + +## Checks whether a given :bro:type:`port` has UDP as transport protocol. +## +## p: The :bro:type:`port` to check. +## +## Returns: True iff *p* is a UDP port. +## +## .. bro:see:: is_icmp_port is_tcp_port +function is_udp_port%(p: port%): bool + %{ + return new Val(p->IsUDP(), TYPE_BOOL); + %} + +## Checks whether a given :bro:type:`port` has ICMP as transport protocol. +## +## p: The :bro:type:`port` to check. +## +## Returns: True iff *p* is a ICMP port. +## +## .. bro:see:: is_tcp_port is_udp_port +function is_icmp_port%(p: port%): bool + %{ + return new Val(p->IsICMP(), TYPE_BOOL); + %} + +%%{ +EnumVal* map_conn_type(TransportProto tp) + { + switch ( tp ) { + case TRANSPORT_UNKNOWN: + return new EnumVal(0, transport_proto); + break; + + case TRANSPORT_TCP: + return new EnumVal(1, transport_proto); + break; + + case TRANSPORT_UDP: + return new EnumVal(2, transport_proto); + break; + + case TRANSPORT_ICMP: + return new EnumVal(3, transport_proto); + break; + + default: + reporter->InternalError("bad connection type in map_conn_type()"); + } + + // Cannot be reached; + assert(false); + return 0; // Make compiler happy. + } +%%} + +## Extracts the transport protocol from a connection. +## +## cid: The connection identifier. +## +## Returns: The transport protocol of the connection identified by *id*. +## +## .. bro:see:: get_port_transport_proto +## get_orig_seq get_resp_seq +function get_conn_transport_proto%(cid: conn_id%): transport_proto + %{ + Connection* c = sessions->FindConnection(cid); + if ( ! c ) + { + builtin_error("unknown connection id in get_conn_transport_proto()", cid); + return new EnumVal(0, transport_proto); + } + + return map_conn_type(c->ConnTransport()); + %} + +## Extracts the transport protocol from a :bro:type:`port`. +## +## p: The port. +## +## Returns: The transport protocol of the port *p*. +## +## .. bro:see:: get_conn_transport_proto +## get_orig_seq get_resp_seq +function get_port_transport_proto%(p: port%): transport_proto + %{ + return map_conn_type(p->PortType()); + %} + +## Checks whether a connection is (still) active. +## +## c: The connection id to check. +## +## Returns: True if the connection identified by *c* exists. +## +## .. bro:see:: lookup_connection +function connection_exists%(c: conn_id%): bool + %{ + if ( sessions->FindConnection(c) ) + return new Val(1, TYPE_BOOL); + else + return new Val(0, TYPE_BOOL); + %} + +## Returns the :bro:id:`connection` record for a given connection identifier. +## +## cid: The connection ID. +## +## Returns: The :bro:id:`connection` record for *cid*. If *cid* does not point +## to an existing connection, the function generates a run-time error +## and returns a dummy value. +## +## .. bro:see:: connection_exists +function lookup_connection%(cid: conn_id%): connection + %{ + Connection* conn = sessions->FindConnection(cid); + if ( conn ) + return conn->BuildConnVal(); + + builtin_error("connection ID not a known connection", cid); + + // Return a dummy connection record. + RecordVal* c = new RecordVal(connection_type); + + RecordVal* id_val = new RecordVal(conn_id); + id_val->Assign(0, new AddrVal((unsigned int) 0)); + id_val->Assign(1, new PortVal(ntohs(0), TRANSPORT_UDP)); + id_val->Assign(2, new AddrVal((unsigned int) 0)); + id_val->Assign(3, new PortVal(ntohs(0), TRANSPORT_UDP)); + c->Assign(0, id_val); + + RecordVal* orig_endp = new RecordVal(endpoint); + orig_endp->Assign(0, new Val(0, TYPE_COUNT)); + orig_endp->Assign(1, new Val(int(0), TYPE_COUNT)); + + RecordVal* resp_endp = new RecordVal(endpoint); + resp_endp->Assign(0, new Val(0, TYPE_COUNT)); + resp_endp->Assign(1, new Val(int(0), TYPE_COUNT)); + + c->Assign(1, orig_endp); + c->Assign(2, resp_endp); + + c->Assign(3, new Val(network_time, TYPE_TIME)); + c->Assign(4, new Val(0.0, TYPE_INTERVAL)); + c->Assign(5, new TableVal(string_set)); // service + c->Assign(6, new StringVal("")); // addl + c->Assign(7, new Val(0, TYPE_COUNT)); // hot + c->Assign(8, new StringVal("")); // history + + return c; + %} + +%%{ +#include "HTTP.h" + +const char* conn_id_string(Val* c) + { + Val* id = (*(c->AsRecord()))[0]; + const val_list* vl = id->AsRecord(); + + addr_type orig_h = (*vl)[0]->AsAddr(); + uint32 orig_p = (*vl)[1]->AsPortVal()->Port(); + addr_type resp_h = (*vl)[2]->AsAddr(); + uint32 resp_p = (*vl)[3]->AsPortVal()->Port(); + + return fmt("%s/%u -> %s/%u\n", dotted_addr(orig_h), orig_p, dotted_addr(resp_h), resp_p); + } +%%} + +## Skips the data of the HTTP entity. +## +## c: The HTTP connection. +## +## is_orig: If true, the client data is skipped and the server data otherwise. +## +## .. bro:see:: skip_smtp_data +function skip_http_entity_data%(c: connection, is_orig: bool%): any + %{ + AnalyzerID id = mgr.CurrentAnalyzer(); + if ( id ) + { + Analyzer* ha = c->FindAnalyzer(id); + + if ( ha ) + { + if ( ha->GetTag() == AnalyzerTag::HTTP ) + static_cast(ha)->SkipEntityData(is_orig); + else + reporter->Error("non-HTTP analyzer associated with connection record"); + } + else + reporter->Error("could not find analyzer for skip_http_entity_data"); + + } + else + reporter->Error("no analyzer associated with connection record"); + + return 0; + %} + +## Unescapes all characters in a URI, i.e., decodes every ``%xx`` group. +## +## URI: The URI to unescape. +## +## Returns: The unescaped URI with all ``%xx`` groups decoded. +## +## .. note:: +## +## Unescaping reserved characters may cause loss of information. RFC 2396: +## A URI is always in an "escaped" form, since escaping or unescaping a +## completed URI might change its semantics. Normally, the only time +## escape encodings can safely be made is when the URI is being created +## from its component parts. +function unescape_URI%(URI: string%): string + %{ + const u_char* line = URI->Bytes(); + const u_char* const line_end = line + URI->Len(); + + return new StringVal(unescape_URI(line, line_end, 0)); + %} + +## Writes the current packet to a file. +## +## file_name: The name of the file to write the packet to. +## +## Returns: True on success. +## +## .. bro:see:: dump_packet get_current_packet send_current_packet +function dump_current_packet%(file_name: string%) : bool + %{ + const struct pcap_pkthdr* hdr; + const u_char* pkt; + + if ( ! current_pktsrc || + ! current_pktsrc->GetCurrentPacket(&hdr, &pkt) ) + return new Val(0, TYPE_BOOL); + + if ( ! addl_pkt_dumper ) + addl_pkt_dumper = new PktDumper(0, true); + + addl_pkt_dumper->Open(file_name->CheckString()); + addl_pkt_dumper->Dump(hdr, pkt); + + return new Val(! addl_pkt_dumper->IsError(), TYPE_BOOL); + %} + +## Returns the currently processed PCAP packet. +## +## Returns: The currently processed packet, which is as a record +## containing the timestamp, ``snaplen,'' and packet data. +## +## .. bro:see:: dump_current_packet dump_packet send_current_packet +function get_current_packet%(%) : pcap_packet + %{ + const struct pcap_pkthdr* hdr; + const u_char* data; + RecordVal* pkt = new RecordVal(pcap_packet); + + if ( ! current_pktsrc || + ! current_pktsrc->GetCurrentPacket(&hdr, &data) ) + { + pkt->Assign(0, new Val(0, TYPE_COUNT)); + pkt->Assign(1, new Val(0, TYPE_COUNT)); + pkt->Assign(2, new Val(0, TYPE_COUNT)); + pkt->Assign(3, new Val(0, TYPE_COUNT)); + pkt->Assign(4, new StringVal("")); + return pkt; + } + + pkt->Assign(0, new Val(uint32(hdr->ts.tv_sec), TYPE_COUNT)); + pkt->Assign(1, new Val(uint32(hdr->ts.tv_usec), TYPE_COUNT)); + pkt->Assign(2, new Val(hdr->caplen, TYPE_COUNT)); + pkt->Assign(3, new Val(hdr->len, TYPE_COUNT)); + pkt->Assign(4, new StringVal(hdr->caplen, (const char*) data)); + + return pkt; + %} + +## Writes a given packet to a file. +## +## pkt: The PCAP packet. +## +## file_name: The name of the file to write *pkt* to. +## +## Returns: True on success +## +## .. bro:see:: get_current_packet dump_current_packet send_current_packet +function dump_packet%(pkt: pcap_packet, file_name: string%) : bool + %{ + struct pcap_pkthdr hdr; + const val_list* pkt_vl = pkt->AsRecord(); + + hdr.ts.tv_sec = (*pkt_vl)[0]->AsCount(); + hdr.ts.tv_usec = (*pkt_vl)[1]->AsCount(); + hdr.caplen = (*pkt_vl)[2]->AsCount(); + hdr.len = (*pkt_vl)[3]->AsCount(); + + if ( ! addl_pkt_dumper ) + addl_pkt_dumper = new PktDumper(0, true); + + addl_pkt_dumper->Open(file_name->CheckString()); + addl_pkt_dumper->Dump(&hdr, (*pkt_vl)[4]->AsString()->Bytes()); + + return new Val(addl_pkt_dumper->IsError(), TYPE_BOOL); + %} + +%%{ +#include "DNS_Mgr.h" +#include "Trigger.h" + +class LookupHostCallback : public DNS_Mgr::LookupCallback { +public: + LookupHostCallback(Trigger* arg_trigger, const CallExpr* arg_call, + bool arg_lookup_name) + { + Ref(arg_trigger); + trigger = arg_trigger; + call = arg_call; + lookup_name = arg_lookup_name; + } + + ~LookupHostCallback() + { + Unref(trigger); + } + + // Overridden from DNS_Mgr:Lookup:Callback. + virtual void Resolved(const char* name) + { + Val* result = new StringVal(name); + trigger->Cache(call, result); + Unref(result); + trigger->Release(); + } + + virtual void Resolved(TableVal* addrs) + { + // No Ref() for addrs. + trigger->Cache(call, addrs); + trigger->Release(); + } + + virtual void Timeout() + { + if ( lookup_name ) + { + Val* result = new StringVal("<\?\?\?>"); + trigger->Cache(call, result); + Unref(result); + } + + else + { + ListVal* lv = new ListVal(TYPE_ADDR); + lv->Append(new AddrVal("0.0.0.0")); + Val* result = lv->ConvertToSet(); + trigger->Cache(call, result); + Unref(result); + Unref(lv); + } + + trigger->Release(); + } + +private: + Trigger* trigger; + const CallExpr* call; + bool lookup_name; +}; +%%} + +## Issues an asynchronous reverse DNS lookup and delays the function result. +## This function can therefore only be called inside a ``when`` condition, +## e.g., ``when ( local host = lookup_addr(10.0.0.1) ) { f(host); }``. +## +## host: The IP address to lookup. +## +## Returns: The DNS name of *host*. +## +## .. bro:see:: lookup_hostname +function lookup_addr%(host: addr%) : string + %{ + // FIXME: It should be easy to adapt the function to synchronous + // lookups if we're reading a trace. + Trigger* trigger = frame->GetTrigger(); + + if ( ! trigger) + { + builtin_error("lookup_addr() can only be called inside a when-condition"); + return new StringVal(""); + } + + frame->SetDelayed(); + trigger->Hold(); + +#ifdef BROv6 + if ( ! is_v4_addr(host) ) + { + // FIXME: This is a temporary work-around until we get this + // fixed. We warn the user once, and always trigger a timeout. + // Ticket #355 records the problem. + static bool warned = false; + if ( ! warned ) + { + reporter->Warning("lookup_addr() only supports IPv4 addresses currently"); + warned = true; + } + + trigger->Timeout(); + return 0; + } + + dns_mgr->AsyncLookupAddr(to_v4_addr(host), + new LookupHostCallback(trigger, frame->GetCall(), true)); +#else + dns_mgr->AsyncLookupAddr(host, + new LookupHostCallback(trigger, frame->GetCall(), true)); +#endif + return 0; + %} + +## Issues an asynchronous DNS lookup and delays the function result. +## This function can therefore only be called inside a ``when`` condition, +## e.g., ``when ( local h = lookup_hostname("www.bro-ids.org") ) { f(h); }``. +## +## host: The hostname to lookup. +## +## Returns: A set of DNS A records associated with *host*. +## +## .. bro:see:: lookup_addr +function lookup_hostname%(host: string%) : addr_set + %{ + // FIXME: Is should be easy to adapt the function to synchronous + // lookups if we're reading a trace. + Trigger* trigger = frame->GetTrigger(); + + if ( ! trigger) + { + builtin_error("lookup_hostname() can only be called inside a when-condition"); + return new StringVal(""); + } + + frame->SetDelayed(); + trigger->Hold(); + + dns_mgr->AsyncLookupName(host->CheckString(), + new LookupHostCallback(trigger, frame->GetCall(), false)); + return 0; + %} + +%%{ +#ifdef USE_GEOIP +extern "C" { +#include +} + +static GeoIP* open_geoip_db(GeoIPDBTypes type) + { + GeoIP* geoip = 0; + + if ( GeoIP_db_avail(type) ) + geoip = GeoIP_open_type(type, GEOIP_MEMORY_CACHE); + + if ( ! geoip ) + reporter->Warning("Failed to open GeoIP database: %s", + GeoIPDBFileName[type]); + return geoip; + } + +#endif +%%} + +## Performs a geo-lookup of an IP address. +## Requires Bro to be built with ``libgeoip``. +## +## a: The IP address to lookup. +## +## Returns: A record with country, region, city, latitude, and longitude. +## +## .. bro:see:: lookup_asn +function lookup_location%(a: addr%) : geo_location + %{ + RecordVal* location = new RecordVal(geo_location); + +#ifdef USE_GEOIP + static bool geoip_initialized = false; + static GeoIP* geoip = 0; + static GeoIP* geoip_v6 = 0; + static bool have_city_db = false; + static bool have_cityv6_db = false; + GeoIPRecord* gir = 0; + const char* cc = 0; + + if ( ! geoip_initialized ) + { + geoip_initialized = true; + geoip = open_geoip_db(GEOIP_CITY_EDITION_REV0); + + if ( ! geoip ) + { + geoip = open_geoip_db(GEOIP_COUNTRY_EDITION); + if ( ! geoip ) + builtin_error("Can't initialize GeoIP City/Country database"); + else + reporter->Warning("Fell back to GeoIP Country database"); + } + else + have_city_db = true; + +#ifdef BROv6 + +#ifdef HAVE_GEOIP_CITY_EDITION_REV0_V6 + geoip_v6 = open_geoip_db(GEOIP_CITY_EDITION_REV0_V6); + if ( geoip_v6 ) + have_cityv6_db = true; +#endif + +#ifdef HAVE_GEOIP_COUNTRY_EDITION_V6 + if ( ! geoip_v6 ) + geoip_v6 = open_geoip_db(GEOIP_COUNTRY_EDITION_V6); +#endif + if ( ! geoip_v6 ) + builtin_error("Can't initialize GeoIPv6 City/Country database"); +#endif + } + +#ifdef BROv6 + +#ifdef HAVE_GEOIP_COUNTRY_EDITION_V6 + if ( geoip_v6 && ! is_v4_addr(a) ) + { + geoipv6_t ga; + memcpy(&ga, a, 16); + if ( have_cityv6_db ) + gir = GeoIP_record_by_ipnum_v6(geoip_v6, ga); + else + cc = GeoIP_country_code_by_ipnum_v6(geoip_v6, ga); + } + else +#endif + + if ( geoip && is_v4_addr(a) ) + { + uint32 addr = to_v4_addr(a); + if ( have_city_db ) + gir = GeoIP_record_by_ipnum(geoip, ntohl(addr)); + else + cc = GeoIP_country_code_by_ipnum(geoip, ntohl(addr)); + } + +#else // not BROv6 + if ( geoip ) + { + if ( have_city_db ) + gir = GeoIP_record_by_ipnum(geoip, ntohl(a)); + else + cc = GeoIP_country_code_by_ipnum(geoip, ntohl(a)); + } +#endif + + if ( gir ) + { + if ( gir->country_code ) + location->Assign(0, new StringVal(gir->country_code)); + + if ( gir->region ) + location->Assign(1, new StringVal(gir->region)); + + if ( gir->city ) + location->Assign(2, new StringVal(gir->city)); + + if ( gir->latitude ) + location->Assign(3, new Val(gir->latitude, + TYPE_DOUBLE)); + + if ( gir->longitude ) + location->Assign(4, new Val(gir->longitude, + TYPE_DOUBLE)); + + GeoIPRecord_delete(gir); + + return location; + } + + else if ( cc ) + { + location->Assign(0, new StringVal(cc)); + return location; + } + +#else // not USE_GEOIP + static int missing_geoip_reported = 0; + + if ( ! missing_geoip_reported ) + { + builtin_error("Bro was not configured for GeoIP support"); + missing_geoip_reported = 1; + } +#endif + + // We can get here even if we have GeoIP support if we weren't + // able to initialize it or it didn't return any information for + // the address. + + return location; + %} + +## Performs an AS lookup of an IP address. +## +## a: The IP address to lookup. +## +## Returns: The number of the AS that contains *a*. +## +## .. bro:see:: lookup_location +function lookup_asn%(a: addr%) : count + %{ +#ifdef USE_GEOIP + static GeoIP* geoip_asn = 0; + static bool geoip_asn_initialized = false; + char* gir = 0; + + if ( ! geoip_asn_initialized ) + { + geoip_asn_initialized = true; + geoip_asn = open_geoip_db(GEOIP_ASNUM_EDITION); + if ( ! geoip_asn ) + builtin_error("Can't initialize GeoIP ASNUM database"); + } + + if ( geoip_asn ) + { +#ifdef BROv6 + +// IPv6 support showed up in 1.4.5. +#ifdef HAVE_GEOIP_COUNTRY_EDITION_V6 + if ( ! is_v4_addr(a) ) + { + geoipv6_t ga; + memcpy(&ga, a, 16); + gir = GeoIP_name_by_ipnum_v6(geoip_asn, ga); + } + else +#endif + + if ( is_v4_addr(a) ) + { + uint32 addr = to_v4_addr(a); + gir = GeoIP_name_by_ipnum(geoip_asn, ntohl(addr)); + } + +#else // not BROv6 + gir = GeoIP_name_by_ipnum(geoip_asn, ntohl(a)); +#endif + } + + if ( gir ) + { + // Move the pointer +2 so we don't return + // the first two characters: "AS". + return new Val(atoi(gir+2), TYPE_COUNT); + } + +#else // not USE_GEOIP + static int missing_geoip_reported = 0; + + if ( ! missing_geoip_reported ) + { + builtin_error("Bro was not configured for GeoIP ASN support"); + missing_geoip_reported = 1; + } +#endif + + // We can get here even if we have GeoIP support, if we weren't + // able to initialize it or it didn't return any information for + // the address. + return new Val(0, TYPE_COUNT); + %} + +%%{ +#include +#include +#include + +// This is the indexed map of X509 certificate stores. +static map x509_stores; + +// ### NOTE: while d2i_X509 does not take a const u_char** pointer, +// here we assume d2i_X509 does not write to , so it is safe to +// convert data to a non-const pointer. Could some X509 guru verify +// this? + +X509* d2i_X509_(X509** px, const u_char** in, int len) + { +#ifdef OPENSSL_D2I_X509_USES_CONST_CHAR + return d2i_X509(px, in, len); +#else + return d2i_X509(px, (u_char**)in, len); +#endif + } + +%%} + + +## Verifies a X.509 certificate. +## +## der_cert: The X.509 certificate in DER format. +## +## cert_stack: Specifies a certificate chain to validate against, with index 0 +## typically being the root CA. Bro uses the Mozilla root CA list +## by default. +## +## root_certs: A list of additional root certificates that extends +## *cert_stack*. +## +## Returns: A status code of the verification which can be converted into an +## ASCII string via :bro:id:`x509_err2str`. +## +## .. bro:see:: x509_err2str +function x509_verify%(der_cert: string, cert_stack: string_vec, root_certs: table_string_of_string%): count + %{ + X509_STORE* ctx = 0; + int i = 0; + + // If this certificate store was built previously, just reuse the old one. + if ( x509_stores.count(root_certs) > 0 ) + ctx = x509_stores[root_certs]; + + if ( ! ctx ) // lookup to see if we have this one built already! + { + ctx = X509_STORE_new(); + TableVal* root_certs2 = root_certs->AsTableVal(); + ListVal* idxs = root_certs2->ConvertToPureList(); + + // Build the validation store + for ( i = 0; i < idxs->Length(); ++i ) + { + Val* key = idxs->Index(i); + StringVal *sv = root_certs2->Lookup(key)->AsStringVal(); + const uint8* data = sv->Bytes(); + X509* x = d2i_X509_(NULL, &data, sv->Len()); + if ( ! x ) + { + builtin_error(fmt("Root CA error: %s", ERR_error_string(ERR_peek_last_error(),NULL))); + return new Val((uint64) ERR_get_error(), TYPE_COUNT); + } + X509_STORE_add_cert(ctx, x); + } + delete idxs; + + // Save the newly constructed certificate store into the cacheing map. + x509_stores[root_certs] = ctx; + } + + const uint8 *cert_data = der_cert->Bytes(); + X509* cert = d2i_X509_(NULL, &cert_data, der_cert->Len()); + if ( ! cert ) + { + builtin_error(fmt("Certificate error: %s", ERR_error_string(ERR_peek_last_error(),NULL))); + return new Val((uint64) ERR_get_error(), TYPE_COUNT); + } + + STACK_OF(X509)* untrusted_certs = sk_X509_new_null(); + if ( ! untrusted_certs ) + { + builtin_error(fmt("Untrusted certificate stack initialization error: %s", ERR_error_string(ERR_peek_last_error(),NULL))); + return new Val((uint64) ERR_get_error(), TYPE_COUNT); + } + + VectorVal *cert_stack_vec = cert_stack->AsVectorVal(); + for ( i = 0; i < (int) cert_stack_vec->Size(); ++i ) + { + StringVal *sv = cert_stack_vec->Lookup(i)->AsStringVal(); + const uint8 *data = sv->Bytes(); + X509* x = d2i_X509_(NULL, &data, sv->Len()); + if ( ! x ) + { + X509_free(cert); + sk_X509_pop_free(untrusted_certs, X509_free); + builtin_error(fmt("Untrusted certificate stack creation error: %s", ERR_error_string(ERR_peek_last_error(),NULL))); + return new Val((uint64) ERR_get_error(), TYPE_COUNT); + } + sk_X509_push(untrusted_certs, x); + } + + X509_STORE_CTX csc; + X509_STORE_CTX_init(&csc, ctx, cert, untrusted_certs); + X509_STORE_CTX_set_time(&csc, 0, (time_t) network_time); + + int result = X509_verify_cert(&csc); + X509_STORE_CTX_cleanup(&csc); + + if ( untrusted_certs ) + sk_X509_pop_free(untrusted_certs, X509_free); + X509_free(cert); + + return new Val((uint64) csc.error, TYPE_COUNT); + %} + +## Converts a X.509 certificate verification error code into an ASCII string. +## +## err_num: The error code. +## +## Returns: A string representation of *err_num*. +## +## .. bro:see:: x509_verify +function x509_err2str%(err_num: count%): string + %{ + return new StringVal(X509_verify_cert_error_string(err_num)); + %} + +## Converts UNIX file permissions given by a mode to an ASCII string. +## +## mode: The permisssions, e.g., 644 or 755. +## +## Returns: A string representation of *mode* in the format +## ``rw[xsS]rw[xsS]rw[xtT]``. +function NFS3::mode2string%(mode: count%): string + %{ + char str[12]; + char *p = str; + + /* usr */ + if (mode & S_IRUSR) + *p++ = 'r'; + else + *p++ = '-'; + + if (mode & S_IWUSR) + *p++ = 'w'; + else + *p++ = '-'; + + switch (mode & (S_IXUSR | S_ISUID)) { + case 0: + *p++ = '-'; + break; + case S_IXUSR: + *p++ = 'x'; + break; + case S_ISUID: + *p++ = 'S'; + break; + case S_IXUSR | S_ISUID: + *p++ = 's'; + break; + } + + /* group */ + if (mode & S_IRGRP) + *p++ = 'r'; + else + *p++ = '-'; + if (mode & S_IWGRP) + *p++ = 'w'; + else + *p++ = '-'; + + switch (mode & (S_IXGRP | S_ISGID)) { + case 0: + *p++ = '-'; + break; + case S_IXGRP: + *p++ = 'x'; + break; + case S_ISGID: + *p++ = 'S'; + break; + case S_IXGRP | S_ISGID: + *p++ = 's'; + break; + } + + /* other */ + if (mode & S_IROTH) + *p++ = 'r'; + else + *p++ = '-'; + if (mode & S_IWOTH) + *p++ = 'w'; + else + *p++ = '-'; + + switch (mode & (S_IXOTH | S_ISVTX)) { + case 0: + *p++ = '-'; + break; + case S_IXOTH: + *p++ = 'x'; + break; + case S_ISVTX: + *p++ = 'T'; + break; + case S_IXOTH | S_ISVTX: + *p++ = 't'; + break; + } + + *p = '\0'; + + return new StringVal(str); + %} + +# =========================================================================== +# +# Controlling Analyzer Behavior +# +# =========================================================================== + +%%{ +#include "DPM.h" +%%} + +## Schedules an analyzer for a future connection from a given IP address and +## port. The function ignores the scheduling request if the connection did +## not occur within the specified time interval. +## +## orig: The IP address originating a connection in the future. +## +## resp: The IP address responding to a connection from *orig*. +## +## resp_p: The destination port at *resp*. +## +## analyzer: The analyzer ID. +## +## tout: The timeout interval after which to ignore the scheduling request. +## +## Returns: True (unconditionally). +## +## .. bro:see:: disable_analyzer analyzer_name +## +## .. todo:: The return value should be changed to any. +function expect_connection%(orig: addr, resp: addr, resp_p: port, + analyzer: count, tout: interval%) : bool + %{ + dpm->ExpectConnection(orig, resp, resp_p->Port(), resp_p->PortType(), + (AnalyzerTag::Tag) analyzer, tout, 0); + return new Val(1, TYPE_BOOL); + %} + +## Disables the analyzer which raised the current event (if the analyzer +## belongs to the given connection). +## +## cid: The connection identifier. +## +## aid: The analyzer ID. +## +## Returns: True if the connection identified by *cid* exists and has analyzer +## *aid*. +## +## .. bro:see:: expect_connection analyzer_name +function disable_analyzer%(cid: conn_id, aid: count%) : bool + %{ + Connection* c = sessions->FindConnection(cid); + if ( ! c ) + { + reporter->Error("cannot find connection"); + return new Val(0, TYPE_BOOL); + } + + Analyzer* a = c->FindAnalyzer(aid); + if ( ! a ) + { + reporter->Error("connection does not have analyzer specified to disable"); + return new Val(0, TYPE_BOOL); + } + + a->Remove(); + return new Val(1, TYPE_BOOL); + %} + +## Translate an analyzer type to an ASCII string. +## +## aid: The analyzer ID. +## +## Returns: The analyzer *aid* as string. +## +## .. bro:see:: expect_connection disable_analyzer current_analyzer +function analyzer_name%(aid: count%) : string + %{ + return new StringVal(Analyzer::GetTagName((AnalyzerTag::Tag) aid)); + %} + +## Informs Bro that it should skip any further processing of the contents of +## a given connection. In particular, Bro will refrain from reassembling the +## TCP byte stream and from generating events relating to any analyzers that +## have been processing the connection. +## +## cid: The connection ID. +## +## Returns: False if *id* does not point to an active connection and true +## otherwise. +## +## .. note:: +## +## Bro will still generate connection-oriented events such as +## :bro:id:`connection_finished`. +function skip_further_processing%(cid: conn_id%): bool + %{ + Connection* c = sessions->FindConnection(cid); + if ( ! c ) + return new Val(0, TYPE_BOOL); + + c->SetSkip(1); + return new Val(1, TYPE_BOOL); + %} + +## Controls whether packet contents belonging to a connection should be +## recorded (when ``-w`` option is provided on the command line). +## +## cid: The connection identifier. +## +## do_record: True to enable packet contens and false to disable for the +## connection identified by *cid*. +## +## Returns: False if *id* does not point to an active connection and true +## otherwise. +## +## .. bro:see:: skip_further_processing +## +## .. note:: +## +## This is independent of whether Bro processes the packets of this +## connection, which is controlled separately by +## :bro:id:`skip_further_processing`. +## +## .. bro:see: get_contents_file set_contents_file +function set_record_packets%(cid: conn_id, do_record: bool%): bool + %{ + Connection* c = sessions->FindConnection(cid); + if ( ! c ) + return new Val(0, TYPE_BOOL); + + c->SetRecordPackets(do_record); + return new Val(1, TYPE_BOOL); + %} + +## Associates a file handle with a connection for writing TCP byte stream +## contents. +## +## cid: The connection ID. +## +## direction: Controls what sides of the connection to record. The argument can +## take one the four values: +## +## - ``CONTENTS_NONE``: Stop recording the connection's content. +## - ``CONTENTS_ORIG``: Record the data sent by the connection +## originator (often the client). +## - ``CONTENTS_RESP``: Record the data sent by the connection +## responder (often the server). +## - ``CONTENTS_BOTH``: Record the data sent in both directions. +## Results in the two directions being +## intermixed in the file, in the order the +## data was seen by Bro. +## +## f: The file handle of the file to write the contents to. +## +## Returns: Returns false if *id* does not point to an active connection and +## true otherwise. +## +## .. note:: +## +## The data recorded to the file reflects the byte stream, not the +## contents of individual packets. Reordering and duplicates are +## removed. If any data is missing, the recording stops at the +## missing data; this can happen, e.g., due to an +## :bro:id:`ack_above_hole` event. +## +## .. bro:see: get_contents_file set_record_packets +function set_contents_file%(cid: conn_id, direction: count, f: file%): bool + %{ + Connection* c = sessions->FindConnection(cid); + if ( ! c ) + return new Val(0, TYPE_BOOL); + + c->GetRootAnalyzer()->SetContentsFile(direction, f); + return new Val(1, TYPE_BOOL); + %} + +## Returns the file handle of the contents file of a connection. +## +## cid: The connection ID. +## +## direction: Controls what sides of the connection to record. SEe +## :bro:id:`set_contents_file` for possible values. +## +## Returns: The :bro:type:`file` handle for the contentents file of the +## connection identified by *cid*. If the connection exists +## but no contents file for *direction*, the function generates a +## error and returns a file handle to ``stderr``. +## +## .. bro:see: set_contents_file set_record_packets +function get_contents_file%(cid: conn_id, direction: count%): file + %{ + Connection* c = sessions->FindConnection(cid); + BroFile* f = c ? c->GetRootAnalyzer()->GetContentsFile(direction) : 0; + + if ( f ) + { + Ref(f); + return new Val(f); + } + + // Return some sort of error value. + if ( ! c ) + builtin_error("unknown connection id in get_contents_file()", cid); + else + builtin_error("no contents file for given direction"); + + return new Val(new BroFile(stderr, "-", "w")); + %} + +## Sets an individual inactivity timeout for a connection and thus +## overrides the global inactivity timeout. +## +## cid: The connection ID. +## +## t: The new inactivity timeout for the connection identified by *cid*. +## +## Returns: The previous timeout interval. +function set_inactivity_timeout%(cid: conn_id, t: interval%): interval + %{ + Connection* c = sessions->FindConnection(cid); + if ( ! c ) + return new Val(0, TYPE_INTERVAL); + + double old_timeout = c->InactivityTimeout(); + c->SetInactivityTimeout(t); + + return new Val(old_timeout, TYPE_INTERVAL); + %} + +## Returns the state of the given login (Telnet or Rlogin) connection. +## +## cid: The connection ID. +## +## Returns: False if the connection is not active or is not tagged as a +## login analyzer. Otherwise the function returns the state, which can +## be one of: +## +## - ``LOGIN_STATE_AUTHENTICATE``: The connection is in its +## initial authentication dialog. +## - ``OGIN_STATE_LOGGED_IN``: The analyzer believes the user has +## successfully authenticated. +## - ``LOGIN_STATE_SKIP``: The analyzer has skipped any further +## processing of the connection. +## - ``LOGIN_STATE_CONFUSED``: The analyzer has concluded that it +## does not correctly know the state of the connection, and/or +## the username associated with it. +## +## .. bro:see: set_login_state +function get_login_state%(cid: conn_id%): count + %{ + Connection* c = sessions->FindConnection(cid); + if ( ! c ) + return new Val(0, TYPE_BOOL); + + Analyzer* la = c->FindAnalyzer(AnalyzerTag::Login); + if ( ! la ) + return new Val(0, TYPE_BOOL); + + return new Val(int(static_cast(la)->LoginState()), + TYPE_COUNT); + %} + +## Sets the login state of a connection with a login analyzer. +## +## cid: The connection ID. +## +## new_state: The new state of the login analyzer. See +## :bro:id:`get_login_state` for possible values. +## +## Returns: Returns false if *cid* is not an active connection +## or does not tagged as login analyzer, and true otherwise. +## +## .. bro:see: get_login_state +function set_login_state%(cid: conn_id, new_state: count%): bool + %{ + Connection* c = sessions->FindConnection(cid); + if ( ! c ) + return new Val(0, TYPE_BOOL); + + Analyzer* la = c->FindAnalyzer(AnalyzerTag::Login); + if ( ! la ) + return new Val(0, TYPE_BOOL); + + static_cast(la)->SetLoginState(login_state(new_state)); + return new Val(1, TYPE_BOOL); + %} + +%%{ +#include "TCP.h" +%%} + +## Get the originator sequence number of a TCP connection. Sequence numbers +## are absolute (i.e., they reflect the values seen directly in packet headers; +## they are not relative to the beginning of the connection). +## +## cid: The connection ID. +## +## Returns: The highest sequence number sent by a connection's originator, or 0 +## if *cid* does not point to an active TCP connection. +## +## .. bro:see:: get_resp_seq +function get_orig_seq%(cid: conn_id%): count + %{ + Connection* c = sessions->FindConnection(cid); + if ( ! c ) + return new Val(0, TYPE_COUNT); + + if ( c->ConnTransport() != TRANSPORT_TCP ) + return new Val(0, TYPE_COUNT); + + Analyzer* tc = c->FindAnalyzer(AnalyzerTag::TCP); + if ( tc ) + return new Val(static_cast(tc)->OrigSeq(), + TYPE_COUNT); + else + { + reporter->Error("connection does not have TCP analyzer"); + return new Val(0, TYPE_COUNT); + } + %} + +## Get the responder sequence number of a TCP connection. Sequence numbers +## are absolute (i.e., they reflect the values seen directly in packet headers; +## they are not relative to the beginning of the connection). +## +## cid: The connection ID. +## +## Returns: The highest sequence number sent by a connection's responder, or 0 +## if *cid* does not point to an active TCP connection. +## +## .. bro:see:: get_orig_seq +function get_resp_seq%(cid: conn_id%): count + %{ + Connection* c = sessions->FindConnection(cid); + if ( ! c ) + return new Val(0, TYPE_COUNT); + + if ( c->ConnTransport() != TRANSPORT_TCP ) + return new Val(0, TYPE_COUNT); + + Analyzer* tc = c->FindAnalyzer(AnalyzerTag::TCP); + if ( tc ) + return new Val(static_cast(tc)->RespSeq(), + TYPE_COUNT); + else + { + reporter->Error("connection does not have TCP analyzer"); + return new Val(0, TYPE_COUNT); + } + %} + +%%{ +#include "SMTP.h" +%%} + +## Skips SMTP data until the next email in a connection. +## +## c: The SMTP connection. +## +## .. bro:see:: skip_http_entity_data +function skip_smtp_data%(c: connection%): any + %{ + Analyzer* sa = c->FindAnalyzer(AnalyzerTag::SMTP); + if ( sa ) + static_cast(sa)->SkipData(); + return 0; + %} + +## Enables all event handlers in a given group. One can tag event handlers with +## the :bro:attr:`&group` attribute to logically group them together, e.g, +## `` event foo() &group="bar"``. This function enables all event handlers that +## belong to such a group. +## +## group: The group. +## +## .. bro:see:: disable_event_group +function enable_event_group%(group: string%) : any + %{ + event_registry->EnableGroup(group->CheckString(), true); + return 0; + %} + +## Disables all event handlers in a given group. +## +## group: The group. +## +## .. bro:see:: enable_event_group +function disable_event_group%(group: string%) : any + %{ + event_registry->EnableGroup(group->CheckString(), false); + return 0; + %} + +# =========================================================================== +# +# Files and Directories +# +# =========================================================================== + +## Opens a file for writing. If a file with the same name already exists, this +## function overwrites it (as opposed to :bro:id:`open_for_append`). +## +## f: The path to the file. +## +## Returns: A :bro:type:`file` handle for subsequent operations. +## +## .. bro:see;: active_file open_for_append close write_file +## get_file_name set_buf flush_all mkdir enable_raw_output +function open%(f: string%): file + %{ + const char* file = f->CheckString(); + + if ( streq(file, "-") ) + return new Val(new BroFile(stdout, "-", "w")); + else + return new Val(new BroFile(file, "w")); + %} + +## Opens a file for writing or appending. If a file with the same name already +## exists, this function appends to it (as opposed to :bro:id:`open`). +## +## f: The path to the file. +## +## Returns: A :bro:type:`file` handle for subsequent operations. +## +## .. bro:see;: active_file open close write_file +## get_file_name set_buf flush_all mkdir enable_raw_output +function open_for_append%(f: string%): file + %{ + return new Val(new BroFile(f->CheckString(), "a")); + %} + +## Closes an open file and flushes any buffered content. +## exists, this function appends to it (as opposed to :bro:id:`open`). +## +## f: A :bro:type:`file` handle to an open file. +## +## Returns: True on success. +## +## .. bro:see;: active_file open open_for_append write_file +## get_file_name set_buf flush_all mkdir enable_raw_output +function close%(f: file%): bool + %{ + return new Val(f->Close(), TYPE_BOOL); + %} + +## Writes data to an open file. +## +## f: A :bro:type:`file` handle to an open file. +## +## data: The data to write to *f*. +## +## Returns: True on success. +## +## .. bro:see;: active_file open open_for_append close +## get_file_name set_buf flush_all mkdir enable_raw_output +function write_file%(f: file, data: string%): bool + %{ + if ( ! f ) + return new Val(0, TYPE_BOOL); + + return new Val(f->Write((const char*) data->Bytes(), data->Len()), + TYPE_BOOL); + %} + +## Alters the buffering behavior of a file. +## +## f: A :bro:type:`file` handle to an open file. +## +## buffered: When true, *f* is fully buffered, i.e., bytes are saved in a +## buffered until the block size has been reached. When +## false, *f* is line buffered, i.e., bytes are saved up until a +## newline occurs. +## +## .. bro:see;: active_file open open_for_append close +## get_file_name write_file flush_all mkdir enable_raw_output +function set_buf%(f: file, buffered: bool%): any + %{ + f->SetBuf(buffered); + return new Val(0, TYPE_VOID); + %} + +## Flushes all open files to disk. +## +## Returns: True on success. +## +## .. bro:see;: active_file open open_for_append close +## get_file_name write_file set_buf mkdir enable_raw_output +function flush_all%(%): bool + %{ + return new Val(fflush(0) == 0, TYPE_BOOL); + %} + +## Creates a new directory. +## +## f: The directory name. +## +## Returns: Returns true if the operation succeeds and false if the +## creation fails or if *f* exists already. +## +## .. bro:see;: active_file open_for_append close write_file +## get_file_name set_buf flush_all enable_raw_output +function mkdir%(f: string%): bool + %{ + const char* filename = f->CheckString(); + if ( mkdir(filename, 0777) < 0 && errno != EEXIST ) + { + builtin_error("cannot create directory", @ARG@[0]); + return new Val(0, TYPE_BOOL); + } + else + return new Val(1, TYPE_BOOL); + %} + +## Checks whether a given file is open. +## +## f: The file to check. +## +## Returns: True if *f* is an open :bro:type:`file`. +## +## .. todo:: Rename to ``is_open``. +function active_file%(f: file%): bool + %{ + return new Val(f->IsOpen(), TYPE_BOOL); + %} + +## Gets the filename associated with a file handle. +## +## f: The file handle to inquire the name for. +## +## Returns: The filename associated with *f*. +## +## .. bro:see:: open +function get_file_name%(f: file%): string + %{ + if ( ! f ) + return new StringVal(""); + + return new StringVal(f->Name()); + %} + +## Rotates a file. +## +## f: An open file handle. +## +## Returns: Rotations statistics which include the original file name, the name +## after the rotation, and the time when *f* was opened/closed. +## +## .. bro:see:: rotate_file_by_name calc_next_rotate +function rotate_file%(f: file%): rotate_info + %{ + RecordVal* info = f->Rotate(); + if ( info ) + return info; + + // Record indicating error. + info = new RecordVal(rotate_info); + info->Assign(0, new StringVal("")); + info->Assign(1, new StringVal("")); + info->Assign(2, new Val(0, TYPE_TIME)); + info->Assign(3, new Val(0, TYPE_TIME)); + + return info; + %} + +## Rotates a file identified by its name. +## +## f: The name of the file to rotate +## +## Returns: Rotations statistics which include the original file name, the name +## after the rotation, and the time when *f* was opened/closed. +## +## .. bro:see:: rotate_file calc_next_rotate +function rotate_file_by_name%(f: string%): rotate_info + %{ + RecordVal* info = new RecordVal(rotate_info); + + bool is_pkt_dumper = false; + bool is_addl_pkt_dumper = false; + + // Special case: one of current dump files. + if ( pkt_dumper && streq(pkt_dumper->FileName(), f->CheckString()) ) + { + is_pkt_dumper = true; + pkt_dumper->Close(); + } + + if ( addl_pkt_dumper && + streq(addl_pkt_dumper->FileName(), f->CheckString()) ) + { + is_addl_pkt_dumper = true; + addl_pkt_dumper->Close(); + } + + FILE* file = rotate_file(f->CheckString(), info); + if ( ! file ) + { + // Record indicating error. + info->Assign(0, new StringVal("")); + info->Assign(1, new StringVal("")); + info->Assign(2, new Val(0, TYPE_TIME)); + info->Assign(3, new Val(0, TYPE_TIME)); + return info; + } + + fclose(file); + + if ( is_pkt_dumper ) + { + info->Assign(2, new Val(pkt_dumper->OpenTime(), TYPE_TIME)); + pkt_dumper->Open(); + } + + if ( is_addl_pkt_dumper ) + info->Assign(2, new Val(addl_pkt_dumper->OpenTime(), TYPE_TIME)); + + return info; + %} + +## Calculates the duration until the next time a file is to be rotated, based +## on a given rotate interval. +## +## i: The rotate interval to base the calculation on. +## +## Returns: The duration until the next file rotation time. +## +## .. bro:see:: rotate_file rotate_file_by_name +function calc_next_rotate%(i: interval%) : interval + %{ + const char* base_time = log_rotate_base_time ? + log_rotate_base_time->AsString()->CheckString() : 0; + return new Val(calc_next_rotate(i, base_time), TYPE_INTERVAL); + %} + +## Returns the size of a given file. +## +## f: The name of the file whose size to lookup. +## +## Returns: The size of *f* in bytes. +function file_size%(f: string%) : double + %{ + struct stat s; + + if ( stat(f->CheckString(), &s) < 0 ) + return new Val(-1.0, TYPE_DOUBLE); + + return new Val(double(s.st_size), TYPE_DOUBLE); + %} + +## Disables sending :bro:id:`print_hook` events to remote peers for a given +## file. This function is equivalent to :bro:attr:`&disable_print_hook`. In a +## distributed setup, communicating Bro instances generate the event +## :bro:id:`print_hook` for each print statement and send it to the remote +## side. When disabled for a particular file, these events will not be +## propagated to other peers. +## +## f: The file to disable :bro:id:`print_hook` events for. +## +## .. bro:see:: enable_raw_output +function disable_print_hook%(f: file%): any + %{ + f->DisablePrintHook(); + return 0; + %} + +## Prevents escaping of non-ASCII character when writing to a file. +## This function is equivalent to :bro:attr:`&disable_print_hook`. +## +## f: The file to disable raw output for. +## +## .. bro:see:: disable_print_hook +function enable_raw_output%(f: file%): any + %{ + f->EnableRawOutput(); + return 0; + %} + +# =========================================================================== +# +# Packet Filtering +# +# =========================================================================== + ## Precompiles a PCAP filter and binds it to a given identifier. ## ## id: The PCAP identifier to reference the filter *s* later on. ## ## s: The PCAP filter. See ``man tcpdump`` for valid expressions. -## +## ## Returns: True if *s* is valid and precompiles successfully. ## ## .. bro:see:: install_pcap_filter @@ -3201,24 +4807,44 @@ function uninstall_dst_net_filter%(snet: subnet%) : bool return new Val(sessions->GetPacketFilter()->RemoveDst(snet), TYPE_BOOL); %} +# =========================================================================== +# +# Communication +# +# =========================================================================== + +## Enables the communication system. By default, the communication is off until +## explicitly enabled, and all other calls to communication-related functions +## will be ignored until done so. +function enable_communication%(%): any + %{ + if ( bro_start_network_time != 0.0 ) + { + builtin_error("communication must be enabled in bro_init"); + return 0; + } + + if ( using_communication ) + // Ignore duplicate calls. + return 0; + + using_communication = 1; + remote_serializer->Init(); + return 0; + %} + ## Flushes in-memory state tagged with the :bro:attr:`&persistence` attribute ## to disk. The function writes the state to the file ``.state/state.bst`` in ## the directory where Bro was started. -## +## ## Returns: True on success. ## -## .. bro:see:: rescan_state +## .. bro:see:: rescan_state function checkpoint_state%(%) : bool %{ return new Val(persistence_serializer->WriteState(true), TYPE_BOOL); %} -## Deprecated. Will be removed. -function dump_config%(%) : bool - %{ - return new Val(persistence_serializer->WriteConfig(true), TYPE_BOOL); - %} - ## Reads persistent state from the \texttt{.state} directory and populates the ## in-memory data structures accordingly. This function is the dual to ## :bro:id:`checkpoint_state`. @@ -3400,7 +5026,7 @@ function set_compression_level%(p: event_peer, level: count%) : bool TYPE_BOOL); %} -## Listens on address a given IP address and port for remote connections. +## Listens on address a given IP address and port for remote connections. ## ## ip: The IP address to bind to. ## @@ -3492,7 +5118,7 @@ function complete_handshake%(p: event_peer%) : bool ## seq: A sequence number (also included by :bro:id:`remote_pong`). ## ## Returns: True if sending the ping succeeds. -## +## ## .. bro:see:: send_state send_id send_current_packet send_capture_filter function send_ping%(p: event_peer, seq: count%) : bool %{ @@ -3505,7 +5131,7 @@ function send_ping%(p: event_peer, seq: count%) : bool ## p: The peer ID return from :bro:id:`connect`. ## ## Returns: True if sending the packet succeeds. -## +## ## .. bro:see:: send_id send_state send_ping send_capture_filter ## dump_packet dump_current_packet get_current_packet function send_current_packet%(p: event_peer%) : bool @@ -3526,24 +5152,6 @@ function send_current_packet%(p: event_peer%) : bool return new Val(remote_serializer->SendPacket(&info, id, pkt), TYPE_BOOL); %} -## Enables detailed collections of statistics about CPU/memory usage, -## connections, TCP states/reassembler, DNS lookups, timers, and script-level -## state. The script variable :bro:id:`profiling_file` holds the name of the -## file. -## -## .. bro:see:: net_stats -## resource_usage -## get_matcher_stats -## dump_rule_stats -## get_gap_summary -function do_profiling%(%) : bool - %{ - if ( profiling_logger ) - profiling_logger->Log(); - - return new Val(1, TYPE_BOOL); - %} - ## Returns the peer who generated the last event. ## ## Returns: The ID of the peer who genereated the last event. @@ -3587,7 +5195,6 @@ function get_local_event_peer%(%) : event_peer return p; %} - ## Sends a capture filter to a remote peer. ## ## p: The peer ID return from :bro:id:`connect`. @@ -3603,6 +5210,162 @@ function send_capture_filter%(p: event_peer, s: string%) : bool return new Val(remote_serializer->SendCaptureFilter(id, s->CheckString()), TYPE_BOOL); %} +## Stops Bro's packet processing. This function is used to synchronize +## distributed trace processing with communication enabled +## (*pseudo-realtime* mode). +## +## .. bro:see: continue_processing suspend_state_updates resume_state_updates +function suspend_processing%(%) : any + %{ + net_suspend_processing(); + return 0; + %} + +## Resumes Bro's packet processing. +## +## .. bro:see: suspend_processing suspend_state_updates resume_state_updates +function continue_processing%(%) : any + %{ + net_continue_processing(); + return 0; + %} + +## Stops propagating :bro:attr:`&synchronized` accesses. +## +## .. bro:see: suspend_processing continue_processing resume_state_updates +function suspend_state_updates%(%) : any + %{ + if ( remote_serializer ) + remote_serializer->SuspendStateUpdates(); + return 0; + %} + +## Resumes propagating :bro:attr:`&synchronized` accesses. +## +## .. bro:see: suspend_processing continue_processing suspend_state_updates +function resume_state_updates%(%) : any + %{ + if ( remote_serializer ) + remote_serializer->ResumeStateUpdates(); + return 0; + %} + +# =========================================================================== +# +# Deprecated Functions +# +# =========================================================================== + +## .. todo:: document and move to the right spot +function match_signatures%(c: connection, pattern_type: int, s: string, + bol: bool, eol: bool, + from_orig: bool, clear: bool%) : bool + %{ + if ( ! rule_matcher ) + return new Val(0, TYPE_BOOL); + + c->Match((Rule::PatternType) pattern_type, s->Bytes(), s->Len(), + from_orig, bol, eol, clear); + + return new Val(1, TYPE_BOOL); + %} + +## Deprecated. Will be removed. +function active_connection%(id: conn_id%): bool + %{ + Connection* c = sessions->FindConnection(id); + return new Val(c ? 1 : 0, TYPE_BOOL); + %} + +## Deprecated. Will be removed. +function connection_record%(cid: conn_id%): connection + %{ + Connection* c = sessions->FindConnection(cid); + if ( c ) + return c->BuildConnVal(); + else + { + // Hard to recover from this until we have union types ... + builtin_error("connection ID not a known connection (fatal)", cid); + exit(0); + return 0; + } + %} + +%%{ +#include "Anon.h" +%%} + +## Preserve prefix as original one in anonymization. +## .. todo: Currently broken. +function preserve_prefix%(a: addr, width: count%): any + %{ + AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50]; + if ( ip_anon ) + { +#ifdef BROv6 + if ( ! is_v4_addr(a) ) + builtin_error("preserve_prefix() not supported for IPv6 addresses"); + else + ip_anon->PreservePrefix(a[3], width); +#else + ip_anon->PreservePrefix(a, width); +#endif + } + + + return 0; + %} + +## .. todo: Currently broken. +function preserve_subnet%(a: subnet%): any + %{ + DEBUG_MSG("%s/%d\n", dotted_addr(a->AsAddr()), a->Width()); + AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50]; + if ( ip_anon ) + { +#ifdef BROv6 + if ( ! is_v4_addr(a->AsAddr()) ) + builtin_error("preserve_subnet() not supported for IPv6 addresses"); + else + ip_anon->PreservePrefix(a->AsAddr()[3], a->Width()); +#else + ip_anon->PreservePrefix(a->AsAddr(), a->Width()); +#endif + } + + return 0; + %} + +## Anonymize an IP address. +## .. todo: Currently broken. +function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr + %{ + int anon_class = cl->InternalInt(); + if ( anon_class < 0 || anon_class >= NUM_ADDR_ANONYMIZATION_CLASSES ) + builtin_error("anonymize_addr(): invalid ip addr anonymization class"); + +#ifdef BROv6 + if ( ! is_v4_addr(a) ) + { + builtin_error("anonymize_addr() not supported for IPv6 addresses"); + return 0; + } + else + return new AddrVal(anonymize_ip(a[3], + (enum ip_addr_anonymization_class_t) anon_class)); +#else + return new AddrVal(anonymize_ip(a, + (enum ip_addr_anonymization_class_t) anon_class)); +#endif + %} + +## Deprecated. Will be removed. +function dump_config%(%) : bool + %{ + return new Val(persistence_serializer->WriteConfig(true), TYPE_BOOL); + %} + ## Deprecated. Will be removed. function make_connection_persistent%(c: connection%) : bool %{ @@ -3610,411 +5373,6 @@ function make_connection_persistent%(c: connection%) : bool return new Val(1, TYPE_BOOL); %} -## Checks whether a given IP address belongs to a local interface. -## -## ip: The IP address to check. -## -## Returns: True if *ip* belongs to a local interface. -function is_local_interface%(ip: addr%) : bool - %{ - static uint32* addrs; - static int len = -1; - - if ( len < 0 ) - { - char host[MAXHOSTNAMELEN]; - - strcpy(host, "localhost"); - gethostname(host, MAXHOSTNAMELEN); - host[MAXHOSTNAMELEN-1] = '\0'; - - struct hostent* ent = gethostbyname(host); - - for ( len = 0; ent->h_addr_list[len]; ++len ) - ; - - addrs = new uint32[len + 1]; - for ( int i = 0; i < len; i++ ) - addrs[i] = *(uint32*) ent->h_addr_list[i]; - - addrs[len++] = 0x0100007f; // 127.0.0.1 - } - -#ifdef BROv6 - if ( ! is_v4_addr(ip) ) - { - builtin_error("is_local_interface() only supports IPv4 addresses"); - return new Val(0, TYPE_BOOL); - } - - uint32 ip4 = to_v4_addr(ip); -#else - uint32 ip4 = ip; -#endif - - for ( int i = 0; i < len; i++ ) - if ( addrs[i] == ip4 ) - return new Val(1, TYPE_BOOL); - - return new Val(0, TYPE_BOOL); - %} - -## Writes the current packet to a file. -## -## file_name: The name of the file to write the packet to. -## -## Returns: True on success. -## -## .. bro:see:: dump_packet get_current_packet send_current_packet -function dump_current_packet%(file_name: string%) : bool - %{ - const struct pcap_pkthdr* hdr; - const u_char* pkt; - - if ( ! current_pktsrc || - ! current_pktsrc->GetCurrentPacket(&hdr, &pkt) ) - return new Val(0, TYPE_BOOL); - - if ( ! addl_pkt_dumper ) - addl_pkt_dumper = new PktDumper(0, true); - - addl_pkt_dumper->Open(file_name->CheckString()); - addl_pkt_dumper->Dump(hdr, pkt); - - return new Val(! addl_pkt_dumper->IsError(), TYPE_BOOL); - %} - -## Returns the currently processed PCAP packet. -## -## Returns: The currently processed packet, which is as a record -## containing the timestamp, ``snaplen,'' and packet data. -## -## .. bro:see:: dump_current_packet dump_packet send_current_packet -function get_current_packet%(%) : pcap_packet - %{ - const struct pcap_pkthdr* hdr; - const u_char* data; - RecordVal* pkt = new RecordVal(pcap_packet); - - if ( ! current_pktsrc || - ! current_pktsrc->GetCurrentPacket(&hdr, &data) ) - { - pkt->Assign(0, new Val(0, TYPE_COUNT)); - pkt->Assign(1, new Val(0, TYPE_COUNT)); - pkt->Assign(2, new Val(0, TYPE_COUNT)); - pkt->Assign(3, new Val(0, TYPE_COUNT)); - pkt->Assign(4, new StringVal("")); - return pkt; - } - - pkt->Assign(0, new Val(uint32(hdr->ts.tv_sec), TYPE_COUNT)); - pkt->Assign(1, new Val(uint32(hdr->ts.tv_usec), TYPE_COUNT)); - pkt->Assign(2, new Val(hdr->caplen, TYPE_COUNT)); - pkt->Assign(3, new Val(hdr->len, TYPE_COUNT)); - pkt->Assign(4, new StringVal(hdr->caplen, (const char*) data)); - - return pkt; - %} - -## Writes a given packet to a file. -## -## pkt: The PCAP packet. -## -## file_name: The name of the file to write *pkt* to. -## -## Returns: True on success -## -## .. bro:see:: get_current_packet dump_current_packet send_current_packet -function dump_packet%(pkt: pcap_packet, file_name: string%) : bool - %{ - struct pcap_pkthdr hdr; - const val_list* pkt_vl = pkt->AsRecord(); - - hdr.ts.tv_sec = (*pkt_vl)[0]->AsCount(); - hdr.ts.tv_usec = (*pkt_vl)[1]->AsCount(); - hdr.caplen = (*pkt_vl)[2]->AsCount(); - hdr.len = (*pkt_vl)[3]->AsCount(); - - if ( ! addl_pkt_dumper ) - addl_pkt_dumper = new PktDumper(0, true); - - addl_pkt_dumper->Open(file_name->CheckString()); - addl_pkt_dumper->Dump(&hdr, (*pkt_vl)[4]->AsString()->Bytes()); - - return new Val(addl_pkt_dumper->IsError(), TYPE_BOOL); - %} - -## Resizes a vector. -## -## aggr: The vector instance. -## -## newsize: The new size of *aggr*. -## -## Returns: The old size of *aggr* and 0 if *aggr* is not a :bro:type:`vector`. -function resize%(aggr: any, newsize: count%) : count - %{ - if ( aggr->Type()->Tag() != TYPE_VECTOR ) - { - builtin_error("resize() operates on vectors"); - return 0; - } - - return new Val(aggr->AsVectorVal()->Resize(newsize), TYPE_COUNT); - %} - -## Tests whether a boolean vector (``vector of bool``) has *any* true -## element. -## -## v: The boolean vector instance. -## -## Returns: True if any element in *v* is true. -## -## :bro:see:: all_set -function any_set%(v: any%) : bool - %{ - if ( v->Type()->Tag() != TYPE_VECTOR || - v->Type()->YieldType()->Tag() != TYPE_BOOL ) - { - builtin_error("any_set() requires vector of bool"); - return new Val(false, TYPE_BOOL); - } - - VectorVal* vv = v->AsVectorVal(); - for ( unsigned int i = 0; i < vv->Size(); ++i ) - if ( vv->Lookup(i) && vv->Lookup(i)->AsBool() ) - return new Val(true, TYPE_BOOL); - - return new Val(false, TYPE_BOOL); - %} - -## Tests whether *all* elements of a boolean vector (``vector of bool``) are -## true. -## -## v: The boolean vector instance. -## -## Returns: True iff all elements in *v* are true. -## -## :bro:see:: any_set -## -## .. note:: -## -## Missing elements count as false. -function all_set%(v: any%) : bool - %{ - if ( v->Type()->Tag() != TYPE_VECTOR || - v->Type()->YieldType()->Tag() != TYPE_BOOL ) - { - builtin_error("all_set() requires vector of bool"); - return new Val(false, TYPE_BOOL); - } - - VectorVal* vv = v->AsVectorVal(); - for ( unsigned int i = 0; i < vv->Size(); ++i ) - if ( ! vv->Lookup(i) || ! vv->Lookup(i)->AsBool() ) - return new Val(false, TYPE_BOOL); - - return new Val(true, TYPE_BOOL); - %} - -%%{ -static Func* sort_function_comp = 0; -static Val** index_map = 0; // used for indirect sorting to support order() - -bool sort_function(Val* a, Val* b) - { - // Sort missing values as "high". - if ( ! a ) - return 0; - if ( ! b ) - return 1; - - val_list sort_func_args; - sort_func_args.append(a->Ref()); - sort_func_args.append(b->Ref()); - - Val* result = sort_function_comp->Call(&sort_func_args); - int int_result = result->CoerceToInt(); - Unref(result); - - sort_func_args.remove_nth(1); - sort_func_args.remove_nth(0); - - return int_result < 0; - } - -bool indirect_sort_function(int a, int b) - { - return sort_function(index_map[a], index_map[b]); - } - -bool int_sort_function (Val* a, Val* b) - { - if ( ! a ) - return 0; - if ( ! b ) - return 1; - - int ia = a->CoerceToInt(); - int ib = b->CoerceToInt(); - - return ia < ib; - } - -bool indirect_int_sort_function(int a, int b) - { - return int_sort_function(index_map[a], index_map[b]); - } -%%} - -## Sorts a vector in place. The second argument is a comparison function that -## takes two arguments: if the vector type is \verb|vector of T|, then the -## comparison function must be ``function(a: T, b: T): bool``, which returns -## ``a < b`` for some type-specific notion of the less-than operator. -## -## v: The vector instance to sort. -## -## Returns: The original vector. -## -## .. bro:see:: order -function sort%(v: any, ...%) : any - %{ - v->Ref(); // we always return v - - if ( v->Type()->Tag() != TYPE_VECTOR ) - { - builtin_error("sort() requires vector"); - return v; - } - - BroType* elt_type = v->Type()->YieldType(); - Func* comp = 0; - - if ( @ARG@.length() > 2 ) - builtin_error("sort() called with extraneous argument"); - - if ( @ARG@.length() == 2 ) - { - Val* comp_val = @ARG@[1]; - if ( ! IsFunc(comp_val->Type()->Tag()) ) - { - builtin_error("second argument to sort() needs to be comparison function"); - return v; - } - - comp = comp_val->AsFunc(); - } - - if ( ! comp && ! IsIntegral(elt_type->Tag()) ) - builtin_error("comparison function required for sort() with non-integral types"); - - vector& vv = *v->AsVector(); - - if ( comp ) - { - FuncType* comp_type = comp->FType()->AsFuncType(); - if ( comp_type->YieldType()->Tag() != TYPE_INT || - ! comp_type->ArgTypes()->AllMatch(elt_type, 0) ) - { - builtin_error("invalid comparison function in call to sort()"); - return v; - } - - sort_function_comp = comp; - - sort(vv.begin(), vv.end(), sort_function); - } - else - sort(vv.begin(), vv.end(), int_sort_function); - - return v; - %} - -## Returns the order of the elements in a vector according to some -## comparison function. See :bro:id:`sort` for details about the comparison -## function. -## -## v: The vector whose order to compute. -## -## Returns: A ``vector of count`` with the indices of the ordered elements. -## -## .. bro:see:: sort -function order%(v: any, ...%) : index_vec - %{ - VectorVal* result_v = - new VectorVal(new VectorType(base_type(TYPE_COUNT))); - - if ( v->Type()->Tag() != TYPE_VECTOR ) - { - builtin_error("order() requires vector"); - return result_v; - } - - BroType* elt_type = v->Type()->YieldType(); - Func* comp = 0; - - if ( @ARG@.length() > 2 ) - builtin_error("order() called with extraneous argument"); - - if ( @ARG@.length() == 2 ) - { - Val* comp_val = @ARG@[1]; - if ( ! IsFunc(comp_val->Type()->Tag()) ) - { - builtin_error("second argument to order() needs to be comparison function"); - return v; - } - - comp = comp_val->AsFunc(); - } - - if ( ! comp && ! IsIntegral(elt_type->Tag()) ) - builtin_error("comparison function required for sort() with non-integral types"); - - vector& vv = *v->AsVector(); - int n = vv.size(); - - // Set up initial mapping of indices directly to corresponding - // elements. We stay zero-based until after the sorting. - vector ind_vv(n); - index_map = new Val*[n]; - int i; - for ( i = 0; i < n; ++i ) - { - ind_vv[i] = i; - index_map[i] = vv[i]; - } - - if ( comp ) - { - FuncType* comp_type = comp->FType()->AsFuncType(); - if ( comp_type->YieldType()->Tag() != TYPE_INT || - ! comp_type->ArgTypes()->AllMatch(elt_type, 0) ) - { - builtin_error("invalid comparison function in call to sort()"); - return v; - } - - sort_function_comp = comp; - - sort(ind_vv.begin(), ind_vv.end(), indirect_sort_function); - } - else - sort(ind_vv.begin(), ind_vv.end(), indirect_int_sort_function); - - delete [] index_map; - index_map = 0; - - // Now spin through ind_vv to read out the rearrangement, - // adjusting indices as we do so. - for ( i = 0; i < n; ++i ) - { - int ind = ind_vv[i]; - result_v->Assign(i, new Val(ind, TYPE_COUNT), 0); - } - - return result_v; - %} - %%{ // Experimental code to add support for IDMEF XML output based on // notices. For now, we're implementing it as a builtin you can call on an @@ -4075,989 +5433,7 @@ function generate_idmef%(src_ip: addr, src_port: port, #endif %} -## Write rule matcher statistics (DFA states, transitions, memory usage, cache -## hits/misses) to a file. -## -## f: The file to write to. -## -## Returns: True (unconditionally). -## -## .. bro:see:: do_profiling -## resource_usage -## get_matcher_stats -## net_stats -## get_gap_summary -## -## .. todo:: The return value should be changed to any or check appropriately. -function dump_rule_stats%(f: file%): bool - %{ - if ( rule_matcher ) - rule_matcher->DumpStats(f); - - return new Val(1, TYPE_BOOL); - %} - -## Checks wheter Bro is terminating. -## -## Returns: True if Bro is in the process of shutting down. -## -## .. bro:see: terminate -function bro_is_terminating%(%): bool - %{ - return new Val(terminating, TYPE_BOOL); - %} - -## Rotates a file. -## -## f: An open file handle. -## -## Returns: Rotations statistics which include the original file name, the name -## after the rotation, and the time when *f* was opened/closed. -## -## .. bro:see:: rotate_file_by_name calc_next_rotate -function rotate_file%(f: file%): rotate_info - %{ - RecordVal* info = f->Rotate(); - if ( info ) - return info; - - // Record indicating error. - info = new RecordVal(rotate_info); - info->Assign(0, new StringVal("")); - info->Assign(1, new StringVal("")); - info->Assign(2, new Val(0, TYPE_TIME)); - info->Assign(3, new Val(0, TYPE_TIME)); - - return info; - %} - -## Rotates a file identified by its name. -## -## f: The name of the file to rotate -## -## Returns: Rotations statistics which include the original file name, the name -## after the rotation, and the time when *f* was opened/closed. -## -## .. bro:see:: rotate_file calc_next_rotate -function rotate_file_by_name%(f: string%): rotate_info - %{ - RecordVal* info = new RecordVal(rotate_info); - - bool is_pkt_dumper = false; - bool is_addl_pkt_dumper = false; - - // Special case: one of current dump files. - if ( pkt_dumper && streq(pkt_dumper->FileName(), f->CheckString()) ) - { - is_pkt_dumper = true; - pkt_dumper->Close(); - } - - if ( addl_pkt_dumper && - streq(addl_pkt_dumper->FileName(), f->CheckString()) ) - { - is_addl_pkt_dumper = true; - addl_pkt_dumper->Close(); - } - - FILE* file = rotate_file(f->CheckString(), info); - if ( ! file ) - { - // Record indicating error. - info->Assign(0, new StringVal("")); - info->Assign(1, new StringVal("")); - info->Assign(2, new Val(0, TYPE_TIME)); - info->Assign(3, new Val(0, TYPE_TIME)); - return info; - } - - fclose(file); - - if ( is_pkt_dumper ) - { - info->Assign(2, new Val(pkt_dumper->OpenTime(), TYPE_TIME)); - pkt_dumper->Open(); - } - - if ( is_addl_pkt_dumper ) - info->Assign(2, new Val(addl_pkt_dumper->OpenTime(), TYPE_TIME)); - - return info; - %} - -## Calculates the duration until the next time a file is to be rotated, based -## on a given rotate interval. -## -## i: The rotate interval to base the calculation on. -## -## Returns: The duration until the next file rotation time. -## -## .. bro:see:: rotate_file rotate_file_by_name -function calc_next_rotate%(i: interval%) : interval - %{ - const char* base_time = log_rotate_base_time ? - log_rotate_base_time->AsString()->CheckString() : 0; - return new Val(calc_next_rotate(i, base_time), TYPE_INTERVAL); - %} - -## Returns the size of a given file. -## -## f: The name of the file whose size to lookup. -## -## Returns: The size of *f* in bytes. -function file_size%(f: string%) : double - %{ - struct stat s; - - if ( stat(f->CheckString(), &s) < 0 ) - return new Val(-1.0, TYPE_DOUBLE); - - return new Val(double(s.st_size), TYPE_DOUBLE); - %} - -## Formats a given time value according to a format string. -## -## fmt: The format string. See ``man strftime`` for the syntax. -## -## d: The time value. -## -## Returns: The time *d* formatted according to *fmt*. -function strftime%(fmt: string, d: time%) : string - %{ - static char buffer[128]; - - time_t t = time_t(d); - - if ( strftime(buffer, 128, fmt->CheckString(), localtime(&t)) == 0 ) - return new StringVal(""); - - return new StringVal(buffer); - %} - -## .. todo:: document -function match_signatures%(c: connection, pattern_type: int, s: string, - bol: bool, eol: bool, - from_orig: bool, clear: bool%) : bool - %{ - if ( ! rule_matcher ) - return new Val(0, TYPE_BOOL); - - c->Match((Rule::PatternType) pattern_type, s->Bytes(), s->Len(), - from_orig, bol, eol, clear); - - return new Val(1, TYPE_BOOL); - %} - -## Returns the hostname of the machine Bro runs on. -## -## Returns: The hostname of the machine Bro runs on. -function gethostname%(%) : string - %{ - char buffer[MAXHOSTNAMELEN]; - if ( gethostname(buffer, MAXHOSTNAMELEN) < 0 ) - strcpy(buffer, ""); - - buffer[MAXHOSTNAMELEN-1] = '\0'; - return new StringVal(buffer); - %} - -%%{ -#include "DNS_Mgr.h" -#include "Trigger.h" - -class LookupHostCallback : public DNS_Mgr::LookupCallback { -public: - LookupHostCallback(Trigger* arg_trigger, const CallExpr* arg_call, - bool arg_lookup_name) - { - Ref(arg_trigger); - trigger = arg_trigger; - call = arg_call; - lookup_name = arg_lookup_name; - } - - ~LookupHostCallback() - { - Unref(trigger); - } - - // Overridden from DNS_Mgr:Lookup:Callback. - virtual void Resolved(const char* name) - { - Val* result = new StringVal(name); - trigger->Cache(call, result); - Unref(result); - trigger->Release(); - } - - virtual void Resolved(TableVal* addrs) - { - // No Ref() for addrs. - trigger->Cache(call, addrs); - trigger->Release(); - } - - virtual void Timeout() - { - if ( lookup_name ) - { - Val* result = new StringVal("<\?\?\?>"); - trigger->Cache(call, result); - Unref(result); - } - - else - { - ListVal* lv = new ListVal(TYPE_ADDR); - lv->Append(new AddrVal("0.0.0.0")); - Val* result = lv->ConvertToSet(); - trigger->Cache(call, result); - Unref(result); - Unref(lv); - } - - trigger->Release(); - } - -private: - Trigger* trigger; - const CallExpr* call; - bool lookup_name; -}; -%%} - -## Issues an asynchronous reverse DNS lookup and delays the function result. -## This function can therefore only be called inside a ``when`` condition, -## e.g., ``when ( local host = lookup_addr(10.0.0.1) ) { f(host); }``. -## -## host: The IP address to lookup. -## -## Returns: The DNS name of *host*. -## -## .. bro:see:: lookup_hostname -function lookup_addr%(host: addr%) : string - %{ - // FIXME: It should be easy to adapt the function to synchronous - // lookups if we're reading a trace. - Trigger* trigger = frame->GetTrigger(); - - if ( ! trigger) - { - builtin_error("lookup_addr() can only be called inside a when-condition"); - return new StringVal(""); - } - - frame->SetDelayed(); - trigger->Hold(); - -#ifdef BROv6 - if ( ! is_v4_addr(host) ) - { - // FIXME: This is a temporary work-around until we get this - // fixed. We warn the user once, and always trigger a timeout. - // Ticket #355 records the problem. - static bool warned = false; - if ( ! warned ) - { - reporter->Warning("lookup_addr() only supports IPv4 addresses currently"); - warned = true; - } - - trigger->Timeout(); - return 0; - } - - dns_mgr->AsyncLookupAddr(to_v4_addr(host), - new LookupHostCallback(trigger, frame->GetCall(), true)); -#else - dns_mgr->AsyncLookupAddr(host, - new LookupHostCallback(trigger, frame->GetCall(), true)); -#endif - return 0; - %} - -## Issues an asynchronous DNS lookup and delays the function result. -## This function can therefore only be called inside a ``when`` condition, -## e.g., ``when ( local h = lookup_hostname("www.bro-ids.org") ) { f(h); }``. -## -## host: The hostname to lookup. -## -## Returns: A set of DNS A records associated with *host*. -## -## .. bro:see:: lookup_addr -function lookup_hostname%(host: string%) : addr_set - %{ - // FIXME: Is should be easy to adapt the function to synchronous - // lookups if we're reading a trace. - Trigger* trigger = frame->GetTrigger(); - - if ( ! trigger) - { - builtin_error("lookup_hostname() can only be called inside a when-condition"); - return new StringVal(""); - } - - frame->SetDelayed(); - trigger->Hold(); - - dns_mgr->AsyncLookupName(host->CheckString(), - new LookupHostCallback(trigger, frame->GetCall(), false)); - return 0; - %} - -## Stops Bro's packet processing. This function is used to synchronize -## distributed trace processing with communication enabled -## (*pseudo-realtime* mode). -## -## .. bro:see: continue_processing suspend_state_updates resume_state_updates -function suspend_processing%(%) : any - %{ - net_suspend_processing(); - return 0; - %} - -## Resumes Bro's packet processing. -## -## .. bro:see: suspend_processing suspend_state_updates resume_state_updates -function continue_processing%(%) : any - %{ - net_continue_processing(); - return 0; - %} - -%%{ -#include "DPM.h" -%%} - -## Schedules an analyzer for a future connection from a given IP address and -## port. The function ignores the scheduling request if the connection did -## not occur within the specified time interval. -## -## orig: The IP address originating a connection in the future. -## -## resp: The IP address responding to a connection from *orig*. -## -## resp_p: The destination port at *resp*. -## -## analyzer: The analyzer ID. -## -## tout: The timeout interval after which to ignore the scheduling request. -## -## Returns: True (unconditionally). -## -## .. bro:see:: disable_analyzer analyzer_name -## -## .. todo:: The return value should be changed to any. -function expect_connection%(orig: addr, resp: addr, resp_p: port, - analyzer: count, tout: interval%) : bool - %{ - dpm->ExpectConnection(orig, resp, resp_p->Port(), resp_p->PortType(), - (AnalyzerTag::Tag) analyzer, tout, 0); - return new Val(1, TYPE_BOOL); - %} - -## Disables the analyzer which raised the current event (if the analyzer -## belongs to the given connection). -## -## cid: The connection identifier. -## -## aid: The analyzer ID. -## -## Returns: True if the connection identified by *cid* exists and has analyzer -## *aid*. -## -## .. bro:see:: expect_connection analyzer_name -function disable_analyzer%(cid: conn_id, aid: count%) : bool - %{ - Connection* c = sessions->FindConnection(cid); - if ( ! c ) - { - reporter->Error("cannot find connection"); - return new Val(0, TYPE_BOOL); - } - - Analyzer* a = c->FindAnalyzer(aid); - if ( ! a ) - { - reporter->Error("connection does not have analyzer specified to disable"); - return new Val(0, TYPE_BOOL); - } - - a->Remove(); - return new Val(1, TYPE_BOOL); - %} - -## Translate an analyzer type to an ASCII string. -## -## aid: The analyzer ID. -## -## Returns: The analyzer *aid* as string. -## -## .. bro:see:: expect_connection disable_analyzer current_analyzer -function analyzer_name%(aid: count%) : string - %{ - return new StringVal(Analyzer::GetTagName((AnalyzerTag::Tag) aid)); - %} - -## Returns the value of a global identifier. -## -## id: The global identifier. -## -## Returns the value of *id*. If *id* does not describe a valid identifier, the -## function returns the string ``""`` or ``""``. -function lookup_ID%(id: string%) : any - %{ - ID* i = global_scope()->Lookup(id->CheckString()); - if ( ! i ) - return new StringVal(""); - - if ( ! i->ID_Val() ) - return new StringVal(""); - - return i->ID_Val()->Ref(); - %} - -## Stops propagating :bro:attr:`&synchronized` accesses. -## -## .. bro:see: suspend_processing continue_processing resume_state_updates -function suspend_state_updates%(%) : any - %{ - if ( remote_serializer ) - remote_serializer->SuspendStateUpdates(); - return 0; - %} - -## Resumes propagating :bro:attr:`&synchronized` accesses. -## -## .. bro:see: suspend_processing continue_processing suspend_state_updates -function resume_state_updates%(%) : any - %{ - if ( remote_serializer ) - remote_serializer->ResumeStateUpdates(); - return 0; - %} - -## Returns the ID of the analyzer which raised the current event. -## -## Returns: The ID of the analyzer which raised hte current event, or 0 if -## none. -function current_analyzer%(%) : count - %{ - return new Val(mgr.CurrentAnalyzer(), TYPE_COUNT); - %} - -## Returns Bro's process ID. -## -## Returns: Bro's process ID. -function getpid%(%) : count - %{ - return new Val(getpid(), TYPE_COUNT); - %} -%%{ -#include -%%} - - -## Send a string to syslog. -## -## s: The string to log via syslog -function syslog%(s: string%): any - %{ - reporter->Syslog("%s", s->CheckString()); - return 0; - %} - -%%{ -#ifdef USE_GEOIP -extern "C" { -#include -} - -static GeoIP* open_geoip_db(GeoIPDBTypes type) - { - GeoIP* geoip = 0; - - if ( GeoIP_db_avail(type) ) - geoip = GeoIP_open_type(type, GEOIP_MEMORY_CACHE); - - if ( ! geoip ) - reporter->Warning("Failed to open GeoIP database: %s", - GeoIPDBFileName[type]); - return geoip; - } - -#endif -%%} - -## Performs a geo-lookup of an IP address. -## Requires Bro to be built with ``libgeoip``. -## -## a: The IP address to lookup. -## -## Returns: A record with country, region, city, latitude, and longitude. -## -## .. bro:see:: lookup_asn -function lookup_location%(a: addr%) : geo_location - %{ - RecordVal* location = new RecordVal(geo_location); - -#ifdef USE_GEOIP - static bool geoip_initialized = false; - static GeoIP* geoip = 0; - static GeoIP* geoip_v6 = 0; - static bool have_city_db = false; - static bool have_cityv6_db = false; - GeoIPRecord* gir = 0; - const char* cc = 0; - - if ( ! geoip_initialized ) - { - geoip_initialized = true; - geoip = open_geoip_db(GEOIP_CITY_EDITION_REV0); - - if ( ! geoip ) - { - geoip = open_geoip_db(GEOIP_COUNTRY_EDITION); - if ( ! geoip ) - builtin_error("Can't initialize GeoIP City/Country database"); - else - reporter->Warning("Fell back to GeoIP Country database"); - } - else - have_city_db = true; - -#ifdef BROv6 - -#ifdef HAVE_GEOIP_CITY_EDITION_REV0_V6 - geoip_v6 = open_geoip_db(GEOIP_CITY_EDITION_REV0_V6); - if ( geoip_v6 ) - have_cityv6_db = true; -#endif - -#ifdef HAVE_GEOIP_COUNTRY_EDITION_V6 - if ( ! geoip_v6 ) - geoip_v6 = open_geoip_db(GEOIP_COUNTRY_EDITION_V6); -#endif - if ( ! geoip_v6 ) - builtin_error("Can't initialize GeoIPv6 City/Country database"); -#endif - } - -#ifdef BROv6 - -#ifdef HAVE_GEOIP_COUNTRY_EDITION_V6 - if ( geoip_v6 && ! is_v4_addr(a) ) - { - geoipv6_t ga; - memcpy(&ga, a, 16); - if ( have_cityv6_db ) - gir = GeoIP_record_by_ipnum_v6(geoip_v6, ga); - else - cc = GeoIP_country_code_by_ipnum_v6(geoip_v6, ga); - } - else -#endif - - if ( geoip && is_v4_addr(a) ) - { - uint32 addr = to_v4_addr(a); - if ( have_city_db ) - gir = GeoIP_record_by_ipnum(geoip, ntohl(addr)); - else - cc = GeoIP_country_code_by_ipnum(geoip, ntohl(addr)); - } - -#else // not BROv6 - if ( geoip ) - { - if ( have_city_db ) - gir = GeoIP_record_by_ipnum(geoip, ntohl(a)); - else - cc = GeoIP_country_code_by_ipnum(geoip, ntohl(a)); - } -#endif - - if ( gir ) - { - if ( gir->country_code ) - location->Assign(0, new StringVal(gir->country_code)); - - if ( gir->region ) - location->Assign(1, new StringVal(gir->region)); - - if ( gir->city ) - location->Assign(2, new StringVal(gir->city)); - - if ( gir->latitude ) - location->Assign(3, new Val(gir->latitude, - TYPE_DOUBLE)); - - if ( gir->longitude ) - location->Assign(4, new Val(gir->longitude, - TYPE_DOUBLE)); - - GeoIPRecord_delete(gir); - - return location; - } - - else if ( cc ) - { - location->Assign(0, new StringVal(cc)); - return location; - } - -#else // not USE_GEOIP - static int missing_geoip_reported = 0; - - if ( ! missing_geoip_reported ) - { - builtin_error("Bro was not configured for GeoIP support"); - missing_geoip_reported = 1; - } -#endif - - // We can get here even if we have GeoIP support if we weren't - // able to initialize it or it didn't return any information for - // the address. - - return location; - %} - -## Performs an AS lookup of an IP address. -## -## a: The IP address to lookup. -## -## Returns: The number of the AS that contains *a*. -## -## .. bro:see:: lookup_location -function lookup_asn%(a: addr%) : count - %{ -#ifdef USE_GEOIP - static GeoIP* geoip_asn = 0; - static bool geoip_asn_initialized = false; - char* gir = 0; - - if ( ! geoip_asn_initialized ) - { - geoip_asn_initialized = true; - geoip_asn = open_geoip_db(GEOIP_ASNUM_EDITION); - if ( ! geoip_asn ) - builtin_error("Can't initialize GeoIP ASNUM database"); - } - - if ( geoip_asn ) - { -#ifdef BROv6 - -// IPv6 support showed up in 1.4.5. -#ifdef HAVE_GEOIP_COUNTRY_EDITION_V6 - if ( ! is_v4_addr(a) ) - { - geoipv6_t ga; - memcpy(&ga, a, 16); - gir = GeoIP_name_by_ipnum_v6(geoip_asn, ga); - } - else -#endif - - if ( is_v4_addr(a) ) - { - uint32 addr = to_v4_addr(a); - gir = GeoIP_name_by_ipnum(geoip_asn, ntohl(addr)); - } - -#else // not BROv6 - gir = GeoIP_name_by_ipnum(geoip_asn, ntohl(a)); -#endif - } - - if ( gir ) - { - // Move the pointer +2 so we don't return - // the first two characters: "AS". - return new Val(atoi(gir+2), TYPE_COUNT); - } - -#else // not USE_GEOIP - static int missing_geoip_reported = 0; - - if ( ! missing_geoip_reported ) - { - builtin_error("Bro was not configured for GeoIP ASN support"); - missing_geoip_reported = 1; - } -#endif - - // We can get here even if we have GeoIP support, if we weren't - // able to initialize it or it didn't return any information for - // the address. - return new Val(0, TYPE_COUNT); - %} - -## Determines whether *c* has been received externally. For example, -## Broccoli or the Time Machine can send packets to Bro via a mechanism that -## one step lower than sending events. This function checks whether the packets -## of a connection stem from one of these external *packet sources*. -## -## c: The connection to test. -## -## Returns: True if *c* has been received externally. -function is_external_connection%(c: connection%) : bool - %{ - return new Val(c && c->IsExternal(), TYPE_BOOL); - %} - -## Disables sending :bro:id:`print_hook` events to remote peers for a given -## file. This function is equivalent to :bro:attr:`&disable_print_hook`. In a -## distributed setup, communicating Bro instances generate the event -## :bro:id:`print_hook` for each print statement and send it to the remote -## side. When disabled for a particular file, these events will not be -## propagated to other peers. -## -## f: The file to disable :bro:id:`print_hook` events for. -## -## .. bro:see:: enable_raw_output -function disable_print_hook%(f: file%): any - %{ - f->DisablePrintHook(); - return 0; - %} - -## Prevents escaping of non-ASCII character when writing to a file. -## This function is equivalent to :bro:attr:`&disable_print_hook`. -## -## f: The file to disable raw output for. -## -## .. bro:see:: disable_print_hook -function enable_raw_output%(f: file%): any - %{ - f->EnableRawOutput(); - return 0; - %} - -%%{ -extern "C" { -#include -} -%%} - -## Determines the MIME type of a piece of data using ``libmagic``. -## -## data: The data to find the MIME type for. -## -## return_mime: If true, the function returns a short MIME type string (e.g., -## ``text/plain`` instead of a more elaborate textual description. -## -## Returns: The MIME type of *data*. -function identify_data%(data: string, return_mime: bool%): string - %{ - const char* descr = ""; - - static magic_t magic_mime = 0; - static magic_t magic_descr = 0; - - magic_t* magic = return_mime ? &magic_mime : &magic_descr; - - if( ! *magic ) - { - *magic = magic_open(return_mime ? MAGIC_MIME : MAGIC_NONE); - - if ( ! *magic ) - { - reporter->Error("can't init libmagic: %s", magic_error(*magic)); - return new StringVal(""); - } - - if ( magic_load(*magic, 0) < 0 ) - { - reporter->Error("can't load magic file: %s", magic_error(*magic)); - magic_close(*magic); - *magic = 0; - return new StringVal(""); - } - } - - descr = magic_buffer(*magic, data->Bytes(), data->Len()); - - return new StringVal(descr); - %} - -## Enables all event handlers in a given group. One can tag event handlers with -## the :bro:attr:`&group` attribute to logically group them together, e.g, -## `` event foo() &group="bar"``. This function enables all event handlers that -## belong to such a group. -## -## group: The group. -## -## .. bro:see:: disable_event_group -function enable_event_group%(group: string%) : any - %{ - event_registry->EnableGroup(group->CheckString(), true); - return 0; - %} - -## Disables all event handlers in a given group. -## -## group: The group. -## -## .. bro:see:: enable_event_group -function disable_event_group%(group: string%) : any - %{ - event_registry->EnableGroup(group->CheckString(), false); - return 0; - %} - - -%%{ -#include -static map entropy_states; -%%} - -## Performs an `entropy `_ test on the given -## data. -## -## data: The data to compute the entropy for. -## -## Returns: The result of the entropy test, which contains the following -## fields. -## -## - ``entropy``: The information density expressed as a number of -## bits per character. -## -## - ``chi_square``: The chi-square test value expressed as an -## absolute number and a percentage which indicates how -## frequently a truly random sequence would exceed the value -## calculated, i.e., the degree to which the sequence tested is -## suspected of being non-random. -## -## If the percentage is greater than 99% or less than 1%, the -## sequence is almost certainly not random. If the percentage is -## between 99% and 95% or between 1% and 5%, the sequence is -## suspect. Percentages between 90\% and 95\% and 5\% and 10\% -## indicate the sequence is "almost suspect." -## -## - ``mean``: The arithmetic mean of all the bytes. If the data -## are close to random, it should be around 127.5. -## -## - ``monte_carlo_pi``: Each successive sequence of six bytes is -## used as 24-bit *x* and *y* coordinates within a square. If -## the distance of the randomly-generated point is less than the -## radius of a circle inscribed within the square, the six-byte -## sequence is considered a "hit." The percentage of hits can -## be used to calculate the value of pi. For very large streams -## the value will approach the correct value of pi if the -## sequence is close to random. -## -## - ``serial_correlation``: This quantity measures the extent to -## which each byte in the file depends upon the previous byte. -## For random sequences this value will be close to zero. -## -## .. bro:see:: entropy_test_init entropy_test_add entropy_test_finish -function find_entropy%(data: string%): entropy_test_result - %{ - double montepi, scc, ent, mean, chisq; - montepi = scc = ent = mean = chisq = 0.0; - RecordVal* ent_result = new RecordVal(entropy_test_result); - RandTest *rt = new RandTest(); - - rt->add((char*) data->Bytes(), data->Len()); - rt->end(&ent, &chisq, &mean, &montepi, &scc); - delete rt; - - ent_result->Assign(0, new Val(ent, TYPE_DOUBLE)); - ent_result->Assign(1, new Val(chisq, TYPE_DOUBLE)); - ent_result->Assign(2, new Val(mean, TYPE_DOUBLE)); - ent_result->Assign(3, new Val(montepi, TYPE_DOUBLE)); - ent_result->Assign(4, new Val(scc, TYPE_DOUBLE)); - return ent_result; - %} - -## Initializes data structures for incremental entropy calculation. -## -## index: An arbitrary unique value per distinct computation. -## -## Returns: True on success. -## -## .. bro:see:: find_entropy entropy_test_add entropy_test_finish -function entropy_test_init%(index: any%): bool - %{ - BroString* s = convert_index_to_string(index); - int status = 0; - - if ( entropy_states.count(*s) < 1 ) - { - entropy_states[*s] = new RandTest(); - status = 1; - } - - delete s; - return new Val(status, TYPE_BOOL); - %} - -## Adds data to an incremental entropy calculation. Before using this function, -## one needs to invoke :bro:id:`entropy_test_init`. -## -## data: The data to add to the entropy calculation. -## -## index: An arbitrary unique value that identifies a particular entropy -## computation. -## -## Returns: True on success. -## -## .. bro:see:: find_entropy entropy_test_add entropy_test_finish -function entropy_test_add%(index: any, data: string%): bool - %{ - BroString* s = convert_index_to_string(index); - int status = 0; - - if ( entropy_states.count(*s) > 0 ) - { - entropy_states[*s]->add((char*) data->Bytes(), data->Len()); - status = 1; - } - - delete s; - return new Val(status, TYPE_BOOL); - %} - -## Finishes an incremental entropy calculation. Before using this function, -## one needs to initialize the computation with :bro:id:`entropy_test_init` and -## add data to it via :bro:id:`entropy_test_add`. -## -## index: An arbitrary unique value that identifies a particular entropy -## computation. -## -## Returns: The result of the entropy test. See :bro:id:`find_entropy` for a -## description of the individual components. -## -## .. bro:see:: find_entropy entropy_test_init entropy_test_add -function entropy_test_finish%(index: any%): entropy_test_result - %{ - BroString* s = convert_index_to_string(index); - double montepi, scc, ent, mean, chisq; - montepi = scc = ent = mean = chisq = 0.0; - RecordVal* ent_result = new RecordVal(entropy_test_result); - - if ( entropy_states.count(*s) > 0 ) - { - RandTest *rt = entropy_states[*s]; - rt->end(&ent, &chisq, &mean, &montepi, &scc); - entropy_states.erase(*s); - delete rt; - } - - ent_result->Assign(0, new Val(ent, TYPE_DOUBLE)); - ent_result->Assign(1, new Val(chisq, TYPE_DOUBLE)); - ent_result->Assign(2, new Val(mean, TYPE_DOUBLE)); - ent_result->Assign(3, new Val(montepi, TYPE_DOUBLE)); - ent_result->Assign(4, new Val(scc, TYPE_DOUBLE)); - - delete s; - return ent_result; - %} - -## Depreacated. Will be removed. +## Deprecated. Will be removed. function bro_has_ipv6%(%) : bool %{ #ifdef BROv6 @@ -5066,319 +5442,3 @@ function bro_has_ipv6%(%) : bool return new Val(0, TYPE_BOOL); #endif %} - -## Creates an identifier that is unique with high probability. -## -## prefix: A custom string prepended to the result. -## -## .. bro:see:: unique_id_from -function unique_id%(prefix: string%) : string - %{ - char tmp[20]; - uint64 uid = calculate_unique_id(UID_POOL_DEFAULT_SCRIPT); - return new StringVal(uitoa_n(uid, tmp, sizeof(tmp), 62, prefix->CheckString())); - %} - -## Creates an identifier that is unique with high probability. -## -## pool: A seed for determinism. -## -## prefix: A custom string prepended to the result. -## -## .. bro:see:: unique_id -function unique_id_from%(pool: int, prefix: string%) : string - %{ - pool += UID_POOL_CUSTOM_SCRIPT; // Make sure we don't conflict with internal pool. - - char tmp[20]; - uint64 uid = calculate_unique_id(pool); - return new StringVal(uitoa_n(uid, tmp, sizeof(tmp), 62, prefix->CheckString())); - %} -%%{ -#include -#include -#include - -// This is the indexed map of X509 certificate stores. -static map x509_stores; - -// ### NOTE: while d2i_X509 does not take a const u_char** pointer, -// here we assume d2i_X509 does not write to , so it is safe to -// convert data to a non-const pointer. Could some X509 guru verify -// this? - -X509* d2i_X509_(X509** px, const u_char** in, int len) - { -#ifdef OPENSSL_D2I_X509_USES_CONST_CHAR - return d2i_X509(px, in, len); -#else - return d2i_X509(px, (u_char**)in, len); -#endif - } - -%%} - - -## Verifies a X.509 certificate. -## -## der_cert: The X.509 certificate in DER format. -## -## cert_stack: Specifies a certificate chain to validate against, with index 0 -## typically being the root CA. Bro uses the Mozilla root CA list -## by default. -## -## root_certs: A list of additional root certificates that extends -## *cert_stack*. -## -## Returns: A status code of the verification which can be converted into an -## ASCII string via :bro:id:`x509_err2str`. -## -## .. bro:see:: x509_err2str -function x509_verify%(der_cert: string, cert_stack: string_vec, root_certs: table_string_of_string%): count - %{ - X509_STORE* ctx = 0; - int i = 0; - - // If this certificate store was built previously, just reuse the old one. - if ( x509_stores.count(root_certs) > 0 ) - ctx = x509_stores[root_certs]; - - if ( ! ctx ) // lookup to see if we have this one built already! - { - ctx = X509_STORE_new(); - TableVal* root_certs2 = root_certs->AsTableVal(); - ListVal* idxs = root_certs2->ConvertToPureList(); - - // Build the validation store - for ( i = 0; i < idxs->Length(); ++i ) - { - Val* key = idxs->Index(i); - StringVal *sv = root_certs2->Lookup(key)->AsStringVal(); - const uint8* data = sv->Bytes(); - X509* x = d2i_X509_(NULL, &data, sv->Len()); - if ( ! x ) - { - builtin_error(fmt("Root CA error: %s", ERR_error_string(ERR_peek_last_error(),NULL))); - return new Val((uint64) ERR_get_error(), TYPE_COUNT); - } - X509_STORE_add_cert(ctx, x); - } - delete idxs; - - // Save the newly constructed certificate store into the cacheing map. - x509_stores[root_certs] = ctx; - } - - const uint8 *cert_data = der_cert->Bytes(); - X509* cert = d2i_X509_(NULL, &cert_data, der_cert->Len()); - if ( ! cert ) - { - builtin_error(fmt("Certificate error: %s", ERR_error_string(ERR_peek_last_error(),NULL))); - return new Val((uint64) ERR_get_error(), TYPE_COUNT); - } - - STACK_OF(X509)* untrusted_certs = sk_X509_new_null(); - if ( ! untrusted_certs ) - { - builtin_error(fmt("Untrusted certificate stack initialization error: %s", ERR_error_string(ERR_peek_last_error(),NULL))); - return new Val((uint64) ERR_get_error(), TYPE_COUNT); - } - - VectorVal *cert_stack_vec = cert_stack->AsVectorVal(); - for ( i = 0; i < (int) cert_stack_vec->Size(); ++i ) - { - StringVal *sv = cert_stack_vec->Lookup(i)->AsStringVal(); - const uint8 *data = sv->Bytes(); - X509* x = d2i_X509_(NULL, &data, sv->Len()); - if ( ! x ) - { - X509_free(cert); - sk_X509_pop_free(untrusted_certs, X509_free); - builtin_error(fmt("Untrusted certificate stack creation error: %s", ERR_error_string(ERR_peek_last_error(),NULL))); - return new Val((uint64) ERR_get_error(), TYPE_COUNT); - } - sk_X509_push(untrusted_certs, x); - } - - X509_STORE_CTX csc; - X509_STORE_CTX_init(&csc, ctx, cert, untrusted_certs); - X509_STORE_CTX_set_time(&csc, 0, (time_t) network_time); - - int result = X509_verify_cert(&csc); - X509_STORE_CTX_cleanup(&csc); - - if ( untrusted_certs ) - sk_X509_pop_free(untrusted_certs, X509_free); - X509_free(cert); - - return new Val((uint64) csc.error, TYPE_COUNT); - %} - -## Converts a X.509 certificate verification error code into an ASCII string. -## -## err_num: The error code. -## -## Returns: A string representation of *err_num*. -## -## .. bro:see:: x509_verify -function x509_err2str%(err_num: count%): string - %{ - return new StringVal(X509_verify_cert_error_string(err_num)); - %} - -## Converts UNIX file permissions given by a mode to an ASCII string. -## -## mode: The permisssions, e.g., 644 or 755. -## -## Returns: A string representation of *mode* in the format -## ``rw[xsS]rw[xsS]rw[xtT]``. -function NFS3::mode2string%(mode: count%): string - %{ - char str[12]; - char *p = str; - - /* usr */ - if (mode & S_IRUSR) - *p++ = 'r'; - else - *p++ = '-'; - - if (mode & S_IWUSR) - *p++ = 'w'; - else - *p++ = '-'; - - switch (mode & (S_IXUSR | S_ISUID)) { - case 0: - *p++ = '-'; - break; - case S_IXUSR: - *p++ = 'x'; - break; - case S_ISUID: - *p++ = 'S'; - break; - case S_IXUSR | S_ISUID: - *p++ = 's'; - break; - } - - /* group */ - if (mode & S_IRGRP) - *p++ = 'r'; - else - *p++ = '-'; - if (mode & S_IWGRP) - *p++ = 'w'; - else - *p++ = '-'; - - switch (mode & (S_IXGRP | S_ISGID)) { - case 0: - *p++ = '-'; - break; - case S_IXGRP: - *p++ = 'x'; - break; - case S_ISGID: - *p++ = 'S'; - break; - case S_IXGRP | S_ISGID: - *p++ = 's'; - break; - } - - /* other */ - if (mode & S_IROTH) - *p++ = 'r'; - else - *p++ = '-'; - if (mode & S_IWOTH) - *p++ = 'w'; - else - *p++ = '-'; - - switch (mode & (S_IXOTH | S_ISVTX)) { - case 0: - *p++ = '-'; - break; - case S_IXOTH: - *p++ = 'x'; - break; - case S_ISVTX: - *p++ = 'T'; - break; - case S_IXOTH | S_ISVTX: - *p++ = 't'; - break; - } - - *p = '\0'; - - return new StringVal(str); - %} - -## Opens a program with ``popen`` and writes a given string to the returned -## stream to send it to the opened process's stdin. -## -## program: The program to execute. -## -## to_write: Data to pipe to the opened program's process via ``stdin``. -## -## Returns: True on success. -## -## .. bro:see:: system system_env -function piped_exec%(program: string, to_write: string%): bool - %{ - const char* prog = program->CheckString(); - - FILE* f = popen(prog, "w"); - if ( ! f ) - { - reporter->Error("Failed to popen %s", prog); - return new Val(0, TYPE_BOOL); - } - - const u_char* input_data = to_write->Bytes(); - int input_data_len = to_write->Len(); - - int bytes_written = fwrite(input_data, 1, input_data_len, f); - - pclose(f); - - if ( bytes_written != input_data_len ) - { - reporter->Error("Failed to write all given data to %s", prog); - return new Val(0, TYPE_BOOL); - } - - return new Val(1, TYPE_BOOL); - %} - -## Enables the communication system. By default, the communication is off until -## explicitly enabled, and all other calls to communication-related functions -## will be ignored until done so. -function enable_communication%(%): any - %{ - if ( bro_start_network_time != 0.0 ) - { - builtin_error("communication must be enabled in bro_init"); - return 0; - } - - if ( using_communication ) - // Ignore duplicate calls. - return 0; - - using_communication = 1; - remote_serializer->Init(); - return 0; - %} - -## Returns the Bro version string. -## -## Returns: Bro's version, e.g., 2.0-beta-47-debug. -function bro_version%(%): string - %{ - return new StringVal(bro_version()); - %} From 50d5571939b74b4a9c91a00339bfdba390d564cc Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Sun, 11 Dec 2011 18:49:00 -0800 Subject: [PATCH 172/964] Give mode2string a more generic name. --- src/bro.bif | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bro.bif b/src/bro.bif index 16b18b0d48..d0569716b0 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -3542,7 +3542,7 @@ function x509_err2str%(err_num: count%): string return new StringVal(X509_verify_cert_error_string(err_num)); %} -function NFS3::mode2string%(mode: count%): string +function file_mode%(mode: count%): string %{ char str[12]; char *p = str; From 72a7814657159bf7d73b77b087b42fdfb9c396e2 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Sun, 11 Dec 2011 19:10:21 -0800 Subject: [PATCH 173/964] Document currently dysfunctional anonymization BiFs. --- src/bro.bif | 166 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 98 insertions(+), 68 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index 9ae9fb94ca..237884bcf6 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -5270,6 +5270,104 @@ function match_signatures%(c: connection, pattern_type: int, s: string, return new Val(1, TYPE_BOOL); %} +%%{ +#include "Anon.h" +%%} + +## Preserves the prefix of an IP address in anonymization. +## +## a: The address to preserve. +## +## width: The number of bits from the top that should remain intact. +## +## .. bro:see:: preserve_subnet anonymize_addr +## +## .. todo:: Currently dysfunctional. +function preserve_prefix%(a: addr, width: count%): any + %{ + AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50]; + if ( ip_anon ) + { +#ifdef BROv6 + if ( ! is_v4_addr(a) ) + builtin_error("preserve_prefix() not supported for IPv6 addresses"); + else + ip_anon->PreservePrefix(a[3], width); +#else + ip_anon->PreservePrefix(a, width); +#endif + } + + + return 0; + %} + +## Preserves the prefix of a subnet in anonymization. +## +## a: The subnet to preserve. +## +## width: The number of bits from the top that should remain intact. +## +## .. bro:see:: preserve_prefix anonymize_addr +## +## .. todo:: Currently dysfunctional. +function preserve_subnet%(a: subnet%): any + %{ + DEBUG_MSG("%s/%d\n", dotted_addr(a->AsAddr()), a->Width()); + AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50]; + if ( ip_anon ) + { +#ifdef BROv6 + if ( ! is_v4_addr(a->AsAddr()) ) + builtin_error("preserve_subnet() not supported for IPv6 addresses"); + else + ip_anon->PreservePrefix(a->AsAddr()[3], a->Width()); +#else + ip_anon->PreservePrefix(a->AsAddr(), a->Width()); +#endif + } + + return 0; + %} + +## Anonymizes an IP address. +## +## a: The address to anonymize. +## +## cl: The anonymization class, which can take on three different values: +## +## - ``ORIG_ADDR``: Tag *a* as an originator address. +## +## - ``RESP_ADDR``: Tag *a* as an responder address. +## +## - ``OTHER_ADDR``: Tag *a* as an arbitrary address. +## +## Returns: An anonymized version of *a*. +## +## .. bro:see:: preserve_prefix preserve_subnet +## +## .. todo:: Currently dysfunctional. +function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr + %{ + int anon_class = cl->InternalInt(); + if ( anon_class < 0 || anon_class >= NUM_ADDR_ANONYMIZATION_CLASSES ) + builtin_error("anonymize_addr(): invalid ip addr anonymization class"); + +#ifdef BROv6 + if ( ! is_v4_addr(a) ) + { + builtin_error("anonymize_addr() not supported for IPv6 addresses"); + return 0; + } + else + return new AddrVal(anonymize_ip(a[3], + (enum ip_addr_anonymization_class_t) anon_class)); +#else + return new AddrVal(anonymize_ip(a, + (enum ip_addr_anonymization_class_t) anon_class)); +#endif + %} + ## Deprecated. Will be removed. function active_connection%(id: conn_id%): bool %{ @@ -5292,74 +5390,6 @@ function connection_record%(cid: conn_id%): connection } %} -%%{ -#include "Anon.h" -%%} - -## Preserve prefix as original one in anonymization. -## .. todo: Currently broken. -function preserve_prefix%(a: addr, width: count%): any - %{ - AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50]; - if ( ip_anon ) - { -#ifdef BROv6 - if ( ! is_v4_addr(a) ) - builtin_error("preserve_prefix() not supported for IPv6 addresses"); - else - ip_anon->PreservePrefix(a[3], width); -#else - ip_anon->PreservePrefix(a, width); -#endif - } - - - return 0; - %} - -## .. todo: Currently broken. -function preserve_subnet%(a: subnet%): any - %{ - DEBUG_MSG("%s/%d\n", dotted_addr(a->AsAddr()), a->Width()); - AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50]; - if ( ip_anon ) - { -#ifdef BROv6 - if ( ! is_v4_addr(a->AsAddr()) ) - builtin_error("preserve_subnet() not supported for IPv6 addresses"); - else - ip_anon->PreservePrefix(a->AsAddr()[3], a->Width()); -#else - ip_anon->PreservePrefix(a->AsAddr(), a->Width()); -#endif - } - - return 0; - %} - -## Anonymize an IP address. -## .. todo: Currently broken. -function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr - %{ - int anon_class = cl->InternalInt(); - if ( anon_class < 0 || anon_class >= NUM_ADDR_ANONYMIZATION_CLASSES ) - builtin_error("anonymize_addr(): invalid ip addr anonymization class"); - -#ifdef BROv6 - if ( ! is_v4_addr(a) ) - { - builtin_error("anonymize_addr() not supported for IPv6 addresses"); - return 0; - } - else - return new AddrVal(anonymize_ip(a[3], - (enum ip_addr_anonymization_class_t) anon_class)); -#else - return new AddrVal(anonymize_ip(a, - (enum ip_addr_anonymization_class_t) anon_class)); -#endif - %} - ## Deprecated. Will be removed. function dump_config%(%) : bool %{ From ff7a1ed9d5a8c300cd44675545e5726f4fdde646 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 12 Dec 2011 11:07:18 -0600 Subject: [PATCH 174/964] Fix some sphinx warnings. --- src/bro.bif | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index 237884bcf6..fa3936546f 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -400,7 +400,7 @@ function exit%(%): int ## Returns: True after successful termination and false when Bro is still in ## the process of shutting down. ## -## .. bro:see:: is_terminating +## .. bro:see:: bro_is_terminating function terminate%(%): bool %{ if ( terminating ) @@ -806,8 +806,8 @@ function identify_data%(data: string, return_mime: bool%): string static map entropy_states; %%} -## Performs an `entropy `_ test on the given -## data. +## Performs an entropy test on the given data. +## See http://www.fourmilab.ch/random. ## ## data: The data to compute the entropy for. ## @@ -3170,7 +3170,7 @@ function dump_current_packet%(file_name: string%) : bool ## Returns the currently processed PCAP packet. ## ## Returns: The currently processed packet, which is as a record -## containing the timestamp, ``snaplen,'' and packet data. +## containing the timestamp, ``snaplen``, and packet data. ## ## .. bro:see:: dump_current_packet dump_packet send_current_packet function get_current_packet%(%) : pcap_packet @@ -4193,7 +4193,7 @@ function skip_smtp_data%(c: connection%): any ## Enables all event handlers in a given group. One can tag event handlers with ## the :bro:attr:`&group` attribute to logically group them together, e.g, -## `` event foo() &group="bar"``. This function enables all event handlers that +## ``event foo() &group="bar"``. This function enables all event handlers that ## belong to such a group. ## ## group: The group. @@ -4918,7 +4918,7 @@ function capture_state_updates%(filename: string%) : bool ## request_remote_sync ## request_remote_logs ## request_remote_events -## set_compression_accept_state +## set_accept_state ## set_compression_level ## send_state ## send_id From 61aa592db5a847d0e9a2538d31850aae6910c04d Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 12 Dec 2011 14:26:54 -0500 Subject: [PATCH 175/964] A few updates for SQL injection detection. - The biggest change is the change in notice names from HTTP::SQL_Injection_Attack_Against to HTTP::SQL_Injection_Victim - A few new SQL injection attacks in the tests that we need to support at some point. --- scripts/policy/protocols/http/detect-sqli.bro | 20 +++++++++---------- .../http/test-sql-injection-regex.bro | 10 ++-------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/scripts/policy/protocols/http/detect-sqli.bro b/scripts/policy/protocols/http/detect-sqli.bro index c4ba7ee74e..837baee117 100644 --- a/scripts/policy/protocols/http/detect-sqli.bro +++ b/scripts/policy/protocols/http/detect-sqli.bro @@ -12,12 +12,12 @@ export { SQL_Injection_Attacker, ## Indicates that a host was seen to have SQL injection attacks against ## it. This is tracked by IP address as opposed to hostname. - SQL_Injection_Attack_Against, + SQL_Injection_Victim, }; redef enum Metrics::ID += { - SQL_ATTACKER, - SQL_ATTACKS_AGAINST, + SQLI_ATTACKER, + SQLI_VICTIM, }; redef enum Tags += { @@ -56,14 +56,14 @@ event bro_init() &priority=3 # determine when it looks like an actual attack and how to respond when # thresholds are crossed. - Metrics::add_filter(SQL_ATTACKER, [$log=F, + Metrics::add_filter(SQLI_ATTACKER, [$log=F, $notice_threshold=sqli_requests_threshold, $break_interval=sqli_requests_interval, $note=SQL_Injection_Attacker]); - Metrics::add_filter(SQL_ATTACKS_AGAINST, [$log=F, - $notice_threshold=sqli_requests_threshold, - $break_interval=sqli_requests_interval, - $note=SQL_Injection_Attack_Against]); + Metrics::add_filter(SQLI_VICTIM, [$log=F, + $notice_threshold=sqli_requests_threshold, + $break_interval=sqli_requests_interval, + $note=SQL_Injection_Victim]); } event http_request(c: connection, method: string, original_URI: string, @@ -73,7 +73,7 @@ event http_request(c: connection, method: string, original_URI: string, { add c$http$tags[URI_SQLI]; - Metrics::add_data(SQL_ATTACKER, [$host=c$id$orig_h], 1); - Metrics::add_data(SQL_ATTACKS_AGAINST, [$host=c$id$resp_h], 1); + Metrics::add_data(SQLI_ATTACKER, [$host=c$id$orig_h], 1); + Metrics::add_data(SQLI_VICTIM, [$host=c$id$resp_h], 1); } } diff --git a/testing/btest/scripts/policy/protocols/http/test-sql-injection-regex.bro b/testing/btest/scripts/policy/protocols/http/test-sql-injection-regex.bro index bf8be22210..2e82eb9dfb 100644 --- a/testing/btest/scripts/policy/protocols/http/test-sql-injection-regex.bro +++ b/testing/btest/scripts/policy/protocols/http/test-sql-injection-regex.bro @@ -42,6 +42,8 @@ event bro_init () #add positive_matches["/index.asp?ARF_ID=(1/(1-(asc(mid(now(),18,1))\(2^7) mod 2)))"]; #add positive_matches["/index.php' and 1=convert(int,(select top 1 table_name from information_schema.tables))--sp_password"]; #add positive_matches["/index.php?id=873 and user=0--"]; + #add positive_matches["?id=1;+if+(1=1)+waitfor+delay+'00:00:01'--9"]; + #add positive_matches["?id=1+and+if(1=1,BENCHMARK(728000,MD5(0x41)),0)9"]; # The positive_matches below are from the mod_security evasion challenge. # All supported attacks are uncommented. @@ -95,14 +97,6 @@ event bro_init () #add negative_matches["/index/hmm.gif?utmdt=Record > Create a Graph"]; #add negative_matches["/index.php?test='||\x0aTO_CHAR(foo_bar.Foo_Bar_ID)||"]; - local regex = - /[\?&][^[:blank:]\x00-\x37\|]+?=[\-[:alnum:]%]+([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]?([[:blank:]\x00-\x37]|\/\*.*?\*\/|\)?;)+.*?([hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[iI][nN][sS][eE][rR][tT])([[:blank:]\x00-\x37]|\/\*.*?\*\/)+/ - | /[\?&][^[:blank:]\x00-\x37\|]+?=[\-0-9%]+([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]?([[:blank:]\x00-\x37]|\/\*.*?\*\/|\)?;)+([xX]?[oO][rR]|[nN]?[aA][nN][dD])([[:blank:]\x00-\x37]|\/\*.*?\*\/)+['"]?(([^a-zA-Z&]+)?=|[eE][xX][iI][sS][tT][sS])/ - | /[\?&][^[:blank:]\x00-\x37]+?=[\-0-9%]*([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]([[:blank:]\x00-\x37]|\/\*.*?\*\/)*(-|=|\+|\|\|)([[:blank:]\x00-\x37]|\/\*.*?\*\/)*([0-9]|\(?[cC][oO][nN][vV][eE][rR][tT]|[cC][aA][sS][tT])/ - | /[\?&][^[:blank:]\x00-\x37\|]+?=([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]([[:blank:]\x00-\x37]|\/\*.*?\*\/|;)*([xX]?[oO][rR]|[nN]?[aA][nN][dD]|[hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[rR][eE][gG][eE][xX][pP]|[iI][nN][sS][eE][rR][tT])([[:blank:]\x00-\x37]|\/\*.*?\*\/|[\[(])+[a-zA-Z&]{2,}/ - | /[\?&][^[:blank:]\x00-\x37]+?=[^\.]*?([cC][hH][aA][rR]|[aA][sS][cC][iI][iI]|[sS][uU][bB][sS][tT][rR][iI][nN][gG]|[tT][rR][uU][nN][cC][aA][tT][eE]|[vV][eE][rR][sS][iI][oO][nN]|[lL][eE][nN][gG][tT][hH])\(/ - | /\/\*![[:digit:]]{5}.*?\*\//; - print "If anything besides this line prints out, there is a problem."; for ( test in positive_matches ) { From b04b5fea16b5e0208291811d33831c6807572ce0 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Mon, 12 Dec 2011 13:12:24 -0800 Subject: [PATCH 176/964] Mark match_signatures as internal. --- src/bro.bif | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index fa3936546f..1db6bf3c82 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -5252,11 +5252,12 @@ function resume_state_updates%(%) : any # =========================================================================== # -# Deprecated Functions +# Internal Functions # # =========================================================================== -## .. todo:: document and move to the right spot +## Manually triggers the signature engine for a given connection. +## This is an internal function. function match_signatures%(c: connection, pattern_type: int, s: string, bol: bool, eol: bool, from_orig: bool, clear: bool%) : bool @@ -5270,6 +5271,12 @@ function match_signatures%(c: connection, pattern_type: int, s: string, return new Val(1, TYPE_BOOL); %} +# =========================================================================== +# +# Deprecated Functions +# +# =========================================================================== + %%{ #include "Anon.h" %%} From 6ba62b200dddbccf65ba10e49016fe43486b783c Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Mon, 12 Dec 2011 13:12:52 -0800 Subject: [PATCH 177/964] Remove X.509 from first-sentence documention. It turns out that Doxygen uses the first dot (in X.509) as marker for the one-sentence summary. --- src/bro.bif | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index 1db6bf3c82..09712229c1 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -3624,7 +3624,7 @@ X509* d2i_X509_(X509** px, const u_char** in, int len) %%} -## Verifies a X.509 certificate. +## Verifies a certificate. ## ## der_cert: The X.509 certificate in DER format. ## @@ -3719,7 +3719,7 @@ function x509_verify%(der_cert: string, cert_stack: string_vec, root_certs: tabl return new Val((uint64) csc.error, TYPE_COUNT); %} -## Converts a X.509 certificate verification error code into an ASCII string. +## Converts a certificate verification error code into an ASCII string. ## ## err_num: The error code. ## From 362b8105fdaeb7fad4fa81c0fe19de95e530f66a Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Mon, 12 Dec 2011 13:18:55 -0800 Subject: [PATCH 178/964] More directive fixes. --- src/bro.bif | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index 09712229c1..1afd98fddb 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1070,7 +1070,7 @@ function resize%(aggr: any, newsize: count%) : count ## ## Returns: True if any element in *v* is true. ## -## :bro:see:: all_set +## .. bro:see:: all_set function any_set%(v: any%) : bool %{ if ( v->Type()->Tag() != TYPE_VECTOR || @@ -1095,7 +1095,7 @@ function any_set%(v: any%) : bool ## ## Returns: True iff all elements in *v* are true. ## -## :bro:see:: any_set +## .. bro:see:: any_set ## ## .. note:: ## From ae57cbe5fc975463a028df3547fb403e37d4b2a8 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 13 Dec 2011 09:52:26 -0600 Subject: [PATCH 179/964] Better persistent state config warning messages (fixes #433). --- src/Expr.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Expr.cc b/src/Expr.cc index f6d1fc568e..5eadaad631 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -359,7 +359,7 @@ bool NameExpr::DoUnserialize(UnserialInfo* info) if ( id ) ::Ref(id); else - reporter->Warning("unserialized unknown global name"); + reporter->Warning("persistent state config changed: unserialized unknown global name"); delete [] name; } From 86cba4c33f2641eefa5417c4455bec5082e35ba4 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 13 Dec 2011 16:17:44 -0600 Subject: [PATCH 180/964] Fix missing action in notice policy for looking up GeoIP data. --- scripts/base/frameworks/notice/actions/add-geodata.bro | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/base/frameworks/notice/actions/add-geodata.bro b/scripts/base/frameworks/notice/actions/add-geodata.bro index bc4021abea..9f6909595c 100644 --- a/scripts/base/frameworks/notice/actions/add-geodata.bro +++ b/scripts/base/frameworks/notice/actions/add-geodata.bro @@ -31,6 +31,7 @@ export { ## Add a helper to the notice policy for looking up GeoIP data. redef Notice::policy += { [$pred(n: Notice::Info) = { return (n$note in Notice::lookup_location_types); }, + $action = ACTION_ADD_GEODATA, $priority = 10], }; } From a543ebbea5b2021953e61c512d2188f7b6ece7bb Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 14 Dec 2011 10:05:52 -0600 Subject: [PATCH 181/964] Add more notice framework documentation. --- .../frameworks/notice/actions/email_admin.bro | 5 ++ .../base/frameworks/notice/actions/page.bro | 2 + .../frameworks/notice/actions/pp-alarms.bro | 10 +-- scripts/base/frameworks/notice/cluster.bro | 9 +- scripts/base/frameworks/notice/main.bro | 86 ++++++++++++++----- scripts/base/frameworks/notice/weird.bro | 36 +++++++- 6 files changed, 117 insertions(+), 31 deletions(-) diff --git a/scripts/base/frameworks/notice/actions/email_admin.bro b/scripts/base/frameworks/notice/actions/email_admin.bro index 56c0d5853d..044deb192a 100644 --- a/scripts/base/frameworks/notice/actions/email_admin.bro +++ b/scripts/base/frameworks/notice/actions/email_admin.bro @@ -1,3 +1,8 @@ +##! Adds a new notice action type which can be used to email notices +##! to the administrators of a particular address space as set by +##! :bro:id:`Site::local_admins` if the notice contains a source +##! or destination address that lies within their space. + @load ../main @load base/utils/site diff --git a/scripts/base/frameworks/notice/actions/page.bro b/scripts/base/frameworks/notice/actions/page.bro index f88064ac47..8002566a1a 100644 --- a/scripts/base/frameworks/notice/actions/page.bro +++ b/scripts/base/frameworks/notice/actions/page.bro @@ -1,3 +1,5 @@ +##! Allows configuration of a pager email address to which notices can be sent. + @load ../main module Notice; diff --git a/scripts/base/frameworks/notice/actions/pp-alarms.bro b/scripts/base/frameworks/notice/actions/pp-alarms.bro index 1284d7885f..1d6f8c7515 100644 --- a/scripts/base/frameworks/notice/actions/pp-alarms.bro +++ b/scripts/base/frameworks/notice/actions/pp-alarms.bro @@ -1,6 +1,6 @@ -#! Notice extension that mails out a pretty-printed version of alarm.log -#! in regular intervals, formatted for better human readability. If activated, -#! that replaces the default summary mail having the raw log output. +##! Notice extension that mails out a pretty-printed version of alarm.log +##! in regular intervals, formatted for better human readability. If activated, +##! that replaces the default summary mail having the raw log output. @load base/frameworks/cluster @load ../main @@ -15,8 +15,8 @@ export { ## :bro:id:`Notice::mail_dest`. const mail_dest_pretty_printed = "" &redef; - ## If an address from one of these networks is reported, we mark - ## the entry with an addition quote symbol (i.e., ">"). Many MUAs + ## If an address from one of these networks is reported, we mark + ## the entry with an addition quote symbol (that is, ">"). Many MUAs ## then highlight such lines differently. global flag_nets: set[subnet] &redef; diff --git a/scripts/base/frameworks/notice/cluster.bro b/scripts/base/frameworks/notice/cluster.bro index 7270e14933..281901cf31 100644 --- a/scripts/base/frameworks/notice/cluster.bro +++ b/scripts/base/frameworks/notice/cluster.bro @@ -1,4 +1,6 @@ -##! Implements notice functionality across clusters. +##! Implements notice functionality across clusters. Worker nodes +##! will disable notice/alarm logging streams and forward notice +##! events to the manager node for logging/processing. @load ./main @load base/frameworks/cluster @@ -7,10 +9,15 @@ module Notice; export { ## This is the event used to transport notices on the cluster. + ## + ## n: The notice information to be sent to the cluster manager for + ## further processing. global cluster_notice: event(n: Notice::Info); } +## Manager can communicate notice suppression to workers. redef Cluster::manager2worker_events += /Notice::begin_suppression/; +## Workers needs need ability to forward notices to manager. redef Cluster::worker2manager_events += /Notice::cluster_notice/; @if ( Cluster::local_node_type() != Cluster::MANAGER ) diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index 7d98c6464c..f3b0ede430 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -2,8 +2,7 @@ ##! are odd or potentially bad. Decisions of the meaning of various notices ##! need to be done per site because Bro does not ship with assumptions about ##! what is bad activity for sites. More extensive documetation about using -##! the notice framework can be found in the documentation section of the -##! http://www.bro-ids.org/ website. +##! the notice framework can be found in :doc:`/notice`. module Notice; @@ -21,10 +20,10 @@ export { ## Scripts creating new notices need to redef this enum to add their own ## specific notice types which would then get used when they call the ## :bro:id:`NOTICE` function. The convention is to give a general category - ## along with the specific notice separating words with underscores and using - ## leading capitals on each word except for abbreviations which are kept in - ## all capitals. For example, SSH::Login is for heuristically guessed - ## successful SSH logins. + ## along with the specific notice separating words with underscores and + ## using leading capitals on each word except for abbreviations which are + ## kept in all capitals. For example, SSH::Login is for heuristically + ## guessed successful SSH logins. type Type: enum { ## Notice reporting a count of how often a notice occurred. Tally, @@ -49,22 +48,33 @@ export { }; ## The notice framework is able to do automatic notice supression by - ## utilizing the $identifier field in :bro:type:`Info` records. + ## utilizing the $identifier field in :bro:type:`Notice::Info` records. ## Set this to "0secs" to completely disable automated notice suppression. const default_suppression_interval = 1hrs &redef; type Info: record { + ## An absolute time indicating when the notice occurred, defaults + ## to the current network time. ts: time &log &optional; + + ## A connection UID which uniquely identifies the endpoints + ## concerned with the notice. uid: string &log &optional; + + ## A connection 4-tuple identifying the endpoints concerned with the + ## notice. id: conn_id &log &optional; - ## These are shorthand ways of giving the uid and id to a notice. The + ## A shorthand way of giving the uid and id to a notice. The ## reference to the actual connection will be deleted after applying ## the notice policy. conn: connection &optional; + ## A shorthand way of giving the uid and id to a notice. The + ## reference to the actual connection will be deleted after applying + ## the notice policy. iconn: icmp_conn &optional; - ## The :bro:enum:`Notice::Type` of the notice. + ## The type of the notice. note: Type &log; ## The human readable message for the notice. msg: string &log &optional; @@ -141,8 +151,9 @@ export { ## This is the record that defines the items that make up the notice policy. type PolicyItem: record { - ## This is the exact positional order in which the :bro:type:`PolicyItem` - ## records are checked. This is set internally by the notice framework. + ## This is the exact positional order in which the + ## :bro:type:`Notice::PolicyItem` records are checked. + ## This is set internally by the notice framework. position: count &log &optional; ## Define the priority for this check. Items are checked in ordered ## from highest value (10) to lowest value (0). @@ -163,8 +174,8 @@ export { suppress_for: interval &log &optional; }; - ## This is the where the :bro:id:`Notice::policy` is defined. All notice - ## processing is done through this variable. + ## Defines a notice policy that is extensible on a per-site basis. + ## All notice processing is done through this variable. const policy: set[PolicyItem] = { [$pred(n: Notice::Info) = { return (n$note in Notice::ignored_types); }, $halt=T, $priority = 9], @@ -193,8 +204,9 @@ export { ## Local system sendmail program. const sendmail = "/usr/sbin/sendmail" &redef; - ## Email address to send notices with the :bro:enum:`ACTION_EMAIL` action - ## or to send bulk alarm logs on rotation with :bro:enum:`ACTION_ALARM`. + ## Email address to send notices with the :bro:enum:`Notice::ACTION_EMAIL` + ## action or to send bulk alarm logs on rotation with + ## :bro:enum:`Notice::ACTION_ALARM`. const mail_dest = "" &redef; ## Address that emails will be from. @@ -207,14 +219,20 @@ export { ## A log postprocessing function that implements emailing the contents ## of a log upon rotation to any configured :bro:id:`Notice::mail_dest`. ## The rotated log is removed upon being sent. + ## + ## info: A record containing the rotated log file information. + ## + ## Returns: True. global log_mailing_postprocessor: function(info: Log::RotationInfo): bool; ## This is the event that is called as the entry point to the ## notice framework by the global :bro:id:`NOTICE` function. By the time ## this event is generated, default values have already been filled out in ## the :bro:type:`Notice::Info` record and synchronous functions in the - ## :bro:id:`Notice:sync_functions` have already been called. The notice + ## :bro:id:`Notice::sync_functions` have already been called. The notice ## policy has also been applied. + ## + ## n: The record containing notice data. global notice: event(n: Info); ## This is a set of functions that provide a synchronous way for scripts @@ -231,30 +249,55 @@ export { const sync_functions: set[function(n: Notice::Info)] = set() &redef; ## This event is generated when a notice begins to be suppressed. + ## + ## n: The record containing notice data regarding the notice type + ## about to be suppressed. global begin_suppression: event(n: Notice::Info); + ## This event is generated on each occurence of an event being suppressed. + ## + ## n: The record containing notice data regarding the notice type + ## being suppressed. global suppressed: event(n: Notice::Info); + ## This event is generated when a notice stops being suppressed. + ## + ## n: The record containing notice data regarding the notice type + ## that was being suppressed. global end_suppression: event(n: Notice::Info); ## Call this function to send a notice in an email. It is already used - ## by default with the built in :bro:enum:`ACTION_EMAIL` and - ## :bro:enum:`ACTION_PAGE` actions. + ## by default with the built in :bro:enum:`Notice::ACTION_EMAIL` and + ## :bro:enum:`Notice::ACTION_PAGE` actions. + ## + ## n: The record of notice data to email. + ## + ## dest: The intended recipient of the notice email. + ## + ## extend: Whether to extend the email using the ``email_body_sections`` + ## field of *n*. global email_notice_to: function(n: Info, dest: string, extend: bool); ## Constructs mail headers to which an email body can be appended for ## sending with sendmail. + ## ## subject_desc: a subject string to use for the mail + ## ## dest: recipient string to use for the mail + ## ## Returns: a string of mail headers to which an email body can be appended global email_headers: function(subject_desc: string, dest: string): string; - ## This event can be handled to access the :bro:type:`Info` + ## This event can be handled to access the :bro:type:`Notice::Info` ## record as it is sent on to the logging framework. + ## + ## rec: The record containing notice data before it is logged. global log_notice: event(rec: Info); - ## This is an internal wrapper for the global NOTICE function. Please + ## This is an internal wrapper for the global :bro:id:`NOTICE` function; ## disregard. + ## + ## n: The record of notice data. global internal_NOTICE: function(n: Notice::Info); } @@ -410,7 +453,8 @@ event notice(n: Notice::Info) &priority=-5 } ## This determines if a notice is being suppressed. It is only used -## internally as part of the mechanics for the global NOTICE function. +## internally as part of the mechanics for the global :bro:id:`NOTICE` +## function. function is_being_suppressed(n: Notice::Info): bool { if ( n?$identifier && [n$note, n$identifier] in suppressing ) diff --git a/scripts/base/frameworks/notice/weird.bro b/scripts/base/frameworks/notice/weird.bro index 379409532c..f894a42464 100644 --- a/scripts/base/frameworks/notice/weird.bro +++ b/scripts/base/frameworks/notice/weird.bro @@ -1,3 +1,12 @@ +##! This script provides a default set of actions to take for "weird activity" +##! events generated from Bro's event engine. Weird activity is defined as +##! unusual or exceptional activity that can indicate malformed connections, +##! traffic that doesn't conform to a particular protocol, malfunctioning +##! or misconfigured hardware, or even an attacker attempting to avoid/confuse +##! a sensor. Without context, it's hard to judge whether a particular +##! category of weird activity is interesting, but this script provides +##! a starting point for the user. + @load base/utils/conn-ids @load base/utils/site @load ./main @@ -5,6 +14,7 @@ module Weird; export { + ## The weird logging stream identifier. redef enum Log::ID += { LOG }; redef enum Notice::Type += { @@ -12,6 +22,7 @@ export { Activity, }; + ## The record type which contains the column fields of the weird log. type Info: record { ## The time when the weird occurred. ts: time &log; @@ -32,19 +43,32 @@ export { peer: string &log &optional; }; + ## Types of actions that may be taken when handling weird activity events. type Action: enum { + ## A dummy action indicating the user does not care what internal + ## decision is made regarding a given type of weird. ACTION_UNSPECIFIED, + ## No action is to be taken. ACTION_IGNORE, + ## Log the weird event every time it occurs. ACTION_LOG, + ## Log the weird event only once. ACTION_LOG_ONCE, + ## Log the weird event once per connection. ACTION_LOG_PER_CONN, + ## Log the weird event once per originator host. ACTION_LOG_PER_ORIG, + ## Always generate a notice associated with the weird event. ACTION_NOTICE, + ## Generate a notice associated with the weird event only once. ACTION_NOTICE_ONCE, + ## Generate a notice for the weird event once per connection. ACTION_NOTICE_PER_CONN, + ## Generate a notice for the weird event once per originator host. ACTION_NOTICE_PER_ORIG, }; + ## A table specifying default/recommended actions per weird type. const actions: table[string] of Action = { ["unsolicited_SYN_response"] = ACTION_IGNORE, ["above_hole_data_without_any_acks"] = ACTION_LOG, @@ -201,7 +225,7 @@ export { ["fragment_overlap"] = ACTION_LOG_PER_ORIG, ["fragment_protocol_inconsistency"] = ACTION_LOG, ["fragment_size_inconsistency"] = ACTION_LOG_PER_ORIG, - ## These do indeed happen! + # These do indeed happen! ["fragment_with_DF"] = ACTION_LOG, ["incompletely_captured_fragment"] = ACTION_LOG, ["bad_IP_checksum"] = ACTION_LOG_PER_ORIG, @@ -215,8 +239,8 @@ export { ## and weird name into this set. const ignore_hosts: set[addr, string] &redef; - # But don't ignore these (for the weird file), it's handy keeping - # track of clustered checksum errors. + ## Don't ignore repeats for weirds in this set. For example, + ## it's handy keeping track of clustered checksum errors. const weird_do_not_ignore_repeats = { "bad_IP_checksum", "bad_TCP_checksum", "bad_UDP_checksum", "bad_ICMP_checksum", @@ -236,7 +260,11 @@ export { ## A state set which tracks unique weirds solely by the name to reduce ## duplicate notices from being raised. global did_notice: set[string, string] &create_expire=1day &redef; - + + ## Handlers of this event are invoked one per write to the weird + ## logging stream before the data is actually written. + ## + ## rec: The weird columns about to be logged to the weird stream. global log_weird: event(rec: Info); } From d89658c19b921c8d06765bff947b09bfd0bf7534 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 14 Dec 2011 12:50:54 -0600 Subject: [PATCH 182/964] Add more signature framework documentation. --- doc/signatures.rst | 24 +++---- scripts/base/frameworks/signatures/main.bro | 71 +++++++++++++-------- 2 files changed, 58 insertions(+), 37 deletions(-) diff --git a/doc/signatures.rst b/doc/signatures.rst index e2f9a8d702..a1e70f8e6f 100644 --- a/doc/signatures.rst +++ b/doc/signatures.rst @@ -34,7 +34,7 @@ Let's look at an example signature first: This signature asks Bro to match the regular expression ``.*root`` on all TCP connections going to port 80. When the signature triggers, Bro -will raise an event ``signature_match`` of the form: +will raise an event :bro:id:`signature_match` of the form: .. code:: bro @@ -45,20 +45,20 @@ triggered the match, ``msg`` is the string specified by the signature's event statement (``Found root!``), and data is the last piece of payload which triggered the pattern match. -To turn such ``signature_match`` events into actual alarms, you can -load Bro's ``signature.bro`` script. This script contains a default -event handler that raises ``SensitiveSignature`` :doc:`Notices ` +To turn such :bro:id:`signature_match` events into actual alarms, you can +load Bro's :doc:`/scripts/base/frameworks/signatures/main` script. +This script contains a default event handler that raises +:bro:enum:`Signatures::Sensitive_Signature` :doc:`Notices ` (as well as others; see the beginning of the script). As signatures are independent of Bro's policy scripts, they are put into their own file(s). There are two ways to specify which files contain signatures: By using the ``-s`` flag when you invoke Bro, or -by extending the Bro variable ``signatures_files`` using the ``+=`` +by extending the Bro variable :bro:id:`signature_files` using the ``+=`` operator. If a signature file is given without a path, it is searched along the normal ``BROPATH``. The default extension of the file name is ``.sig``, and Bro appends that automatically when neccesary. - Signature language ================== @@ -90,7 +90,7 @@ one of ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``; and against. The following keywords are defined: ``src-ip``/``dst-ip `` - Source and destination address, repectively. Addresses can be + Source and destination address, respectively. Addresses can be given as IP addresses or CIDR masks. ``src-port``/``dst-port`` ```` @@ -126,7 +126,7 @@ CIDR notation for netmasks and is translated into a corresponding bitmask applied to the packet's value prior to the comparison (similar to the optional ``& integer``). -Putting all together, this is an example conditiation that is +Putting all together, this is an example condition that is equivalent to ``dst- ip == 1.2.3.4/16, 5.6.7.8/24``: .. code:: bro-sig @@ -134,7 +134,7 @@ equivalent to ``dst- ip == 1.2.3.4/16, 5.6.7.8/24``: header ip[16:4] == 1.2.3.4/16, 5.6.7.8/24 Internally, the predefined header conditions are in fact just -short-cuts and mappend into a generic condition. +short-cuts and mapped into a generic condition. Content Conditions ~~~~~~~~~~~~~~~~~~ @@ -265,7 +265,7 @@ Actions define what to do if a signature matches. Currently, there are two actions defined: ``event `` - Raises a ``signature_match`` event. The event handler has the + Raises a :bro:id:`signature_match` event. The event handler has the following type: .. code:: bro @@ -339,10 +339,10 @@ Things to keep in mind when writing signatures respectively. Generally, Bro follows `flex's regular expression syntax `_. - See the DPD signatures in ``policy/sigs/dpd.bro`` for some examples + See the DPD signatures in ``base/frameworks/dpd/dpd.sig`` for some examples of fairly complex payload patterns. -* The data argument of the ``signature_match`` handler might not carry +* The data argument of the :bro:id:`signature_match` handler might not carry the full text matched by the regular expression. Bro performs the matching incrementally as packets come in; when the signature eventually fires, it can only pass on the most recent chunk of data. diff --git a/scripts/base/frameworks/signatures/main.bro b/scripts/base/frameworks/signatures/main.bro index 4811fdd5a9..26f78a68d1 100644 --- a/scripts/base/frameworks/signatures/main.bro +++ b/scripts/base/frameworks/signatures/main.bro @@ -1,30 +1,36 @@ -##! Script level signature support. +##! Script level signature support. See the +##! :doc:`signature documentation ` for more information about +##! Bro's signature engine. @load base/frameworks/notice module Signatures; export { + ## Add various signature-related notice types. redef enum Notice::Type += { - ## Generic for alarm-worthy + ## Generic notice type for notice-worthy signature matches. Sensitive_Signature, ## Host has triggered many signatures on the same host. The number of - ## signatures is defined by the :bro:id:`vert_scan_thresholds` variable. + ## signatures is defined by the + ## :bro:id:`Signatures::vert_scan_thresholds` variable. Multiple_Signatures, - ## Host has triggered the same signature on multiple hosts as defined by the - ## :bro:id:`horiz_scan_thresholds` variable. + ## Host has triggered the same signature on multiple hosts as defined + ## by the :bro:id:`Signatures::horiz_scan_thresholds` variable. Multiple_Sig_Responders, - ## The same signature has triggered multiple times for a host. The number - ## of times the signature has be trigger is defined by the - ## :bro:id:`count_thresholds` variable. To generate this notice, the - ## :bro:enum:`SIG_COUNT_PER_RESP` action must be set for the signature. + ## The same signature has triggered multiple times for a host. The + ## number of times the signature has been triggered is defined by the + ## :bro:id:`Signatures::count_thresholds` variable. To generate this + ## notice, the :bro:enum:`Signatures::SIG_COUNT_PER_RESP` action must + ## bet set for the signature. Count_Signature, ## Summarize the number of times a host triggered a signature. The - ## interval between summaries is defined by the :bro:id:`summary_interval` - ## variable. + ## interval between summaries is defined by the + ## :bro:id:`Signatures::summary_interval` variable. Signature_Summary, }; + ## The signature logging stream identifier. redef enum Log::ID += { LOG }; ## These are the default actions you can apply to signature matches. @@ -39,8 +45,8 @@ export { SIG_QUIET, ## Generate a notice. SIG_LOG, - ## The same as :bro:enum:`SIG_FILE`, but ignore for aggregate/scan - ## processing. + ## The same as :bro:enum:`Signatures::SIG_LOG`, but ignore for + ## aggregate/scan processing. SIG_FILE_BUT_NO_SCAN, ## Generate a notice and set it to be alarmed upon. SIG_ALARM, @@ -49,22 +55,33 @@ export { ## Alarm once and then never again. SIG_ALARM_ONCE, ## Count signatures per responder host and alarm with the - ## :bro:enum:`Count_Signature` notice if a threshold defined by - ## :bro:id:`count_thresholds` is reached. + ## :bro:enum:`Signatures::Count_Signature` notice if a threshold + ## defined by :bro:id:`Signatures::count_thresholds` is reached. SIG_COUNT_PER_RESP, ## Don't alarm, but generate per-orig summary. SIG_SUMMARY, }; - + + ## The record type which contains the column fields of the signature log. type Info: record { + ## The network time at which a signature matching type of event to + ## be logged has occurred. ts: time &log; + ## The host which triggered the signature match event. src_addr: addr &log &optional; + ## The host port on which the signature-matching activity occurred. src_port: port &log &optional; + ## The destination host which was sent the payload that triggered the + ## signature match. dst_addr: addr &log &optional; + ## The destination host port which was sent the payload that triggered + ## the signature match. dst_port: port &log &optional; ## Notice associated with signature event note: Notice::Type &log; + ## The name of the signature that matched. sig_id: string &log &optional; + ## A more descriptive message of the signature-matching event. event_msg: string &log &optional; ## Extracted payload data or extra message. sub_msg: string &log &optional; @@ -82,22 +99,26 @@ export { ## Signature IDs that should always be ignored. const ignored_ids = /NO_DEFAULT_MATCHES/ &redef; - ## Alarm if, for a pair [orig, signature], the number of different - ## responders has reached one of the thresholds. + ## Generate a notice if, for a pair [orig, signature], the number of + ## different responders has reached one of the thresholds. const horiz_scan_thresholds = { 5, 10, 50, 100, 500, 1000 } &redef; - ## Alarm if, for a pair [orig, resp], the number of different signature - ## matches has reached one of the thresholds. + ## Generate a notice if, for a pair [orig, resp], the number of different + ## signature matches has reached one of the thresholds. const vert_scan_thresholds = { 5, 10, 50, 100, 500, 1000 } &redef; - ## Alarm if a :bro:enum:`SIG_COUNT_PER_RESP` signature is triggered as - ## often as given by one of these thresholds. + ## Generate a notice if a :bro:enum:`Signatures::SIG_COUNT_PER_RESP` + ## signature is triggered as often as given by one of these thresholds. const count_thresholds = { 5, 10, 50, 100, 500, 1000, 10000, 1000000, } &redef; - ## The interval between when :bro:id:`Signature_Summary` notices are - ## generated. + ## The interval between when :bro:enum:`Signatures::Signature_Summary` + ## notice are generated. const summary_interval = 1 day &redef; - + + ## This event can be handled to access/alter data about to be logged + ## to the signature logging stream. + ## + ## rec: The record of signature data about to be logged. global log_signature: event(rec: Info); } From 303993254e63ad5bcfd789212a8011d9ee581298 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 14 Dec 2011 16:07:36 -0600 Subject: [PATCH 183/964] Add more DPD and packet filter framework docs. --- scripts/base/frameworks/dpd/main.bro | 4 +++- scripts/base/frameworks/packet-filter/main.bro | 13 +++++++++---- scripts/base/frameworks/packet-filter/netstats.bro | 4 +++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/scripts/base/frameworks/dpd/main.bro b/scripts/base/frameworks/dpd/main.bro index d9288bdd04..e8488c3ec1 100644 --- a/scripts/base/frameworks/dpd/main.bro +++ b/scripts/base/frameworks/dpd/main.bro @@ -7,14 +7,16 @@ module DPD; redef signature_files += "base/frameworks/dpd/dpd.sig"; export { + ## Add the DPD logging stream identifier. redef enum Log::ID += { LOG }; + ## The record type defining the columns to log in the DPD logging stream. type Info: record { ## Timestamp for when protocol analysis failed. ts: time &log; ## Connection unique ID. uid: string &log; - ## Connection ID. + ## Connection ID containing the 4-tuple which identifies endpoints. id: conn_id &log; ## Transport protocol for the violation. proto: transport_proto &log; diff --git a/scripts/base/frameworks/packet-filter/main.bro b/scripts/base/frameworks/packet-filter/main.bro index 1097315172..16e3ff9789 100644 --- a/scripts/base/frameworks/packet-filter/main.bro +++ b/scripts/base/frameworks/packet-filter/main.bro @@ -9,17 +9,22 @@ module PacketFilter; export { + ## Add the packet filter logging stream. redef enum Log::ID += { LOG }; - + + ## Add notice types related to packet filter errors. redef enum Notice::Type += { ## This notice is generated if a packet filter is unable to be compiled. Compile_Failure, - ## This notice is generated if a packet filter is unable to be installed. + ## This notice is generated if a packet filter is fails to install. Install_Failure, }; - + + ## The record type defining columns to be logged in the packet filter + ## logging stream. type Info: record { + ## The time at which the packet filter installation attempt was made. ts: time &log; ## This is a string representation of the node that applied this @@ -40,7 +45,7 @@ export { ## By default, Bro will examine all packets. If this is set to false, ## it will dynamically build a BPF filter that only select protocols ## for which the user has loaded a corresponding analysis script. - ## The latter used to be default for Bro versions < 1.6. That has now + ## The latter used to be default for Bro versions < 2.0. That has now ## changed however to enable port-independent protocol analysis. const all_packets = T &redef; diff --git a/scripts/base/frameworks/packet-filter/netstats.bro b/scripts/base/frameworks/packet-filter/netstats.bro index 69b5026515..9fbaa5cd1d 100644 --- a/scripts/base/frameworks/packet-filter/netstats.bro +++ b/scripts/base/frameworks/packet-filter/netstats.bro @@ -1,4 +1,6 @@ ##! This script reports on packet loss from the various packet sources. +##! When Bro is reading input from trace files, this script will not +##! report any packet loss statistics. @load base/frameworks/notice @@ -6,7 +8,7 @@ module PacketFilter; export { redef enum Notice::Type += { - ## Bro reported packets dropped by the packet filter. + ## Indicates packets were dropped by the packet filter. Dropped_Packets, }; From 55c982fa14526fa2ccf0a23dc3c8650f5993c1f5 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 14 Dec 2011 07:36:06 -0800 Subject: [PATCH 184/964] Adding Broxygen comments to init-bare.bro. I've left a few TODOs in there for protocol-specific fields that I couldn't directly figure out in their meaning. Feel free to fill in where you can. --- aux/broccoli | 2 +- aux/broctl | 2 +- scripts/base/init-bare.bro | 2264 ++++++++++++++++++++++++------------ src/event.bif | 5 +- 4 files changed, 1549 insertions(+), 724 deletions(-) diff --git a/aux/broccoli b/aux/broccoli index 05d51b55dd..d8f9d4698e 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 05d51b55dd5a8430151cff74f7876bd73385d5cd +Subproject commit d8f9d4698e8e02f493a669c5adcf897506671b5d diff --git a/aux/broctl b/aux/broctl index 406c689c88..be772bbada 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 406c689c88997c4459eaf36bd2c65bce2e847c2d +Subproject commit be772bbada79b106db33fb9de5f56fa71226adc5 diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 859a69f2dc..3ad114d6dc 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -2,172 +2,311 @@ @load base/types.bif # Type declarations + +## An ordered array of strings. The entries are indexed by succesive numbers. Note +## that it depends on the usage whether the first index is zero or one. +## +## .. todo:: We need this type definition only for declaring builtin functions via +## ``bifcl``. We should extend ``bifcl`` to understand composite types directly and +## then remove this alias. type string_array: table[count] of string; + +## A set of strings. +## +## .. todo:: We need this type definition only for declaring builtin functions via +## ``bifcl``. We should extend ``bifcl`` to understand composite types directly and +## then remove this alias. type string_set: set[string]; + +## A set of addresses. +## +## .. todo:: We need this type definition only for declaring builtin functions via +## ``bifcl``. We should extend ``bifcl`` to understand composite types directly and +## then remove this alias. type addr_set: set[addr]; + +## A set of counts. +## +## .. todo:: We need this type definition only for declaring builtin functions via +## ``bifcl``. We should extend ``bifcl`` to understand composite types directly and +## then remove this alias. type count_set: set[count]; + +## A vector of counts, used by some builtin functions to store a list of indices. +## +## .. todo:: We need this type definition only for declaring builtin functions via +## ``bifcl``. We should extend ``bifcl`` to understand composite types directly and +## then remove this alias. type index_vec: vector of count; + +## A vector of strings. +## +## .. todo:: We need this type definition only for declaring builtin functions via +## ``bifcl``. We should extend ``bifcl`` to understand composite types directly and +## then remove this alias. type string_vec: vector of string; +## A table of strings indexed by strings. +## +## .. todo:: We need this type definition only for declaring builtin functions via +## ``bifcl``. We should extend ``bifcl`` to understand composite types directly and +## then remove this alias. type table_string_of_string: table[string] of string; -type transport_proto: enum { unknown_transport, tcp, udp, icmp }; +## A connection's transport-layer protocol. Note that Bro uses the term +## "connection" broadly, using flow semantics for ICMP and UDP. +type transport_proto: enum { + unknown_transport, ##< An unknown transport-layer protocol. + tcp, ##< TCP. + udp, ##< UDP. + icmp ##< ICMP. +}; +## A connection's identifying 4-tuple of endpoints and ports. +## +## .. note:: It's actually a 5-tuple: the transport-layer protocol is stored as +## part of the port values, `orig_p` and `resp_p`, and can be extracted from them +## with :bro:id:`get_port_transport_proto`. type conn_id: record { - orig_h: addr; - orig_p: port; - resp_h: addr; - resp_p: port; + orig_h: addr; ##< The originator's IP address. + orig_p: port; ##< The originator's port number. + resp_h: addr; ##< The responder's IP address. + resp_p: port; ##< The responder's port number. } &log; +## Specifics about an ICMP conversation. ICMP events typically pass this in +## addition to :bro:id:`conn_id`. +## +## .. bro:see:: icmp_echo_reply icmp_echo_request icmp_redirect icmp_sent +## icmp_time_exceeded icmp_unreachable type icmp_conn: record { - orig_h: addr; - resp_h: addr; - itype: count; - icode: count; - len: count; -}; - -type icmp_hdr: record { - icmp_type: count; ##< type of message + orig_h: addr; ##< The originator's IP address. + resp_h: addr; ##< The responder's IP address. + itype: count; ##< The ICMP type of the packet that triggered the instantiation of the record. + icode: count; ##< The ICMP code of the packet that triggered the instantiation of the record. + len: count; ##< The length of the ICMP payload of the packet that triggered the instantiation of the record. }; +## Packet context part of an ICMP message. The fields of this record reflect the +## packet that is described by the context. +## +## .. bro:see:: icmp_time_exceeded icmp_unreachable type icmp_context: record { - id: conn_id; - len: count; - proto: count; - frag_offset: count; + id: conn_id; ##< The packet's 4-tuple. + len: count; ##< The lenght of the packet's IP header. + proto: count; ##< The packet's transport-layer protocol. + frag_offset: count; ##< The packet's fragementation offset. + ## True if the packet's IP header is fully included in the context. If that is not + ## the case, the other fields will all be set to null values. bad_hdr_len: bool; - bad_checksum: bool; - MF: bool; - DF: bool; + bad_checksum: bool; ##< True if the packet's IP checksum is not correct. + MF: bool; ##< True if the packets *more fragements* flag is set. + DF: bool; ##< True if the packets *don't fragment* flag is set. }; +# A DNS mapping between IP address and hostname resolved by Bro's internal +# resolver. +# +# .. bro:see:: dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +# dns_mapping_unverified dns_mapping_valid type dns_mapping: record { + ## The time when the mapping was created, which corresponds to the when the DNS + ## query was sent out. creation_time: time; - + ## If the mapping is the result of a name lookup, the queried host name; otherwise + ## empty. req_host: string; + ## If the mapping is the result of a pointer lookup, the queried address; otherwise + ## null. req_addr: addr; - + ## True if the lookup returned success. Only then, the result ields are valid. valid: bool; + ## If the mapping is the result of a pointer lookup, the resolved hostname; + ## otherwise empty. hostname: string; + ## If the mapping is the result of an address lookup, the resolved address(es); + ## otherwise empty. addrs: addr_set; }; +## A parsed host/port combination describing server endpoint for an upcoming +## data transfert. +## +## .. bro:see:: fmt_ftp_port parse_eftp_port parse_ftp_epsv parse_ftp_pasv +## parse_ftp_port type ftp_port: record { - h: addr; - p: port; - valid: bool; ##< true if format was right -}; - -type endpoint: record { - size: count; ##< logical size (for TCP: from seq numbers) - state: count; - - ## Number of packets on the wire - ## Set if :bro:id:`use_conn_size_analyzer` is true. - num_pkts: count &optional; - ## Number of IP-level bytes on the wire - ## Set if :bro:id:`use_conn_size_analyzer` is true. - num_bytes_ip: count &optional; + h: addr; ##< The host's address. + p: port; ##< The host's port. + valid: bool; ##< True if format was right. Only then, *h* and *p* are valid. }; +## Statistics about what a TCP endpoint sent. +## +## .. bro:see:: conn_stats type endpoint_stats: record { - num_pkts: count; - num_rxmit: count; - num_rxmit_bytes: count; - num_in_order: count; - num_OO: count; - num_repl: count; + num_pkts: count; ##< Number of packets. + num_rxmit: count; ##< Number of retransmission. + num_rxmit_bytes: count; ##< Number of retransmitted bytes. + num_in_order: count; ##< Number of in-order packets. + num_OO: count; ##< Number out-of-order packets. + num_repl: count; ##< Number of replicated packets (last packet was sent again). + ## Endian type used by the endpoint, if it it could be determined from the sequence + ## numbers used. This is one of :bro:see:`ENDIAN_UNKNOWN`, :bro:see:`ENDIAN_BIG`, + ## :bro:see:`ENDIAN_LITTLE`, and :bro:see:`ENDIAN_CONFUSED`. endian_type: count; }; +## A unique analyzer instance ID. Each time instantiates a protocol analyzers +## for a connection, it assigns it a unique ID that can be used to reference +## that instance. +## +## .. bro:see:: analyzer_name disable_analyzer protocol_confirmation +## protocol_violation +## +## .. todo::While we declare an alias for the type here, the events/functions still +## use ``count``. That should be changed. type AnalyzerID: count; +## Statistics about an endpoint. +## +## todo::Where is this used? +type endpoint: record { + size: count; ##< Logical size of data sent (for TCP: derived from sequence numbers). + ## Endpoint state. For TCP connection, one of the constants: + ## :bro:see:`TCP_INACTIVE` :bro:see:`TCP_SYN_SENT` :bro:see:`TCP_SYN_ACK_SENT` + ## :bro:see:`TCP_PARTIAL` :bro:see:`TCP_ESTABLISHED` :bro:see:`TCP_CLOSED` + ## :bro:see:`TCP_RESET`. For UDP, one of :bro:see:`UDP_ACTIVE` and + ## :bro:see:`UDP_INACTIVE`. + state: count; + ## Number of packets sent. Only set if :bro:id:`use_conn_size_analyzer` is true. + num_pkts: count &optional; + ## Number of IP-level bytes sent. Only set if :bro:id:`use_conn_size_analyzer` is + ## true. + num_bytes_ip: count &optional; +}; + +# A connection. This is Bro's basic connection type describing IP- and +# transport-layer information about the conversation. Note that Bro uses a +# liberal interpreation of "connection" and associates instances of this type +# also with UDP and ICMP flows. type connection: record { - id: conn_id; - orig: endpoint; - resp: endpoint; - start_time: time; + id: conn_id; ##< The connection's identifying 4-tuple. + orig: endpoint; ##< Statistics about originator side. + resp: endpoint; ##< Statistics about responder side. + start_time: time; ##< The timestamp of the connection's first packet. + ## The duration of the conversation. Roughly speaking, this is the interval between + ## first and last data packet (low-level TCP details may adjust it somewhat in + ## ambigious cases). duration: interval; - service: string_set; ##< if empty, service hasn't been determined - addl: string; - hot: count; ##< how hot; 0 = don't know or not hot - history: string; + ## The set of services the connection is using as determined by Bro's dynamic + ## protocol detection. Each entry is the label of an analyzer that confirmed that + ## it could parse the connection payload. While typically, there will be at + ## most one entry for each connection, in principle it is possible that more than + ## one protocol analyzer is able to parse the same data. If so, all will + ## be recorded. Also note that the recorced services are independent of any + ## transport-level protocols. + service: set[string]; + addl: string; ##< Deprecated. + hot: count; ##< Deprecated. + history: string; ##< State history of TCP connections. See *history* in :bro:see:`Conn::Info`. + ## A globally unique connection identifier. For each connection, Bro creates an ID + ## that is very likely unique across independent Bro runs. These IDs can thus be + ## used to tag and locate information associated with that connection. uid: string; }; +## Fields of a SYN packet. +## +## .. bro:see:: connection_SYN_packet type SYN_packet: record { - is_orig: bool; - DF: bool; - ttl: count; - size: count; - win_size: count; - win_scale: int; - MSS: count; - SACK_OK: bool; + is_orig: bool; ##< True if the packet was sent the connection's originator. + DF: bool; ##< True if the *don't fragment* is set in the IP header. + ttl: count; ##< The IP header's time-to-live. + size: count; ##< The size of the packet's payload as specified in the IP header. + win_size: count; ##< The window size from the TCP header. + win_scale: int; ##< The window scale option if present, or -1 if not. + MSS: count; ##< The maximum segement size if present, or 0 if not. + SACK_OK: bool; ##< True if the *SACK* option is present. }; -## This record is used for grabbing packet capturing information from -## the core with the :bro:id:`net_stats` BiF. All counts are cumulative. +## Packet capture statistics. All counts are cumulative. +## +## .. bro:see:: net_stats type NetStats: record { - pkts_recvd: count &default=0; ##< Packets received by Bro. - pkts_dropped: count &default=0; ##< Packets dropped. - pkts_link: count &default=0; ##< Packets seen on the link (not always available). + pkts_recvd: count &default=0; ##< Packets received by Bro. + pkts_dropped: count &default=0; ##< Packets reported dropped by the system. + ## Packets seen on the link. Note that this may differ + ## from *pkts_recvd* because of a potential capture_filter. See + ## :doc:`/scripts/base/frameworks/packet-filter/main`. Depending on the packet + ## capture system, this value may not be available and will then be always set to + ## zero. + pkts_link: count &default=0; }; +## Statistics about Bro's resource consumption. +## +## .. bro:see:: resource_usage +## +## .. note:: All process-level values refer to Bro's main process only, not to +## the child process it spawns for doing communication. type bro_resources: record { - version: string; ##< Bro version string - debug: bool; ##< true if compiled with --enable-debug - start_time: time; ##< start time of process - real_time: interval; ##< elapsed real time since Bro started running - user_time: interval; ##< user CPU seconds - system_time: interval; ##< system CPU seconds - mem: count; ##< maximum memory consumed, in KB - minor_faults: count; ##< page faults not requiring actual I/O - major_faults: count; ##< page faults requiring actual I/O - num_swap: count; ##< times swapped out - blocking_input: count; ##< blocking input operations - blocking_output: count; ##< blocking output operations - num_context: count; ##< number of involuntary context switches + version: string; ##< Bro version string. + debug: bool; ##< True if compiled with --enable-debug. + start_time: time; ##< Start time of process. + real_time: interval; ##< Elapsed real time since Bro started running. + user_time: interval; ##< User CPU seconds. + system_time: interval; ##< System CPU seconds. + mem: count; ##< Maximum memory consumed, in KB. + minor_faults: count; ##< Page faults not requiring actual I/O. + major_faults: count; ##< Page faults requiring actual I/O. + num_swap: count; ##< Times swapped out. + blocking_input: count; ##< Blocking input operations. + blocking_output: count; ##< Blocking output operations. + num_context: count; ##< Number of involuntary context switches. - num_TCP_conns: count; ##< current number of TCP connections - num_UDP_conns: count; - num_ICMP_conns: count; - num_fragments: count; ##< current number of fragments pending reassembly - num_packets: count; ##< total number packets processed to date - num_timers: count; ##< current number of pending timers - num_events_queued: count; ##< total number of events queued so far - num_events_dispatched: count; ##< same for events dispatched + num_TCP_conns: count; ##< Current number of TCP connections in memory. + num_UDP_conns: count; ##< Current number of UDP flows in memory. + num_ICMP_conns: count; ##< Current number of ICMP flows in memory. + num_fragments: count; ##< Current number of fragments pending reassembly. + num_packets: count; ##< Total number packets processed to date. + num_timers: count; ##< Current number of pending timers. + num_events_queued: count; ##< Total number of events queued so far. + num_events_dispatched: count; ##< Total number of events dispatched so far. - max_TCP_conns: count; ##< maximum number of TCP connections, etc. - max_UDP_conns: count; - max_ICMP_conns: count; - max_fragments: count; - max_timers: count; + max_TCP_conns: count; ##< Maximum number of concurrent TCP connections so far. + max_UDP_conns: count; ##< Maximum number of concurrent UDP connections so far. + max_ICMP_conns: count; ##< Maximum number of concurrent ICMP connections so far. + max_fragments: count; ##< Maximum number of concurrently buffered fragements so far. + max_timers: count; ##< Maximum number of concurrent timers pending so far. }; - -## Summary statistics of all DFA_State_Caches. +## Summary statistics of all regular expression matchers. +## +## .. bro:see:: get_matcher_stats type matcher_stats: record { - matchers: count; ##< number of distinct RE matchers - dfa_states: count; ##< number of DFA states across all matchers - computed: count; ##< number of computed DFA state transitions - mem: count; ##< number of bytes used by DFA states - hits: count; ##< number of cache hits - misses: count; ##< number of cache misses - avg_nfa_states: count; ##< average # NFA states across all matchers + matchers: count; ##< Number of distinct RE matchers. + dfa_states: count; ##< Number of DFA states across all matchers. + computed: count; ##< Number of computed DFA state transitions. + mem: count; ##< Number of bytes used by DFA states. + hits: count; ##< Number of cache hits. + misses: count; ##< Number of cache misses. + avg_nfa_states: count; ##< Average number of NFA states across all matchers. }; -## Info provided to gap_report, and also available by get_gap_summary(). +## Statistics about number of gaps in TCP connections. +## +## .. bro:see:: gap_report get_gap_summary type gap_info: record { - ack_events: count; ##< how many ack events *could* have had gaps - ack_bytes: count; ##< how many bytes those covered - gap_events: count; ##< how many *did* have gaps - gap_bytes: count; ##< how many bytes were missing in the gaps: + ack_events: count; ##< How many ack events *could* have had gaps. + ack_bytes: count; ##< How many bytes those covered. + gap_events: count; ##< How many *did* have gaps. + gap_bytes: count; ##< How many bytes were missing in the gaps. }; -# This record should be read-only. +## Deprecated. +## +## .. todo:: Remove. It's still declared internally but doesn't seem used anywhere +## else. type packet: record { conn: connection; is_orig: bool; @@ -175,38 +314,90 @@ type packet: record { timestamp: time; }; +## Table type used to map variable names to their memory allocation. +## +## .. bro:see:: global_sizes +## +## .. todo:: We need this type definition only for declaring builtin functions via +## ``bifcl``. We should extend ``bifcl`` to understand composite types directly and +## then remove this alias. type var_sizes: table[string] of count; ##< indexed by var's name, returns size +## Meta-information about a script-level identifier. +## +## .. bro:see:: global_ids id_table type script_id: record { - type_name: string; - exported: bool; - constant: bool; - enum_constant: bool; - redefinable: bool; - value: any &optional; + type_name: string; ##< The name of the identifier's type. + exported: bool; ##< True if the identifier is exported. + constant: bool; ##< True if the identifier is a constant. + enum_constant: bool; ##< True if the identifier is an enum value. + redefinable: bool; ##< True if the identifier is declared with the :bro:attr:`redef` attribute. + value: any &optional; ##< The current value of the identifier. }; +## Table type used to map script-level identifiers to meta-information +## describing them. +## +## .. bro:see:: global_ids script_id +## +## .. todo:: We need this type definition only for declaring builtin functions via +## ``bifcl``. We should extend ``bifcl`` to understand composite types directly and +## then remove this alias. type id_table: table[string] of script_id; +## Meta-information about a record-field. +## +## .. bro:see:: record_fields record_field_table type record_field: record { - type_name: string; - log: bool; + type_name: string; ##< The name of the field's type. + log: bool; ##< True of the field is declared with :bro:attr:`log` attribute. + ## The current value of the field in the record instance passed into + ## :bro:see:`record_fields` (if it has one). value: any &optional; - default_val: any &optional; + default_val: any &optional; ##< The value of the :bro:attr:`default` attribute if defined. }; +## Table type used to map record field declarations to meta-information describing +## them. +## +## .. bro:see:: record_fields record_field +## +## .. todo:: We need this type definition only for declaring builtin functions via +## ``bifcl``. We should extend ``bifcl`` to understand composite types directly and +## then remove this alias. type record_field_table: table[string] of record_field; +# todo::Do we still needs these here? Can they move into the packet filter +# framework? +# # The following two variables are defined here until the core is not # dependent on the names remaining as they are now. -## This is the list of capture filters indexed by some user-definable ID. + +## Set of BPF capture filters to use for capturing, indexed by a user-definable +## ID (which must be unique). If Bro is *not* configured to examine +## :bro:id:`all_packets`, all packets matching at least one of the filters in this +## table (and all in :bro:id:`restrict_filters`) will be analyzed. +## +## .. bro:see:: PacketFilter PacketFilter::all_packets +## PacketFilter::unrestricted_filter restrict_filters global capture_filters: table[string] of string &redef; -## This is the list of restriction filters indexed by some user-definable ID. + +## Set of BPF filters to restrict capturing, indexed by a user-definable ID (which +## must be unique). If Bro is *not* configured to examine :bro:id:`all_packets`, +## only packets matching *all* of the filters in this table (and +## any in :bro:id:`capture_filters`) will be analyzed. +## +## .. bro:see:: PacketFilter PacketFilter::all_packets +## PacketFilter::unrestricted_filter capture_filters global restrict_filters: table[string] of string &redef; -# {precompile,install}_pcap_filter identify the filter by IDs +## Enum type identifying dynamic BPF filters. These are used by +## :bro:see:`precompile_pcap_filter` and :bro:see:`precompile_pcap_filter`. type PcapFilterID: enum { None }; +## Deprecated. +## +## .. bro:see:: anonymize_addr type IPAddrAnonymization: enum { KEEP_ORIG_ADDR, SEQUENTIALLY_NUMBERED, @@ -215,34 +406,54 @@ type IPAddrAnonymization: enum { PREFIX_PRESERVING_MD5, }; +## Deprecated. +## +## .. bro:see:: anonymize_addr type IPAddrAnonymizationClass: enum { - ORIG_ADDR, ##< client address - RESP_ADDR, ##< server address + ORIG_ADDR, + RESP_ADDR, OTHER_ADDR, }; - -## Events are generated by event_peer's (which may be either ourselves, or -## some remote process). +## A locally unique ID identifying a communication peer. The ID is returned by +## :bro:id:`connect`. +## +## .. bro:see:: connect Communication type peer_id: count; +## A communication peer. +## +## .. bro:see:: complete_handshake disconnect finished_send_state +## get_event_peer get_local_event_peer remote_capture_filter +## remote_connection_closed remote_connection_error +## remote_connection_established remote_connection_handshake_done +## remote_event_registered remote_log_peer remote_pong +## request_remote_events request_remote_logs request_remote_sync +## send_capture_filter send_current_packet send_id send_ping send_state +## set_accept_state set_compression_level +## +## .. todo::The type's name is to narrow these days, should rename. type event_peer: record { - id: peer_id; ##< locally unique ID of peer (returned by connect()) - host: addr; + id: peer_id; ##< Locally unique ID of peer (returned by :bro:id:`connect`). + host: addr; ##< The IP address of the peer. + ## Either the port we connected to at the peer; or our port the peer + ## connected to if the session is remotely initiated. p: port; - is_local: bool; ##< true if this peer describes the current process. - descr: string; ##< source's external_source_description - class: string &optional; # self-assigned class of the peer + is_local: bool; ##< True if this record describes the local process. + descr: string; ##< The peer's :bro:see:`peer_description`. + class: string &optional; ##< The self-assigned *class* of the peer. See :bro:see:`Communication::Node`. }; +## Deprecated. +## +## .. bro:see:: rotate_file rotate_file_by_name rotate_interval type rotate_info: record { - old_name: string; ##< original filename - new_name: string; ##< file name after rotation - open: time; ##< time when opened - close: time; ##< time when closed + old_name: string; ##< Original filename. + new_name: string; ##< File name after rotation. + open: time; ##< Time when opened. + close: time; ##< Time when closed. }; - ### The following aren't presently used, though they should be. # # Structures needed for subsequence computations (str_smith_waterman): # # @@ -251,6 +462,9 @@ type rotate_info: record { # SW_MULTIPLE, # }; +## Paramerts for the Smith-Waterman algorithm. +## +## .. bro:see:: str_smith_waterman type sw_params: record { ## Minimum size of a substring, minimum "granularity". min_strlen: count &default = 3; @@ -259,45 +473,73 @@ type sw_params: record { sw_variant: count &default = 0; }; +## Helper type for return value of Smith-Waterman algorithm. +## +## .. bro:see:: str_smith_waterman sw_substring_vec sw_substring sw_align_vec sw_params type sw_align: record { - str: string; ##< string a substring is part of - index: count; ##< at which offset + str: string; ##< String a substring is part of. + index: count; ##< Offset substring is located. }; +## Helper type for return value of Smith-Waterman algorithm. +## +## .. bro:see:: str_smith_waterman sw_substring_vec sw_substring sw_align sw_params type sw_align_vec: vector of sw_align; +## Helper type for return value of Smith-Waterman algorithm. +## +## .. bro:see:: str_smith_waterman sw_substring_vec sw_align_vec sw_align sw_params +## type sw_substring: record { - str: string; ##< a substring - aligns: sw_align_vec; ##< all strings of which it's a substring - new: bool; ##< true if start of new alignment + str: string; ##< A substring. + aligns: sw_align_vec; ##< All strings of which it's a substring. + new: bool; ##< True if start of new alignment. }; +## Return type for Smith-Waterman algorithm. +## +## .. bro:see:: str_smith_waterman sw_substring sw_align_vec sw_align sw_params +## +## .. todo:: We need this type definition only for declaring builtin functions via +## ``bifcl``. We should extend ``bifcl`` to understand composite types directly and +## then remove this alias. type sw_substring_vec: vector of sw_substring; -## Policy-level handling of pcap packets. +## Policy-level representation of a packet passed on by libpcap. The data includes +## the complete packet as returned by libpcap, including the link-layer header. +## +## .. bro:see:: dump_packet get_current_packet type pcap_packet: record { - ts_sec: count; - ts_usec: count; - caplen: count; - len: count; - data: string; + ts_sec: count; ##< The non-fractional part of the packet's timestamp (i.e., full seconds since the epoch). + ts_usec: count; ##< The fractional part of the packet's timestamp. + caplen: count; ##< The number of bytes captured (<= *len*). + len: count; ##< The length of the packet in bytes, including ` for more information, Bro uses the same +## code. +## +## .. bro:see:: entropy_test_add entropy_test_finish entropy_test_init find_entropy type entropy_test_result: record { - entropy: double; - chi_square: double; - mean: double; - monte_carlo_pi: double; - serial_correlation: double; + entropy: double; ##< Information density. + chi_square: double; ##< Chi-Square value. + mean: double; ##< Arithmetic Mean. + monte_carlo_pi: double; ##< Monte-carlo value for pi. + serial_correlation: double; ##< Serial correlation coefficient. }; # Prototypes of Bro built-in functions. @@ -305,13 +547,19 @@ type entropy_test_result: record { @load base/bro.bif @load base/reporter.bif +## Deprecated. This is superseded by the new logging framework. global log_file_name: function(tag: string): string &redef; + +## Deprecated. This is superseded by the new logging framework. global open_log_file: function(tag: string): file &redef; -## Where to store the persistent state. +## Specifies a directory for Bro store its persistent state. All globals can +## be declared persistent via the :bro:attr:`persistent` attribute. const state_dir = ".state" &redef; -## Length of the delays added when storing state incrementally. +## Length of the delays inserted when storing state incrementally. To avoid +## dropping packets when serializing larger volumes of persistent state to +## disk, Bro interleaves the operation with continued packet processing. const state_write_delay = 0.01 secs &redef; global done_with_network = F; @@ -328,6 +576,7 @@ function open_log_file(tag: string): file return open(log_file_name(tag)); } +## Internal function. function add_interface(iold: string, inew: string): string { if ( iold == "" ) @@ -335,8 +584,12 @@ function add_interface(iold: string, inew: string): string else return fmt("%s %s", iold, inew); } + +## Network interfaces to listen on. Use ``redef interfaces += "eth0"`` to +## extend. global interfaces = "" &add_func = add_interface; +## Internal function. function add_signature_file(sold: string, snew: string): string { if ( sold == "" ) @@ -344,11 +597,15 @@ function add_signature_file(sold: string, snew: string): string else return cat(sold, " ", snew); } + +## Signature files to read. Use ``redef signature_files += "foo.sig"`` to +## extend. Signature files will be searched relative to ``BRO_PATH``. global signature_files = "" &add_func = add_signature_file; +## ``p0f`` fingerprint file to use. Will be searched relative to ``BRO_PATH``. const passive_fingerprint_file = "base/misc/p0f.fp" &redef; -# TODO: testing to see if I can remove these without causing problems. +# todo::testing to see if I can remove these without causing problems. #const ftp = 21/tcp; #const ssh = 22/tcp; #const telnet = 23/tcp; @@ -361,17 +618,24 @@ const passive_fingerprint_file = "base/misc/p0f.fp" &redef; #const bgp = 179/tcp; #const rlogin = 513/tcp; -const TCP_INACTIVE = 0; -const TCP_SYN_SENT = 1; -const TCP_SYN_ACK_SENT = 2; -const TCP_PARTIAL = 3; -const TCP_ESTABLISHED = 4; -const TCP_CLOSED = 5; -const TCP_RESET = 6; +# TCP values for :bro:see:`endpoint` *state* field. +# todo::these should go into an enum to make them autodoc'able. +const TCP_INACTIVE = 0; ##< Endpoint is still inactive. +const TCP_SYN_SENT = 1; ##< Endpoint has sent SYN. +const TCP_SYN_ACK_SENT = 2; ##< Endpoint has sent SYN/ACK. +const TCP_PARTIAL = 3; ##< Endpoint has sent data but no initial SYN. +const TCP_ESTABLISHED = 4; ##< Endpoint has finished initial handshake regularly. +const TCP_CLOSED = 5; ##< Endpoint has closed connection. +const TCP_RESET = 6; ##< Endpoint has sent RST. + +# UDP values for :bro:see:`endpoint` *state* field. +# todo::these should go into an enum to make them autodoc'able. +const UDP_INACTIVE = 0; ##< Endpoint is still inactive. +const UDP_ACTIVE = 1; ##< Endpoint has sent something. ## If true, don't verify checksums. Useful for running on altered trace -## files, and for saving a few cycles, but of course dangerous, too ... -## Note that the -C command-line option overrides the setting of this +## files, and for saving a few cycles, but at the risk of analyzing invalid +## data. Note that the ``-C`` command-line option overrides the setting of this ## variable. const ignore_checksums = F &redef; @@ -379,13 +643,13 @@ const ignore_checksums = F &redef; ## (one missing its initial establishment negotiation) is seen. const partial_connection_ok = T &redef; -## If true, instantiate connection state when a SYN ack is seen -## but not the initial SYN (even if partial_connection_ok is false). +## If true, instantiate connection state when a SYN/ACK is seen but not the initial +## SYN (even if :bro:see:`partial_connection_ok` is false). const tcp_SYN_ack_ok = T &redef; -## If a connection state is removed there may still be some undelivered -## data waiting in the reassembler. If true, pass this to the signature -## engine before flushing the state. +## If true, pass any undelivered to the signature engine before flushing the state. +## If a connection state is removed, there may still be some data waiting in the +## reassembler. const tcp_match_undelivered = T &redef; ## Check up on the result of an initial SYN after this much time. @@ -416,33 +680,55 @@ const tcp_reset_delay = 5 secs &redef; const tcp_partial_close_delay = 3 secs &redef; ## If a connection belongs to an application that we don't analyze, -## time it out after this interval. If 0 secs, then don't time it out. +## time it out after this interval. If 0 secs, then don't time it out (but +## :bro:see:`tcp_inactivity_timeout`/:bro:see:`udp_inactivity_timeout`/:bro:see:`icmp_inactivity_timeout` +## still apply). const non_analyzed_lifetime = 0 secs &redef; -## If a connection is inactive, time it out after this interval. -## If 0 secs, then don't time it out. +## If a TCP connection is inactive, time it out after this interval. If 0 secs, +## then don't time it out. +## +## .. bro:see:: udp_inactivity_timeout icmp_inactivity_timeout set_inactivity_timeout const tcp_inactivity_timeout = 5 min &redef; -## See :bro:id:`tcp_inactivity_timeout` + +## If a UDP flow is inactive, time it out after this interval. If 0 secs, then +## don't time it out. +## +## .. bro:see:: tcp_inactivity_timeout icmp_inactivity_timeout set_inactivity_timeout const udp_inactivity_timeout = 1 min &redef; -## See :bro:id:`tcp_inactivity_timeout` + +## If an ICMP flow is inactive, time it out after this interval. If 0 secs, then +## don't time it out. +## +## .. bro:see:: tcp_inactivity_timeout udp_inactivity_timeout set_inactivity_timeout const icmp_inactivity_timeout = 1 min &redef; -## This many FINs/RSTs in a row constitutes a "storm". +## Number of FINs/RSTs in a row that constitute a "storm". Storms are reported via +## as ``weird`` via the notice framework, and they must also come within +## intervals of at most :bro:see:`tcp_storm_interarrival_thresh`. +## +## .. bro:see:: tcp_storm_interarrival_thresh const tcp_storm_thresh = 1000 &redef; -## The FINs/RSTs must come with this much time or less between them. +## FINs/RSTs must come with this much time or less between them to be +## considered a "storm". +## +## .. bro:see:: tcp_storm_thresh const tcp_storm_interarrival_thresh = 1 sec &redef; -## Maximum amount of data that might plausibly be sent in an initial -## flight (prior to receiving any acks). Used to determine whether we -## must not be seeing our peer's acks. Set to zero to turn off this -## determination. +## Maximum amount of data that might plausibly be sent in an initial flight (prior +## to receiving any acks). Used to determine whether we must not be seeing our +## peer's ACKs. Set to zero to turn off this determination. +## +## .. bro:see:: tcp_max_above_hole_without_any_acks tcp_excessive_data_without_further_acks const tcp_max_initial_window = 4096; -## If we're not seeing our peer's acks, the maximum volume of data above -## a sequence hole that we'll tolerate before assuming that there's -## been a packet drop and we should give up on tracking a connection. -## If set to zero, then we don't ever give up. +## If we're not seeing our peer's ACKs, the maximum volume of data above a sequence +## hole that we'll tolerate before assuming that there's been a packet drop and we +## should give up on tracking a connection. If set to zero, then we don't ever give +## up. +## +## .. bro:see:: tcp_max_initial_window tcp_excessive_data_without_further_acks const tcp_max_above_hole_without_any_acks = 4096; ## If we've seen this much data without any of it being acked, we give up @@ -450,87 +736,160 @@ const tcp_max_above_hole_without_any_acks = 4096; ## stuff. If set to zero, then we don't ever give up. Ideally, Bro would ## track the current window on a connection and use it to infer that data ## has in fact gone too far, but for now we just make this quite beefy. +## +## .. bro:see:: tcp_max_initial_window tcp_max_above_hole_without_any_acks const tcp_excessive_data_without_further_acks = 10 * 1024 * 1024; -## For services without a handler, these sets define which -## side of a connection is to be reassembled. +## For services without an a handler, these sets define originator-side ports that +## still trigger reassembly. +## +## .. :bro:see:: tcp_reassembler_ports_resp const tcp_reassembler_ports_orig: set[port] = {} &redef; -## See :bro:id:`tcp_reassembler_ports_orig` + +## For services without an a handler, these sets define responder-side ports that +## still trigger reassembly. +## +## .. :bro:see:: tcp_reassembler_ports_orig const tcp_reassembler_ports_resp: set[port] = {} &redef; -## These sets define destination ports for which the contents -## of the originator (responder, respectively) stream should -## be delivered via tcp_contents. +## Defines destination TCP ports for which the contents of the originator stream +## should be delivered via :bro:see:`tcp_contents`. +## +## .. bro:see:: tcp_content_delivery_ports_resp tcp_content_deliver_all_orig +## tcp_content_deliver_all_resp udp_content_delivery_ports_orig +## udp_content_delivery_ports_resp udp_content_deliver_all_orig +## udp_content_deliver_all_resp tcp_contents const tcp_content_delivery_ports_orig: table[port] of bool = {} &redef; -## See :bro:id:`tcp_content_delivery_ports_orig` + +## Defines destination TCP ports for which the contents of the responder stream should +## be delivered via :bro:see:`tcp_contents`. +## +## .. bro:see:: tcp_content_delivery_ports_orig tcp_content_deliver_all_orig +## tcp_content_deliver_all_resp udp_content_delivery_ports_orig +## udp_content_delivery_ports_resp udp_content_deliver_all_orig +## udp_content_deliver_all_resp tcp_contents const tcp_content_delivery_ports_resp: table[port] of bool = {} &redef; -# To have all TCP orig->resp/resp->orig traffic reported via tcp_contents, -# redef these to T. +## If true, all TCP originator-side traffic is reported via +## :bro:see:`tcp_contents`. +## +## .. bro:see:: tcp_content_delivery_ports_orig tcp_content_delivery_ports_resp +## tcp_content_deliver_all_resp udp_content_delivery_ports_orig +## udp_content_delivery_ports_resp udp_content_deliver_all_orig +## udp_content_deliver_all_resp tcp_contents const tcp_content_deliver_all_orig = F &redef; -## See :bro:id:`tcp_content_deliver_all_orig` + +## If true, all TCP responder-side traffic is reported via +## :bro:see:`tcp_contents`. +## +## .. bro:see:: tcp_content_delivery_ports_orig +## tcp_content_delivery_ports_resp +## tcp_content_deliver_all_orig udp_content_delivery_ports_orig +## udp_content_delivery_ports_resp udp_content_deliver_all_orig +## udp_content_deliver_all_resp tcp_contents const tcp_content_deliver_all_resp = F &redef; -## These sets define destination ports for which the contents -## of the originator (responder, respectively) stream should -## be delivered via udp_contents. +## Defines UDP destination ports for which the contents of the originator stream +## should be delivered via :bro:see:`udp_contents`. +## +## .. bro:see:: tcp_content_delivery_ports_orig +## tcp_content_delivery_ports_resp +## tcp_content_deliver_all_orig tcp_content_deliver_all_resp +## udp_content_delivery_ports_resp udp_content_deliver_all_orig +## udp_content_deliver_all_resp udp_contents const udp_content_delivery_ports_orig: table[port] of bool = {} &redef; -## See :bro:id:`udp_content_delivery_ports_orig` + +## Defines UDP destination ports for which the contents of the originator stream +## should be delivered via :bro:see:`udp_contents`. +## +## .. bro:see:: tcp_content_delivery_ports_orig +## tcp_content_delivery_ports_resp tcp_content_deliver_all_orig +## tcp_content_deliver_all_resp udp_content_delivery_ports_orig +## udp_content_deliver_all_orig udp_content_deliver_all_resp udp_contents const udp_content_delivery_ports_resp: table[port] of bool = {} &redef; -## To have all UDP orig->resp/resp->orig traffic reported via udp_contents, -## redef these to T. +## If true, all UDP originator-side traffic is reported via +## :bro:see:`tcp_contents`. +## +## .. bro:see:: tcp_content_delivery_ports_orig +## tcp_content_delivery_ports_resp tcp_content_deliver_all_resp +## tcp_content_delivery_ports_orig udp_content_delivery_ports_orig +## udp_content_delivery_ports_resp udp_content_deliver_all_resp +## udp_contents const udp_content_deliver_all_orig = F &redef; -## See :bro:id:`udp_content_deliver_all_orig` + +## If true, all UDP responder-side traffic is reported via +## :bro:see:`tcp_contents`. +## +## .. bro:see:: tcp_content_delivery_ports_orig +## tcp_content_delivery_ports_resp tcp_content_deliver_all_resp +## tcp_content_delivery_ports_orig udp_content_delivery_ports_orig +## udp_content_delivery_ports_resp udp_content_deliver_all_orig +## udp_contents const udp_content_deliver_all_resp = F &redef; -## Check for expired table entries after this amount of time +## Check for expired table entries after this amount of time. +## +## .. bro:see:: table_incremental_step table_expire_delay const table_expire_interval = 10 secs &redef; -## When expiring/serializing, don't work on more than this many table -## entries at a time. +## When expiring/serializing table entries, don't work on more than this many table +## at a time. +## +## .. bro:see:: table_expire_interval table_expire_delay const table_incremental_step = 5000 &redef; -## When expiring, wait this amount of time before checking the next chunk -## of entries. +## When expiring table entries, wait this amount of time before checking the next +## chunk of entries. +## +## .. :bro:see:: table_expire_interval table_incremental_step const table_expire_delay = 0.01 secs &redef; ## Time to wait before timing out a DNS request. const dns_session_timeout = 10 sec &redef; -## Time to wait before timing out a NTP request. + +## Time to wait before timing out an NTP request. const ntp_session_timeout = 300 sec &redef; -## Time to wait before timing out a RPC request. + +## Time to wait before timing out an RPC request. const rpc_timeout = 24 sec &redef; -## Time window for reordering packets (to deal with timestamp -## discrepency between multiple packet sources). -const packet_sort_window = 0 usecs &redef; - -## How long to hold onto fragments for possible reassembly. A value -## of 0.0 means "forever", which resists evasion, but can lead to -## state accrual. +## How long to hold onto fragments for possible reassembly. A value of 0.0 means +## "forever", which resists evasion, but can lead to state accrual. const frag_timeout = 0.0 sec &redef; +## Time window for reordering packets. This is used for dealing with timestamp +## discrepency between multiple packet sources. +## +## .. note:: Setting this can have a major performance impact as now packets need +## to be potentially copied and buffered. +const packet_sort_window = 0 usecs &redef; + ## If positive, indicates the encapsulation header size that should -## be skipped over for each captured packet .... +## be skipped. This either applies to all packets, or if +## :bro:see:`tunnel_port` is set, only to packets on that port. +## +## .. :bro:see:: tunnel_port const encap_hdr_size = 0 &redef; -## ... or just for the following UDP port. + +## A UDP port that specifies which connections to apply :bro:see:`encap_hdr_size` +## to. +## +## .. :bro:see:: encap_hdr_size const tunnel_port = 0/udp &redef; -## Whether to use the ConnSize analyzer to count the number of -## packets and IP-level bytes transfered by each endpoint. If -## true, these values are returned in the connection's endpoint -## record val. +## Whether to use the ``ConnSize`` analyzer to count the number of packets and +## IP-level bytes transfered by each endpoint. If true, these values are returned +## in the connection's :bro:see:`endpoint` record value. const use_conn_size_analyzer = T &redef; -const UDP_INACTIVE = 0; -const UDP_ACTIVE = 1; # means we've seen something from this endpoint - -const ENDIAN_UNKNOWN = 0; -const ENDIAN_LITTLE = 1; -const ENDIAN_BIG = 2; -const ENDIAN_CONFUSED = 3; +# todo::these should go into an enum to make them autodoc'able. +const ENDIAN_UNKNOWN = 0; ##< Endian not yet determined. +const ENDIAN_LITTLE = 1; ##< Little endian. +const ENDIAN_BIG = 2; ##< Big endian. +const ENDIAN_CONFUSED = 3; ##< Tried to determine endian, but failed. +## Deprecated. function append_addl(c: connection, addl: string) { if ( c$addl == "" ) @@ -540,6 +899,7 @@ function append_addl(c: connection, addl: string) c$addl = fmt("%s %s", c$addl, addl); } +## Deprecated. function append_addl_marker(c: connection, addl: string, marker: string) { if ( c$addl == "" ) @@ -550,54 +910,65 @@ function append_addl_marker(c: connection, addl: string, marker: string) } -# Values for set_contents_file's "direction" argument. -# TODO: these should go into an enum to make them autodoc'able -const CONTENTS_NONE = 0; # turn off recording of contents -const CONTENTS_ORIG = 1; # record originator contents -const CONTENTS_RESP = 2; # record responder contents -const CONTENTS_BOTH = 3; # record both originator and responder contents - -const ICMP_UNREACH_NET = 0; -const ICMP_UNREACH_HOST = 1; -const ICMP_UNREACH_PROTOCOL = 2; -const ICMP_UNREACH_PORT = 3; -const ICMP_UNREACH_NEEDFRAG = 4; -const ICMP_UNREACH_ADMIN_PROHIB = 13; -# The above list isn't exhaustive ... +# Values for :bro:see:`set_contents_file` *direction* argument. +# todo::these should go into an enum to make them autodoc'able +const CONTENTS_NONE = 0; ##< Turn off recording of contents. +const CONTENTS_ORIG = 1; ##< Record originator contents. +const CONTENTS_RESP = 2; ##< Record responder contents. +const CONTENTS_BOTH = 3; ##< Record both originator and responder contents. +# Values for code of ICMP *unreachable* messages. The list is not exhaustive. +# todo::these should go into an enum to make them autodoc'able +# +# .. bro:see:: :bro:see:`icmp_unreachable ` +const ICMP_UNREACH_NET = 0; ##< Network unreachable. +const ICMP_UNREACH_HOST = 1; ##< Host unreachable. +const ICMP_UNREACH_PROTOCOL = 2; ##< Protocol unreachable. +const ICMP_UNREACH_PORT = 3; ##< Port unreachable. +const ICMP_UNREACH_NEEDFRAG = 4; ##< Fragement needed. +const ICMP_UNREACH_ADMIN_PROHIB = 13; ##< Adminstratively prohibited. # Definitions for access to packet headers. Currently only used for # discarders. -const IPPROTO_IP = 0; # dummy for IP -const IPPROTO_ICMP = 1; # control message protocol -const IPPROTO_IGMP = 2; # group mgmt protocol -const IPPROTO_IPIP = 4; # IP encapsulation in IP -const IPPROTO_TCP = 6; # TCP -const IPPROTO_UDP = 17; # user datagram protocol -const IPPROTO_RAW = 255; # raw IP packet +# todo::these should go into an enum to make them autodoc'able +const IPPROTO_IP = 0; ##< Dummy for IP. +const IPPROTO_ICMP = 1; ##< Control message protocol. +const IPPROTO_IGMP = 2; ##< Group management protocol. +const IPPROTO_IPIP = 4; ##< IP encapsulation in IP. +const IPPROTO_TCP = 6; ##< TCP. +const IPPROTO_UDP = 17; ##< User datagram protocol. +const IPPROTO_RAW = 255; ##< Raw IP packet. +## Values extracted from an IP header. +## +## .. bro:see:: pkt_hdr discarder_check_ip type ip_hdr: record { - hl: count; ##< header length (in bytes) - tos: count; ##< type of service - len: count; ##< total length - id: count; ##< identification - ttl: count; ##< time to live - p: count; ##< protocol - src: addr; ##< source address - dst: addr; ##< dest address + hl: count; ##< Header length in bytes. + tos: count; ##< Type of service. + len: count; ##< Total length. + id: count; ##< Identification. + ttl: count; ##< Time to live. + p: count; ##< Protocol. + src: addr; ##< Source address. + dst: addr; ##< Destination address. }; -## TCP flags. -const TH_FIN = 1; -const TH_SYN = 2; -const TH_RST = 4; -const TH_PUSH = 8; -const TH_ACK = 16; -const TH_URG = 32; -const TH_FLAGS = 63; ##< (TH_FIN|TH_SYN|TH_RST|TH_ACK|TH_URG) +# TCP flags. +# +# todo::these should go into an enum to make them autodoc'able +const TH_FIN = 1; ##< FIN. +const TH_SYN = 2; ##< SYN. +const TH_RST = 4; ##< RST. +const TH_PUSH = 8; ##< PUSH. +const TH_ACK = 16; ##< ACK. +const TH_URG = 32; ##< URG. +const TH_FLAGS = 63; ##< Mask combining all flags. +## Values extracted from a TCP header. +## +## .. bro:see:: pkt_hdr discarder_check_tcp type tcp_hdr: record { - sport: port; ##< source port + sport: port; ##< source port. dport: port; ##< destination port seq: count; ##< sequence number ack: count; ##< acknowledgement number @@ -607,36 +978,114 @@ type tcp_hdr: record { win: count; ##< window }; +## Values extracted from a UDP header. +## +## .. bro:see:: pkt_hdr discarder_check_udp type udp_hdr: record { sport: port; ##< source port dport: port; ##< destination port ulen: count; ##< udp length }; - -## Holds an ip_hdr and one of tcp_hdr, udp_hdr, or icmp_hdr. -type pkt_hdr: record { - ip: ip_hdr; - tcp: tcp_hdr &optional; - udp: udp_hdr &optional; - icmp: icmp_hdr &optional; +## Values extracted from an ICMP header. +## +## .. bro:see:: pkt_hdr discarder_check_icmp +type icmp_hdr: record { + icmp_type: count; ##< type of message }; +## A packet header, consisting of an IP header and transport-layer header. +## +## .. bro:see:: new_packet +type pkt_hdr: record { + ip: ip_hdr; ##< The IP header. + tcp: tcp_hdr &optional; ##< The TCP header if a TCP packet. + udp: udp_hdr &optional; ##< The UDP header if a UDP packet. + icmp: icmp_hdr &optional; ##< The ICMP header if an ICMP packet. +}; -## If you add elements here, then for a given BPF filter as index, when -## a packet matching that filter is captured, the corresponding event handler -## will be invoked. +## Definition of "secondary filters". A secondary filter is a BPF filter given as +## index in this table. For each such filter, the corresponding event is raised for +## all matching packets. global secondary_filters: table[string] of event(filter: string, pkt: pkt_hdr) &redef; -global discarder_maxlen = 128 &redef; ##< maximum amount of data passed to fnc +## Maximum length of payload passed to discarder functions. +## +## .. :bro:see:: discarder_check_tcp discarder_check_udp discarder_check_icmp +## discarder_check_ip +global discarder_maxlen = 128 &redef; +## Function for skipping packets based on their IP header. If defined, this +## function will be called for all IP packets before Bro performs any further +## analysis. If the function signals to discard a packet, no further processing +## will be performed on it. +## +## i: The IP header of the considered packet. +## +## Returns: True if the packet should not be analyzed any further. +## +## .. :bro:see:: discarder_check_tcp discarder_check_udp discarder_check_icmp +## discarder_maxlen +## +## .. note:: This is very low-level functionality and potentially expensive. +## Avoid using it. global discarder_check_ip: function(i: ip_hdr): bool; -global discarder_check_tcp: function(i: ip_hdr, t: tcp_hdr, d: string): bool; -global discarder_check_udp: function(i: ip_hdr, u: udp_hdr, d: string): bool; -global discarder_check_icmp: function(i: ip_hdr, ih: icmp_hdr): bool; -# End of definition of access to packet headers, discarders. +## Function for skipping packets based on their TCP header. If defined, this +## function will be called for all TCP packets before Bro performs any further +## analysis. If the function signals to discard a packet, no further processing +## will be performed on it. +## +## i: The IP header of the considered packet. +## t: The TCP header. +## d: Up to :bro:see:`discarder_maxlen` bytes of the TCP payload. +## +## Returns: True if the packet should not be analyzed any further. +## +## .. :bro:see:: discarder_check_ip discarder_check_udp discarder_check_icmp +## discarder_maxlen +## +## .. note:: This is very low-level functionality and potentially expensive. +## Avoid using it. +global discarder_check_tcp: function(i: ip_hdr, t: tcp_hdr, d: string): bool; + +## Function for skipping packets based on their UDP header. If defined, this +## function will be called for all UDP packets before Bro performs any further +## analysis. If the function signals to discard a packet, no further processing +## will be performed on it. +## +## i: The IP header of the considered packet. +## t: The UDP header. +## d: Up to :bro:see:`discarder_maxlen` bytes of the UDP payload. +## +## Returns: True if the packet should not be analyzed any further. +## +## .. :bro:see:: discarder_check_ip discarder_check_tcp discarder_check_icmp +## discarder_maxlen +## +## .. note:: This is very low-level functionality and potentially expensive. +## Avoid using it. +global discarder_check_udp: function(i: ip_hdr, u: udp_hdr, d: string): bool; + +## Function for skipping packets based on their ICMP header. If defined, this +## function will be called for all ICMP packets before Bro performs any further +## analysis. If the function signals to discard a packet, no further processing +## will be performed on it. +## +## i: The IP header of the considered packet. +## ih: The ICMP header. +## +## Returns: True if the packet should not be analyzed any further. +## +## .. :bro:see:: discarder_check_ip discarder_check_tcp discarder_check_udp +## discarder_maxlen +## +## .. note:: This is very low-level functionality and potentially expensive. +## Avoid using it. +global discarder_check_icmp: function(i: ip_hdr, ih: icmp_hdr): bool; + +## Bro's watchdog interval. const watchdog_interval = 10 sec &redef; ## The maximum number of timers to expire after processing each new @@ -650,56 +1099,141 @@ const max_timer_expires = 300 &redef; const max_remote_events_processed = 10 &redef; # These need to match the definitions in Login.h. -# TODO: use enum to make them autodoc'able -const LOGIN_STATE_AUTHENTICATE = 0; # trying to authenticate -const LOGIN_STATE_LOGGED_IN = 1; # successful authentication -const LOGIN_STATE_SKIP = 2; # skip any further processing -const LOGIN_STATE_CONFUSED = 3; # we're confused +# +# .. bro:see:: get_login_state +# +# todo::use enum to make them autodoc'able +const LOGIN_STATE_AUTHENTICATE = 0; # Trying to authenticate. +const LOGIN_STATE_LOGGED_IN = 1; # Successful authentication. +const LOGIN_STATE_SKIP = 2; # Skip any further processing. +const LOGIN_STATE_CONFUSED = 3; # We're confused. # It would be nice to replace these function definitions with some # form of parameterized types. + +## Returns minimum of two ``double`` values. +## +## a: First value. +## b: Second value. +## +## Returns: The minimum of *a* and *b*. function min_double(a: double, b: double): double { return a < b ? a : b; } + +## Returns maximum of two ``double`` values. +## +## a: First value. +## b: Second value. +## +## Returns: The maximum of *a* and *b*. function max_double(a: double, b: double): double { return a > b ? a : b; } + +## Returns minimum of two ``interval`` values. +## +## a: First value. +## b: Second value. +## +## Returns: The minimum of *a* and *b*. function min_interval(a: interval, b: interval): interval { return a < b ? a : b; } + +## Returns maximum of two ``interval`` values. +## +## a: First value. +## b: Second value. +## +## Returns: The maximum of *a* and *b*. function max_interval(a: interval, b: interval): interval { return a > b ? a : b; } + +## Returns minimum of two ``count`` values. +## +## a: First value. +## b: Second value. +## +## Returns: The minimum of *a* and *b*. function min_count(a: count, b: count): count { return a < b ? a : b; } + +## Returns maximum of two ``count`` values. +## +## a: First value. +## b: Second value. +## +## Returns: The maximum of *a* and *b*. function max_count(a: count, b: count): count { return a > b ? a : b; } +## TODO. global skip_authentication: set[string] &redef; + +## TODO. global direct_login_prompts: set[string] &redef; + +## TODO. global login_prompts: set[string] &redef; + +## TODO. global login_non_failure_msgs: set[string] &redef; + +## TODO. global login_failure_msgs: set[string] &redef; + +## TODO. global login_success_msgs: set[string] &redef; + +## TODO. global login_timeouts: set[string] &redef; +## A MIME header key/value pair. +## +## .. bro:see:: mime_header_list http_all_headers mime_all_headers mime_one_header type mime_header_rec: record { - name: string; - value: string; + name: string; ##< The header name. + value: string; ##< The header value. }; + +## A list of MIME headers. +## +## .. bro:see:: mime_header_rec http_all_headers mime_all_headers type mime_header_list: table[count] of mime_header_rec; + +## The length of MIME data segments delivered to handlers of +## :bro:see:`mime_segment_data`. +## +## .. bro:see:: mime_segment_data mime_segment_overlap_length global mime_segment_length = 1024 &redef; + +## The number of bytes of overlap between successive segments passed to +## :bro:see:`mime_segment_data`. global mime_segment_overlap_length = 0 &redef; +## An RPC portmapper mapping. +## +## .. bro:see:: pm_mappings type pm_mapping: record { - program: count; - version: count; - p: port; + program: count; ##< The RPC program. + version: count; ##< The program version. + p: port; ##< The port. }; +## Table of RPC portmapper mappings. +## +## .. bro:see:: pm_request_dump type pm_mappings: table[count] of pm_mapping; +## An RPC portmapper request. +## +## .. bro:see:: pm_attempt_getport pm_request_getport type pm_port_request: record { - program: count; - version: count; - is_tcp: bool; + program: count; ##< The RPC program. + version: count; ##< The program version. + is_tcp: bool; ##< True if using TCP. }; +## An RPC portmapper *callit* request. +## +## .. bro:see:: pm_attempt_callit pm_request_callit type pm_callit_request: record { - program: count; - version: count; - proc: count; - arg_size: count; + program: count; ##< The RPC program. + version: count; ##< The program version. + proc: count; ##< The procedure being called. + arg_size: count; ##< The size of the argument. }; # See const.bif @@ -713,6 +1247,10 @@ type pm_callit_request: record { # const RPC_AUTH_ERROR = 7; # const RPC_UNKNOWN_ERROR = 8; +## Mapping of numerical RPC status codes to readable messages. +## +## .. bro:see:: pm_attempt_callit pm_attempt_dump pm_attempt_getport +## pm_attempt_null pm_attempt_set pm_attempt_unset rpc_dialogue rpc_reply const RPC_status = { [RPC_SUCCESS] = "ok", [RPC_PROG_UNAVAIL] = "prog unavail", @@ -728,247 +1266,307 @@ const RPC_status = { module NFS3; export { - ## Should the read and write events return the file data that has been - ## read/written? + ## If true, :bro:see:`nfs_proc_read` and :bro:see:`nfs_proc_write` events return + ## the file data that has been read/written. + ## + ## .. .. bro:see:: return_data_max return_data_first_only const return_data = F &redef; - ## If bro:id:`nfs_return_data` is true, how much data should be returned at most. + ## If bro:id:`NFS3::return_data` is true, how much data should be returned at + ## most. const return_data_max = 512 &redef; - ## If nfs_return_data is true, whether to *only* return data if the read or write - ## offset is 0, i.e., only return data for the beginning of the file. + ## If bro:id:`NFS3::return_data` is true, whether to *only* return data if the read + ## or write offset is 0, i.e., only return data for the beginning of the file. const return_data_first_only = T &redef; - ## This record summarizes the general results and status of NFSv3 request/reply - ## pairs. It's part of every NFSv3 event. + ## Record summarizing the general results and status of NFSv3 request/reply pairs. + ## + ## Note that when *rpc_stats* or *nfs_stats* indicates not successful, the reply + ## record passed to the correpsonding event will be empty and contain uninitialized + ## fields, so don't use it. Also note that time and duration values might not be + ## fully accurate. For TCP, we record times when the corresponding chunk of data + ## is delivered to the analyzer. Depending on the reassembler, this might be well + ## after the first packet of the request was received. + ## + ## .. bro:see:: nfs_proc_create nfs_proc_getattr nfs_proc_lookup + ## nfs_proc_mkdir nfs_proc_not_implemented nfs_proc_null + ## nfs_proc_read nfs_proc_readdir nfs_proc_readlink nfs_proc_remove + ## nfs_proc_rmdir nfs_proc_write nfs_reply_status type info_t: record { - ## If this indicates not successful, the reply record in the - ## events will be empty and contain uninitialized fields, so - ## don't use it. - rpc_stat: rpc_status; + ## The RPC status. + rpc_stat: rpc_status; + ## The NFS status. nfs_stat: status_t; - - ## The start time, duration, and length in bytes of the request (call). Note that - ## the start and end time might not be accurate. For TCP, we record the - ## time when a chunk of data is delivered to the analyzer. Depending on the - ## Reassembler, this might be well after the first packet of the request - ## was received. + ## The start time of the request. req_start: time; - ## See :bro:id:`req_start` + ## The duration of the request. req_dur: interval; - ## See :bro:id:`req_start` + ## The length in bytes of the request. req_len: count; - - ## Like :bro:id:`req_start` but for reply. + ## The start time of the reply. rep_start: time; - ## Like :bro:id:`req_dur` but for reply. + ## The duration of the reply. rep_dur: interval; - ## Like :bro:id:`req_len` but for reply. + ## The length in bytes of the reply. rep_len: count; }; - # NFSv3 types. Type names are based on RFC 1813. + ## NFS file attributes. Field names are based on RFC 1813. + ## + ## .. bro:see:: nfs_proc_getattr type fattr_t: record { - ftype: file_type_t; - mode: count; - nlink: count; - uid: count; - gid: count; - size: count; - used: count; - rdev1: count; - rdev2: count; - fsid: count; - fileid: count; - atime: time; - mtime: time; - ctime: time; + ftype: file_type_t; ##< File type. + mode: count; ##< Mode + nlink: count; ##< Number of links. + uid: count; ##< User ID. + gid: count; ##< Group ID. + size: count; ##< Size. + used: count; ##< TODO. + rdev1: count; ##< TODO. + rdev2: count; ##< TODO. + fsid: count; ##< TODO. + fileid: count; ##< TODO. + atime: time; ##< Time of last access. + mtime: time; ##< Time of last modification. + ctime: time; ##< Time of creation. }; + ## NFS *readdir* arguments. + ## + ## .. bro:see:: nfs_proc_readdir type diropargs_t : record { - dirfh: string; ##< the file handle of the directory - fname: string; ##< the name of the file we are interested in + dirfh: string; ##< The file handle of the directory. + fname: string; ##< The name of the file we are interested in. }; - # Note, we don't need a "post_op_attr" type. We use an "fattr_t &optional" - # instead. - - ## If the lookup failed, dir_attr may be set. - ## If the lookup succeeded, fh is always set and obj_attr and dir_attr may be set. + ## NFS lookup reply. If the lookup failed, *dir_attr* may be set. If the lookup + ## succeeded, *fh* is always set and *obj_attr* and *dir_attr* may be set. + ## + ## .. bro:see:: nfs_proc_lookup type lookup_reply_t: record { - fh: string &optional; ##< file handle of object looked up - obj_attr: fattr_t &optional; ##< optional attributes associated w/ file - dir_attr: fattr_t &optional; ##< optional attributes associated w/ dir. + fh: string &optional; ##< File handle of object looked up. + obj_attr: fattr_t &optional; ##< Optional attributes associated w/ file + dir_attr: fattr_t &optional; ##< Optional attributes associated w/ dir. }; + ## NFS *read* arguments. + ## + ## .. bro:see:: nfs_proc_read type readargs_t: record { - fh: string; ##< file handle to read from - offset: count; ##< offset in file - size: count; ##< number of bytes to read + fh: string; ##< File handle to read from. + offset: count; ##< Offset in file. + size: count; ##< Number of bytes to read. }; - ## If the lookup fails, attr may be set. If the lookup succeeds, attr may be set - ## and all other fields are set. + ## NFS *read* reply. If the lookup fails, *attr* may be set. If the lookup succeeds, + ## *attr* may be set and all other fields are set. type read_reply_t: record { - attr: fattr_t &optional; ##< attributes - size: count &optional; ##< number of bytes read - eof: bool &optional; ##< did the read end at EOF - data: string &optional; ##< the actual data; not yet implemented. + attr: fattr_t &optional; ##< Attributes. + size: count &optional; ##< Number of bytes read. + eof: bool &optional; ##< Sid the read end at EOF. + data: string &optional; ##< The actual data; not yet implemented. }; - ## If the request fails, attr may be set. If the request succeeds, attr may be - ## set and all other fields are set. + ## NFS *readline* reply. If the request fails, *attr* may be set. If the request + ## succeeds, *attr* may be set and all other fields are set. + ## + ## .. bro:see:: nfs_proc_readlink type readlink_reply_t: record { - attr: fattr_t &optional; ##< attributes - nfspath: string &optional; ##< the contents of the symlink; in general a pathname as text + attr: fattr_t &optional; ##< Attributes. + nfspath: string &optional; ##< Contents of the symlink; in general a pathname as text. }; + ## NFS *write* arguments. + ## + ## .. bro:see:: nfs_proc_write type writeargs_t: record { - fh: string; ##< file handle to write to - offset: count; ##< offset in file - size: count; ##< number of bytes to write - stable: stable_how_t; ##< how and when data is commited - data: string &optional; ##< the actual data; not implemented yet + fh: string; ##< File handle to write to. + offset: count; ##< Offset in file. + size: count; ##< Number of bytes to write. + stable: stable_how_t; ##< How and when data is commited. + data: string &optional; ##< The actual data; not implemented yet. }; + ## NFS *wcc* attributes. + ## + ## .. bro:see:: NFS3::write_reply_t type wcc_attr_t: record { - size: count; - atime: time; - mtime: time; + size: count; ##< The dize. + atime: time; ##< Access time. + mtime: time; ##< Modification time. }; - ## If the request fails, pre|post attr may be set. If the request succeeds, - ## pre|post attr may be set and all other fields are set. + ## NFS *write* reply. If the request fails, *pre|post* attr may be set. If the + ## request succeeds, *pre|post* attr may be set and all other fields are set. + ## + ## .. bro:see:: nfs_proc_write type write_reply_t: record { - preattr: wcc_attr_t &optional; ##< pre operation attributes - postattr: fattr_t &optional; ##< post operation attributes - size: count &optional; - commited: stable_how_t &optional; - verf: count &optional; ##< write verifier cookue + preattr: wcc_attr_t &optional; ##< Pre operation attributes. + postattr: fattr_t &optional; ##< Post operation attributes. + size: count &optional; ##< Size. + commited: stable_how_t &optional; ##< TODO. + verf: count &optional; ##< Write verifier cookie. }; - ## reply for create, mkdir, symlink - ## If the proc failed, dir_*_attr may be set. If the proc succeeded, fh and - ## the attr's may be set. Note: no guarantee that fh is set after - ## success. + ## NFS reply for *create*, *mkdir*, and *symlink*. If the proc + ## failed, *dir_\*_attr* may be set. If the proc succeeded, *fh* and the *attr*'s + ## may be set. Note: no guarantee that *fh* is set after success. + ## + ## .. bro:see:: nfs_proc_create nfs_proc_mkdir type newobj_reply_t: record { - fh: string &optional; ##< file handle of object created - obj_attr: fattr_t &optional; ##< optional attributes associated w/ new object - dir_pre_attr: wcc_attr_t &optional; ##< optional attributes associated w/ dir - dir_post_attr: fattr_t &optional; ##< optional attributes associated w/ dir + fh: string &optional; ##< File handle of object created. + obj_attr: fattr_t &optional; ##< Optional attributes associated w/ new object. + dir_pre_attr: wcc_attr_t &optional; ##< Optional attributes associated w/ dir. + dir_post_attr: fattr_t &optional; ##< Optional attributes associated w/ dir. }; - ## reply for remove, rmdir - ## Corresponds to "wcc_data" in the spec. + ## NFS reply for *remove*, *rmdir*. Corresponds to *wcc_data* in the spec. + ## + ## .. bro:see:: nfs_proc_remove nfs_proc_rmdir type delobj_reply_t: record { - dir_pre_attr: wcc_attr_t &optional; ##< optional attributes associated w/ dir - dir_post_attr: fattr_t &optional; ##< optional attributes associated w/ dir + dir_pre_attr: wcc_attr_t &optional; ##< Optional attributes associated w/ dir. + dir_post_attr: fattr_t &optional; ##< Optional attributes associated w/ dir. }; - ## This record is used for both readdir and readdirplus. + ## NFS *readdir* arguments. Used for both *readdir* and *readdirplus*. + ## + ## .. bro:see:: nfs_proc_readdir type readdirargs_t: record { - isplus: bool; ##< is this a readdirplus request? - dirfh: string; ##< the directory filehandle - cookie: count; ##< cookie / pos in dir; 0 for first call - cookieverf: count; ##< the cookie verifier - dircount: count; ##< "count" field for readdir; maxcount otherwise (in bytes) - maxcount: count &optional; ##< only used for readdirplus. in bytes + isplus: bool; ##< Is this a readdirplus request? + dirfh: string; ##< The directory filehandle. + cookie: count; ##< Cookie / pos in dir; 0 for first call. + cookieverf: count; ##< The cookie verifier. + dircount: count; ##< "count" field for readdir; maxcount otherwise (in bytes). + maxcount: count &optional; ##< Only used for readdirplus. in bytes. }; - ## fh and attr are used for readdirplus. However, even for readdirplus they may - ## not be filled out. + ## NFS *direntry*. *fh* and *attr* are used for *readdirplus*. However, even + ## for *readdirplus* they may not be filled out. + ## + ## .. bro:see:: NFS3::direntry_vec_t NFS3::readdir_reply_t type direntry_t: record { - fileid: count; ##< e.g., inode number - fname: string; ##< filename - cookie: count; - attr: fattr_t &optional; ##< readdirplus: the FH attributes for the entry - fh: string &optional; ##< readdirplus: the FH for the entry + fileid: count; ##< E.g., inode number. + fname: string; ##< Filename. + cookie: count; ##< Cookie value. + attr: fattr_t &optional; ##< *readdirplus*: the *fh* attributes for the entry. + fh: string &optional; ##< *readdirplus*: the *fh* for the entry }; + ## Vector of NFS *direntry*. + ## + ## .. bro:see:: NFS3::readdir_reply_t type direntry_vec_t: vector of direntry_t; - ## Used for readdir and readdirplus. - ## If error: dir_attr might be set. If success: dir_attr may be set, all others + ## NFS *readdir* reply. Used for *readdir* and *readdirplus*. If an is + ## returned, *dir_attr* might be set. On success, *dir_attr* may be set, all others ## must be set. type readdir_reply_t: record { - isplus: bool; ##< is the reply for a readdirplus request - dir_attr: fattr_t &optional; - cookieverf: count &optional; - entries: direntry_vec_t &optional; - eof: bool; ##< if true, no more entries in dir. + isplus: bool; ##< True if the reply for a *readdirplus* request. + dir_attr: fattr_t &optional; ##< Directory attributes. + cookieverf: count &optional; ##< TODO. + entries: direntry_vec_t &optional; ##< Returned directory entries. + eof: bool; ##< If true, no more entries in directory. }; + ## NFS *fsstat*. type fsstat_t: record { - attrs: fattr_t &optional; - tbytes: double; - fbytes: double; - abytes: double; - tfiles: double; - ffiles: double; - afiles: double; - invarsec: interval; + attrs: fattr_t &optional; ##< Attributes. + tbytes: double; ##< TODO. + fbytes: double; ##< TODO. + abytes: double; ##< TODO. + tfiles: double; ##< TODO. + ffiles: double; ##< TODO. + afiles: double; ##< TODO. + invarsec: interval; ##< TODO. }; } # end export module GLOBAL; +## An NTP message. +## +## .. bro:see:: ntp_message type ntp_msg: record { - id: count; - code: count; - stratum: count; - poll: count; - precision: int; - distance: interval; - dispersion: interval; - ref_t: time; - originate_t: time; - receive_t: time; - xmit_t: time; + id: count; ##< Message ID. + code: count; ##< Message code. + stratum: count; ##< Stratum. + poll: count; ##< Poll. + precision: int; ##< Precision. + distance: interval; ##< Distance. + dispersion: interval; ##< Dispersion. + ref_t: time; ##< Reference time. + originate_t: time; ##< Originating time. + receive_t: time; ##< Receive time. + xmit_t: time; ##< Send time. }; -## Maps Samba command numbers to descriptive names. +## Maps SMB command numbers to descriptive names. global samba_cmds: table[count] of string &redef &default = function(c: count): string { return fmt("samba-unknown-%d", c); }; +## An SMB command header. +## +## .. bro:see:: smb_com_close smb_com_generic_andx smb_com_logoff_andx +## smb_com_negotiate smb_com_negotiate_response smb_com_nt_create_andx +## smb_com_read_andx smb_com_setup_andx smb_com_trans_mailslot +## smb_com_trans_pipe smb_com_trans_rap smb_com_transaction +## smb_com_transaction2 smb_com_tree_connect_andx smb_com_tree_disconnect +## smb_com_write_andx smb_error smb_get_dfs_referral smb_message type smb_hdr : record { - command: count; - status: count; - flags: count; - flags2: count; - tid: count; - pid: count; - uid: count; - mid: count; + command: count; ##< The command number (see :bro:see:`samba_cmds` ). + status: count; ##< The status code. + flags: count; ##< Flag set 1. + flags2: count; ##< Flag set 2. + tid: count; ##< TODO. + pid: count; ##< Process ID. + uid: count; ##< User ID. + mid: count; ##< TODO. }; +## An SMB transaction. +## +## .. bro:see:: smb_com_trans_mailslot smb_com_trans_pipe smb_com_trans_rap +## smb_com_transaction smb_com_transaction2 type smb_trans : record { - word_count: count; - total_param_count: count; - total_data_count: count; - max_param_count: count; - max_data_count: count; - max_setup_count: count; + word_count: count; ##< TODO. + total_param_count: count; ##< TODO. + total_data_count: count; ##< TODO. + max_param_count: count; ##< TODO. + max_data_count: count; ##< TODO. + max_setup_count: count; ##< TODO. # flags: count; # timeout: count; - param_count: count; - param_offset: count; - data_count: count; - data_offset: count; - setup_count: count; - setup0: count; - setup1: count; - setup2: count; - setup3: count; - byte_count: count; - parameters: string; + param_count: count; ##< TODO. + param_offset: count; ##< TODO. + data_count: count; ##< TODO. + data_offset: count; ##< TODO. + setup_count: count; ##< TODO. + setup0: count; ##< TODO. + setup1: count; ##< TODO. + setup2: count; ##< TODO. + setup3: count; ##< TODO. + byte_count: count; ##< TODO. + parameters: string; ##< TODO. }; + +## SMB transaction data. +## +## .. bro:see:: smb_com_trans_mailslot smb_com_trans_pipe smb_com_trans_rap +## smb_com_transaction smb_com_transaction2 +## +## .. todo:: Should this really be a record type? type smb_trans_data : record { - data : string; + data : string; ##< The transaction's data. }; +## Deprecated. +## +## .. todo:: Remove. It's still declared internally but doesn't seem used anywhere +## else. type smb_tree_connect : record { flags: count; password: string; @@ -976,177 +1574,268 @@ type smb_tree_connect : record { service: string; }; +## Deprecated. +## +## .. todo:: Remove. It's still declared internally but doesn't seem used anywhere +## else. type smb_negotiate : table[count] of string; -## A list of router addresses offered by the server. +## A list of router addresses offered by a DHCP server. +## +## .. bro:see:: dhcp_ack dhcp_offer type dhcp_router_list: table[count] of addr; +## A DHCP message. +## +## .. bro:see:: dhcp_ack dhcp_decline dhcp_discover dhcp_inform dhcp_nak +## dhcp_offer dhcp_release dhcp_request type dhcp_msg: record { - op: count; ##< message OP code. 1 = BOOTREQUEST, 2 = BOOTREPLY - m_type: count; ##< the type of DHCP message - xid: count; ##< transaction ID of a DHCP session - h_addr: string; ##< hardware address of the client - ciaddr: addr; ##< original IP address of the client - yiaddr: addr; ##< IP address assigned to the client + op: count; ##< Message OP code. 1 = BOOTREQUEST, 2 = BOOTREPLY + m_type: count; ##< The type of DHCP message. + xid: count; ##< Transaction ID of a DHCP session. + h_addr: string; ##< Hardware address of the client. + ciaddr: addr; ##< Original IP address of the client. + yiaddr: addr; ##< IP address assigned to the client. }; +## A DNS message. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl +## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end dns_message +## dns_query_reply dns_rejected dns_request type dns_msg: record { - id: count; + id: count; ##< Transaction ID. - opcode: count; - rcode: count; + opcode: count; ##< Operation code. + rcode: count; ##< Return code. - QR: bool; - AA: bool; - TC: bool; - RD: bool; - RA: bool; - Z: count; + QR: bool; ##< Query response flag. + AA: bool; ##< Authoritative answer flag. + TC: bool; ##< Truncated packet flag. + RD: bool; ##< Recursion desired flag. + RA: bool; ##< Recursion available flag. + Z: count; ##< TODO. - num_queries: count; - num_answers: count; - num_auth: count; - num_addl: count; + num_queries: count; ##< Number of query records. + num_answers: count; ##< Number of answer records. + num_auth: count; ##< Number of authoritative records. + num_addl: count; ##< Number of additional records. }; +## A DNS SOA record. +## +## .. bro:see:: dns_SOA_reply type dns_soa: record { - mname: string; ##< primary source of data for zone - rname: string; ##< mailbox for responsible person - serial: count; ##< version number of zone - refresh: interval; ##< seconds before refreshing - retry: interval; ##< how long before retrying failed refresh - expire: interval; ##< when zone no longer authoritative - minimum: interval; ##< minimum TTL to use when exporting + mname: string; ##< Primary source of data for zone. + rname: string; ##< Mailbox for responsible person. + serial: count; ##< Version number of zone. + refresh: interval; ##< Seconds before refreshing. + retry: interval; ##< How long before retrying failed refresh. + expire: interval; ##< When zone no longer authoritative. + minimum: interval; ##< Minimum TTL to use when exporting. }; +## An additional DNS EDNS record. +## +## .. bro:see:: dns_EDNS_addl type dns_edns_additional: record { - query: string; - qtype: count; - t: count; - payload_size: count; - extended_rcode: count; - version: count; - z_field: count; - TTL: interval; - is_query: count; + query: string; ##< Query. + qtype: count; ##< Query type. + t: count; ##< TODO. + payload_size: count; ##< TODO. + extended_rcode: count; ##< Extended return code. + version: count; ##< Version. + z_field: count; ##< TODO. + TTL: interval; ##< Time-to-live. + is_query: count; ##< TODO. }; +## An additional DNS TSIG record. +## +## bro:see:: dns_TSIG_addl type dns_tsig_additional: record { - query: string; - qtype: count; - alg_name: string; - sig: string; - time_signed: time; - fudge: time; - orig_id: count; - rr_error: count; - is_query: count; + query: string; ##< Query. + qtype: count; ##< Query type. + alg_name: string; ##< Algorithm name. + sig: string; ##< Signature. + time_signed: time; ##< Time when signed. + fudge: time; ##< TODO. + orig_id: count; ##< TODO. + rr_error: count; ##< TODO. + is_query: count; ##< TODO. }; -# Different values for "answer_type" in the following. DNS_QUERY -# shouldn't occur, it's just for completeness. -# TODO: use enums to help autodoc -const DNS_QUERY = 0; -const DNS_ANS = 1; -const DNS_AUTH = 2; -const DNS_ADDL = 3; +# DNS answer types. +# +# .. .. bro:see:: dns_answerr +# +# todo::use enum to make them autodoc'able +const DNS_QUERY = 0; ##< A query. This shouldn't occur, just for completeness. +const DNS_ANS = 1; ##< An answer record. +const DNS_AUTH = 2; ##< An authorative record. +const DNS_ADDL = 3; ##< An additional record. +## The general part of a DNS reply. +## +## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_HINFO_reply +## dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply +## dns_TXT_reply dns_WKS_reply type dns_answer: record { + ## Answer type. One of :bro:see:`DNS_QUERY`, :bro:see:`DNS_ANS`, + ## :bro:see:`DNS_AUTH` and :bro:see:`DNS_ADDL`. answer_type: count; - query: string; - qtype: count; - qclass: count; - TTL: interval; + query: string; ##< Query. + qtype: count; ##< Query type. + qclass: count; ##< Query class. + TTL: interval; ##< Time-to-live. }; -## For servers in these sets, omit processing the AUTH records -## they include in their replies. +## For DNS servers in these sets, omit processing the AUTH records they include in +## their replies. +## +## .. bro:see:: dns_skip_all_auth dns_skip_addl global dns_skip_auth: set[addr] &redef; -## For servers in these sets, omit processing the ADDL records -## they include in their replies. + +## For DNS servers in these sets, omit processing the ADDL records they include in +## their replies. +## +## .. bro:see:: dns_skip_all_addl dns_skip_auth global dns_skip_addl: set[addr] &redef; -## If the following are true, then all AUTH records are skipped. +## If true, all DNS AUTH records are skipped. +## +## .. bro:see:: dns_skip_all_addl dns_skip_auth global dns_skip_all_auth = T &redef; -## If the following are true, then all ADDL records are skipped. + +## If true, all DNS ADDL records are skipped. +## +## .. bro:see:: dns_skip_all_auth dns_skip_addl global dns_skip_all_addl = T &redef; -## If a DNS request includes more than this many queries, assume it's -## non-DNS traffic and do not process it. Set to 0 to turn off this -## functionality. +## If a DNS request includes more than this many queries, assume it's non-DNS +## traffic and do not process it. Set to 0 to turn off this functionality. global dns_max_queries = 5; -## The maxiumum size in bytes for an SSL cipherspec. If we see a packet that -## has bigger cipherspecs, we won't do a comparisons of cipherspecs. +## The maxiumum size in bytes for an SSL cipher specifcation. If we see a packet +## that has bigger cipherspecs, we won't do a comparisons of cipherspecs. const ssl_max_cipherspec_size = 68 &redef; -type X509_extensions: table[count] of string; +# todo::Is this still used? +# type X509_extensions: table[count] of string; +## An X509 certificate. +## +## .. bro:see:: x509_certificate type X509: record { - version: count; - serial: string; - subject: string; - issuer: string; - not_valid_before: time; - not_valid_after: time; + version: count; ##< Version number. + serial: string; ##< Serial number. + subject: string; ##< Subject. + issuer: string; ##< Issuer. + not_valid_before: time; ##< Timestamp before when certificate is not valid. + not_valid_after: time; ##< Timestamp after when certificate is not valid. }; ## This is indexed with the CA's name and yields a DER (binary) encoded certificate. -const root_ca_certs: table[string] of string = {} &redef; +## todo::Is this still used? +# const root_ca_certs: table[string] of string = {} &redef; +## HTTP session statistics. +## +## .. bro:see:: http_stats type http_stats_rec: record { - num_requests: count; - num_replies: count; - request_version: double; - reply_version: double; + num_requests: count; ##< Number of requests. + num_replies: count; ##< Number of replies. + request_version: double; ##< HTTP version of the requests. + reply_version: double; ##< HTTP Version of the replies. }; +## HTTP message statistics. +## +## .. bro:see:: http_message_done type http_message_stat: record { - ## when the request/reply line was complete + ## When the request/reply line was complete. start: time; - ## whether the message is interrupted - interrupted: bool; - ## reason phrase if interrupted - finish_msg: string; - ## length of body processed (before finished/interrupted) - body_length: count; - ## total len of gaps within body_length - content_gap_length: count; - ## length of headers (including the req/reply line, but not CR/LF's) - header_length: count; + ## Whether the message was interrupted. + interrupted: bool; + ## Reason phrase if interrupted. + finish_msg: string; + ## Length of body processed (before finished/interrupted). + body_length: count; + ## Total length of gaps within body_length. + content_gap_length: count; + ## Length of headers (including the req/reply line, but not CR/LF's). + header_length: count; }; +## Maximum number of HTTP entity data delivered to events. The amount of data +## can be limited for better performance, zero disables truncation. +## +## .. bro:see:: http_entity_data skip_http_entity_data skip_http_data global http_entity_data_delivery_size = 1500 &redef; -## Truncate URIs longer than this to prevent over-long URIs (usually sent -## by worms) from slowing down event processing. A value of -1 means "do -## not truncate". +## Skip HTTP data for performance considerations. The skipped +## portion will not go through TCP reassembly. +## +## .. bro:see:: http_entity_data skip_http_entity_data http_entity_data_delivery_size +const skip_http_data = F &redef; + +## Maximum length of HTTP URIs passed to events. Longer ones will be truncated +## to prevent over-long URIs (usually sent by worms) from slowing down event +## processing. A value of -1 means "do not truncate". +## +## .. bro:see:: http_request const truncate_http_URI = -1 &redef; -## IRC-related globals to which the event engine is sensitive. +## IRC join information. +## +## .. bro:see:: irc_join_list type irc_join_info: record { nick: string; channel: string; password: string; usermode: string; }; + +## Set of IRC join information. +## +## .. bro:see:: irc_join_message type irc_join_list: set[irc_join_info]; + +## Deprecated. +## +## .. todo:: Remove. It's still declared internally but doesn't seem used anywhere +## else. global irc_servers : set[addr] &redef; -## Stepping-stone globals. +## Internal to the stepping stone detector. const stp_delta: interval &redef; + +## Internal to the stepping stone detector. const stp_idle_min: interval &redef; -## Don't do analysis on these sources. Used to avoid overload from scanners. +## Internal to the stepping stone detector. global stp_skip_src: set[addr] &redef; +## Deprecated. const interconn_min_interarrival: interval &redef; + +## Deprecated. const interconn_max_interarrival: interval &redef; + +## Deprecated. const interconn_max_keystroke_pkt_size: count &redef; + +## Deprecated. const interconn_default_pkt_size: count &redef; + +## Deprecated. const interconn_stat_period: interval &redef; + +## Deprecated. const interconn_stat_backoff: double &redef; +## Deprecated. type interconn_endp_stats: record { num_pkts: count; num_keystrokes_two_in_row: count; @@ -1160,9 +1849,13 @@ type interconn_endp_stats: record { num_normal_lines: count; }; +## Deprecated. const backdoor_stat_period: interval &redef; + +## Deprecated. const backdoor_stat_backoff: double &redef; +## Deprecated. type backdoor_endp_stats: record { is_partial: bool; num_pkts: count; @@ -1174,295 +1867,422 @@ type backdoor_endp_stats: record { num_7bit_ascii: count; }; +## Description of a signature match. +## +## .. bro:see:: signature_match type signature_state: record { - sig_id: string; ##< ID of the signature - conn: connection; ##< Current connection - is_orig: bool; ##< True if current endpoint is originator - payload_size: count; ##< Payload size of the first pkt of curr. endpoint - + sig_id: string; ##< ID of the matching signature. + conn: connection; ##< Matching connection. + is_orig: bool; ##< True if matching endpoint is originator. + payload_size: count; ##< Payload size of the first matching packet of current endpoint. }; -# This type is no longer used -# TODO: remove any use of this from the core. +# Deprecated. +# +# .. todo:: This type is no longer used. Remove any reference of this from the +# core. type software_version: record { - major: int; # Major version number - minor: int; # Minor version number - minor2: int; # Minor subversion number - addl: string; # Additional version string (e.g. "beta42") + major: int; + minor: int; + minor2: int; + addl: string; }; -# This type is no longer used -# TODO: remove any use of this from the core. +# Deprecated. +# +# .. todo:: This type is no longer used. Remove any reference of this from the +# core. type software: record { - name: string; # Unique name of a software, e.g., "OS" + name: string; version: software_version; }; -# The following describe the quality of signature matches used -# for passive fingerprinting. +## Quality of passive fingerprinting matches. +## +## .. .. bro:see:: OS_version type OS_version_inference: enum { - direct_inference, generic_inference, fuzzy_inference, + direct_inference, ##< TODO. + generic_inference, ##< TODO. + fuzzy_inference, ##< TODO. }; +## Passive fingerprinting match. +## +## .. bro:see:: OS_version_found type OS_version: record { - genre: string; # Linux, Windows, AIX, ... - detail: string; # kernel version or such - dist: count; # how far is the host away from the sensor (TTL)? - match_type: OS_version_inference; + genre: string; ##< Linux, Windows, AIX, ... + detail: string; ##< Lernel version or such. + dist: count; ##< How far is the host away from the sensor (TTL)?. + match_type: OS_version_inference; ##< Quality of the match. }; -# Defines for which subnets we should do passive fingerprinting. +## Defines for which subnets we should do passive fingerprinting. +## +## .. bro:see:: OS_version_found global generate_OS_version_event: set[subnet] &redef; -# Type used to report load samples via load_sample(). For now, -# it's a set of names (event names, source file names, and perhaps -# 's, which were seen during the sample. +# Type used to report load samples via :bro:see:`load_sample`. For now, it's a +# set of names (event names, source file names, and perhaps ````, which were seen during the sample. type load_sample_info: set[string]; -# NetFlow-related data structures. - -## The following provides a mean to sort together NetFlow headers and flow -## records at the script level. rcvr_id equals the name of the file -## (e.g., netflow.dat) or the socket address (e.g., 127.0.0.1:5555), -## or an explicit name if specified to -y or -Y; pdu_id is just a serial -## number, ignoring any overflows. +## ID for NetFlow header. This is primarily a means to sort together NetFlow +## headers and flow records at the script level. type nfheader_id: record { + ## Name of the NetFlow file (e.g., ``netflow.dat``) or the receiving socket address + ## (e.g., ``127.0.0.1:5555``), or an explicit name if specified to + ## ``-y`` or ``-Y``. rcvr_id: string; + ## A serial number, ignoring any overflows. pdu_id: count; }; +## A NetFlow v5 header. +## +## .. bro:see:: netflow_v5_header type nf_v5_header: record { - h_id: nfheader_id; ##< ID for sorting, per the above - cnt: count; - sysuptime: interval; ##< router's uptime - exporttime: time; ##< when the data was exported - flow_seq: count; - eng_type: count; - eng_id: count; - sample_int: count; - exporter: addr; + h_id: nfheader_id; ##< ID for sorting. + cnt: count; ##< TODO. + sysuptime: interval; ##< Router's uptime. + exporttime: time; ##< When the data was exported. + flow_seq: count; ##< Sequence number. + eng_type: count; ##< Engine type. + eng_id: count; ##< Engine ID. + sample_int: count; ##< Sampling interval. + exporter: addr; ##< Exporter address. }; -type nf_v5_record: record { - h_id: nfheader_id; - id: conn_id; - nexthop: addr; - input: count; - output: count; - pkts: count; - octets: count; - first: time; - last: time; - tcpflag_fin: bool; ##< Taken from tcpflags in NF V5; or directly. - tcpflag_syn: bool; - tcpflag_rst: bool; - tcpflag_psh: bool; - tcpflag_ack: bool; - tcpflag_urg: bool; - proto: count; - tos: count; - src_as: count; - dst_as: count; - src_mask: count; - dst_mask: count; +## A NetFlow v5 record. +## +## .. bro:see:: netflow_v5_record +type nf_v5_record: record { + h_id: nfheader_id; ##< ID for sorting. + id: conn_id; ##< Connection ID. + nexthop: addr; ##< Address of next hop. + input: count; ##< Input interface. + output: count; ##< Output interface. + pkts: count; ##< Number of packets. + octets: count; ##< Number of bytes. + first: time; ##< Timestamp of first packet. + last: time; ##< Timestamp of last packet. + tcpflag_fin: bool; ##< FIN flag for TCP flows. + tcpflag_syn: bool; ##< SYN flag for TCP flows. + tcpflag_rst: bool; ##< RST flag for TCP flows. + tcpflag_psh: bool; ##< PSH flag for TCP flows. + tcpflag_ack: bool; ##< ACK flag for TCP flows. + tcpflag_urg: bool; ##< URG flag for TCP flows. + proto: count; ##< IP protocol. + tos: count; ##< Type of service. + src_as: count; ##< Source AS. + dst_as: count; ##< Destination AS. + src_mask: count; ##< Source mask. + dst_mask: count; ##< Destination mask. }; -## The peer record and the corresponding set type used by the -## BitTorrent analyzer. +## A BitTorrent peer. +## +## .. bro:see:: bittorrent_peer_set type bittorrent_peer: record { - h: addr; - p: port; + h: addr; ##< The peer's address. + p: port; ##< The peer's port. }; + +## A set of BitTorrent peers. +## +## .. bro:see:: bt_tracker_response type bittorrent_peer_set: set[bittorrent_peer]; -## The benc value record and the corresponding table type used by the -## BitTorrenttracker analyzer. Note that "benc" = Bencode ("Bee-Encode"), -## per http://en.wikipedia.org/wiki/Bencode. +## BitTorrent "benc" value. Note that "benc" = Bencode ("Bee-Encode"), per +## http://en.wikipedia.org/wiki/Bencode. +## +## .. bro:see:: bittorrent_benc_dir type bittorrent_benc_value: record { - i: int &optional; - s: string &optional; - d: string &optional; - l: string &optional; + i: int &optional; ##< TODO. + s: string &optional; ##< TODO. + d: string &optional; ##< TODO. + l: string &optional; ##< TODO. }; + +## A table of BitTorrent "benc" values. +## +## .. bro:see:: bt_tracker_response type bittorrent_benc_dir: table[string] of bittorrent_benc_value; -## The header table type used by the bittorrenttracker analyzer. +## Header table type used by BitTorrent analyzer. +## +## .. bro:see:: bt_tracker_request bt_tracker_response +## bt_tracker_response_not_ok type bt_tracker_headers: table[string] of string; @load base/event.bif -# The filter the user has set via the -f command line options, or -# empty if none. +## BPF filter the user has set via the -f command line options. Empty if none. const cmd_line_bpf_filter = "" &redef; -## Rotate logs every x interval. +## Deprecated. const log_rotate_interval = 0 sec &redef; -## If set, rotate logs at given time + i * log_rotate_interval. -## (string is time in 24h format, e.g., "18:00"). +## Deprecated. const log_rotate_base_time = "0:00" &redef; -## Rotate logs when they reach this size (in bytes). Note, the -## parameter is a double rather than a count to enable easy expression -## of large values such as 1e7 or exceeding 2^32. +## Deprecated. const log_max_size = 0.0 &redef; -## Default public key for encrypting log files. +## Deprecated. const log_encryption_key = "" &redef; -## Write profiling info into this file. +## Write profiling info into this file in regular intervals. The easiest way to +## activate profiling is loading :doc:`/scripts/policy/misc/profiling`. +## +## .. bro:see:: profiling_interval expensive_profiling_multiple segment_profiling global profiling_file: file &redef; -## Update interval for profiling (0 disables). +## Update interval for profiling (0 disables). The easiest way to activate +## profiling is loading :doc:`/scripts/policy/misc/profiling`. +## +## .. bro:see:: profiling_file expensive_profiling_multiple segment_profiling const profiling_interval = 0 secs &redef; -## Multiples of profiling_interval at which (expensive) memory -## profiling is done (0 disables). +## Multiples of profiling_interval at which (more expensive) memory profiling is +## done (0 disables). +## +## .. bro:see:: profiling_interval profiling_file segment_profiling const expensive_profiling_multiple = 0 &redef; ## If true, then write segment profiling information (very high volume!) -## in addition to statistics. +## in addition to profiling statistics. +## +## .. bro:see:: profiling_interval expensive_profiling_multiple profiling_file const segment_profiling = F &redef; -## Output packet profiling information every secs (mode 1), -## every packets (mode 2), or every bytes (mode 3). -## Mode 0 disables. +## Output modes for packet profiling information. +## +## .. bro:see:: pkt_profile_mode pkt_profile_freq pkt_profile_mode pkt_profile_file type pkt_profile_modes: enum { - PKT_PROFILE_MODE_NONE, - PKT_PROFILE_MODE_SECS, - PKT_PROFILE_MODE_PKTS, - PKT_PROFILE_MODE_BYTES, + PKT_PROFILE_MODE_NONE, ##< No output. + PKT_PROFILE_MODE_SECS, ##< Output every :bro:see:`pkt_profile_freq` seconds. + PKT_PROFILE_MODE_PKTS, ##< Output every :bro:see:`pkt_profile_freq` packets. + PKT_PROFILE_MODE_BYTES, ##< Output every :bro:see:`pkt_profile_freq` bytes. }; + +## Output modes for packet profiling information. +## +## .. bro:see:: pkt_profile_modes pkt_profile_freq pkt_profile_mode pkt_profile_file const pkt_profile_mode = PKT_PROFILE_MODE_NONE &redef; ## Frequency associated with packet profiling. +## +## .. bro:see:: pkt_profile_modes pkt_profile_mode pkt_profile_mode pkt_profile_file const pkt_profile_freq = 0.0 &redef; ## File where packet profiles are logged. +## +## .. bro:see:: pkt_profile_modes pkt_profile_freq pkt_profile_mode pkt_profile_mode global pkt_profile_file: file &redef; -## Rate at which to generate load_sample events, *if* you've also -## defined a load_sample handler. Units are inverse number of packets; -## e.g., a value of 20 means "roughly one in every 20 packets". +## Rate at which to generate :bro:see:`load_sample` events. As all +## events, the event is only generated if you've also defined a +## :bro:see:`load_sample` handler. Units are inverse number of packets; e.g., a +## value of 20 means "roughly one in every 20 packets". +## +## .. bro:see:: load_sample global load_sample_freq = 20 &redef; -## Rate at which to generate gap_report events assessing to what -## degree the measurement process appears to exhibit loss. +## Rate at which to generate :bro:see:`gap_report` events assessing to what degree +## the measurement process appears to exhibit loss. +## +## .. bro:see:: gap_report const gap_report_freq = 1.0 sec &redef; -## Whether we want content_gap and drop reports for partial connections -## (a connection is partial if it is missing a full handshake). Note that -## gap reports for partial connections might not be reliable. +## Whether we want :bro:see:`content_gap` and :bro:see:`gap_report` for partial +## connections. A connection is partial if it is missing a full handshake. Note +## that gap reports for partial connections might not be reliable. +## +## .. bro:see:: content_gap gap_report partial_connection const report_gaps_for_partial = F &redef; -## Globals associated with entire-run statistics on gaps (useful -## for final summaries). - -## The CA certificate file to authorize remote Bros. +## The CA certificate file to authorize remote Bros/Broccolis. +## +## .. bro:see:: ssl_private_key ssl_passphrase const ssl_ca_certificate = "" &redef; ## File containing our private key and our certificate. +## +## .. bro:see:: ssl_ca_certificate ssl_passphrase const ssl_private_key = "" &redef; ## The passphrase for our private key. Keeping this undefined ## causes Bro to prompt for the passphrase. +## +## .. bro:see:: ssl_private_key ssl_ca_certificate const ssl_passphrase = "" &redef; -## Whether the Bro-level packet filter drops packets per default or not. +## Default mode for Bro's user-space dynamic packet filter. If true, packets that +## aren't explicitly allowed through, are dropped from any further processing. +## +## .. note:: This is not the BPF packet filter but an additional dynamic filter +## that Bro optionally applies just before normal processing starts. +## +## .. bro:see:: install_dst_addr_filter install_dst_net_filter +## install_src_addr_filter install_src_net_filter uninstall_dst_addr_filter +## uninstall_dst_net_filter uninstall_src_addr_filter uninstall_src_net_filter const packet_filter_default = F &redef; ## Maximum size of regular expression groups for signature matching. const sig_max_group_size = 50 &redef; -## If true, send logger messages to syslog. +## Deprecated. No longer functional. const enable_syslog = F &redef; -## This is transmitted to peers receiving our events. +## Description transmitted to remote communication peers for identification. const peer_description = "bro" &redef; -## If true, broadcast events/state received from one peer to other peers. +## If true, broadcast events received from one peer to all other peers. +## +## .. bro:see:: forward_remote_state_changes ## -## .. note:: These options are only temporary. They will disappear when we get -## a more sophisticated script-level communication framework. +## .. note:: This option is only temporary and will disappear once we get a more +## sophisticated script-level communication framework. const forward_remote_events = F &redef; -## See :bro:id:`forward_remote_events` + +## If true, broadcast state updates received from one peer to all other peers. +## +## .. bro:see:: forward_remote_events +## +## .. note:: This option is only temporary and will disappear once we get a more +## sophisticated script-level communication framework. const forward_remote_state_changes = F &redef; +## Place-holder constant indicating "no peer". const PEER_ID_NONE = 0; -## Whether to use the connection tracker. +## Deprecated. +## +## .. todo:: The connection compressor is scheduled to be removed from Bro. const use_connection_compressor = F &redef; -## Whether compressor should handle refused connections itself. +## Deprecated. +## +## .. todo:: The connection compressor is scheduled to be removed from Bro. const cc_handle_resets = F &redef; -## Whether compressor should only take care of initial SYNs. -## (By default on, this is basically "connection compressor lite".) +## Deprecated. +## +## .. todo:: The connection compressor is scheduled to be removed from Bro. const cc_handle_only_syns = T &redef; -## Whether compressor instantiates full state when originator sends a -## non-control packet. +## Deprecated. +## +## .. todo:: The connection compressor is scheduled to be removed from Bro. const cc_instantiate_on_data = F &redef; -# Signature payload pattern types -# TODO: use enum to help autodoc -const SIG_PATTERN_PAYLOAD = 0; -const SIG_PATTERN_HTTP = 1; -const SIG_PATTERN_FTP = 2; -const SIG_PATTERN_FINGER = 3; +# Signature payload pattern types. +# todo::use enum to help autodoc +# todo::Still used? +#const SIG_PATTERN_PAYLOAD = 0; +#const SIG_PATTERN_HTTP = 1; +#const SIG_PATTERN_FTP = 2; +#const SIG_PATTERN_FINGER = 3; -# Log-levels for remote_log. -# Eventually we should create a general logging framework and merge these in. -# TODO: use enum to help autodoc -const REMOTE_LOG_INFO = 1; -const REMOTE_LOG_ERROR = 2; +# Deprecated. +# todo::Should use the new logging framework directly. +const REMOTE_LOG_INFO = 1; ##< Deprecated. +const REMOTE_LOG_ERROR = 2; ##< Deprecated. -# Sources for remote_log. -# TODO: use enum to help autodoc -const REMOTE_SRC_CHILD = 1; -const REMOTE_SRC_PARENT = 2; -const REMOTE_SRC_SCRIPT = 3; +# Source of logging messages from the communication framework. +# todo::these should go into an enum to make them autodoc'able. +const REMOTE_SRC_CHILD = 1; ##< Message from the child process. +const REMOTE_SRC_PARENT = 2; ##< Message from the parent process. +const REMOTE_SRC_SCRIPT = 3; ##< Message from a policy script. ## Synchronize trace processing at a regular basis in pseudo-realtime mode. +## +## .. bro:see:: remote_trace_sync_peers const remote_trace_sync_interval = 0 secs &redef; -## Number of peers across which to synchronize trace processing. +## Number of peers across which to synchronize trace processing in +## pseudo-realtime mode. +## +## .. bro:see:: remote_trace_sync_interval const remote_trace_sync_peers = 0 &redef; -## Whether for &synchronized state to send the old value as a consistency check. +## Whether for :bro:attr:`synchronized` state to send the old value as a +## consistency check. const remote_check_sync_consistency = F &redef; ## Analyzer tags. The core automatically defines constants -## ANALYZER_*, e.g., ANALYZER_HTTP. +## ``ANALYZER_*``, e.g., ``ANALYZER_HTTP``. +## +## .. bro:see:: dpd_config +## +## .. todo::We should autodoc these automaticallty generated constants. type AnalyzerTag: count; -# DPD configuration. - +## Set of ports activating a particular protocol analysis. +## +## .. bro:see:: dpd_config type dpd_protocol_config: record { - ports: set[port] &optional; + ports: set[port] &optional; ##< Set of ports. }; +## Port configuration for Bro's "dynamic protocol detection". Protocol +## analyzers can be activated via either well-known ports or content analysis. +## This table defines the ports. +## +## .. bro:see:: dpd_reassemble_first_packets dpd_buffer_size +## dpd_match_only_beginning dpd_ignore_ports const dpd_config: table[AnalyzerTag] of dpd_protocol_config = {} &redef; ## Reassemble the beginning of all TCP connections before doing -## signature-matching for protocol detection. +## signature-matching. Enabling this provides more accurate matching at the +## expensive of CPU cycles. +## +## .. bro:see:: dpd_config dpd_buffer_size +## dpd_match_only_beginning dpd_ignore_ports +## +## .. note:: Despite the name, this option affects *all* signature matching, not +## only signatures used for dynamic protocol detection. const dpd_reassemble_first_packets = T &redef; -## Size of per-connection buffer in bytes. If the buffer is full, data is -## deleted and lost to analyzers that are activated afterwards. +## Size of per-connection buffer used for dynamic protocol detection. For each +## connection, Bro buffers this initial amount of payload in memory so that +## complete protocol analysis can start even after the initial packets have +## already passed through (i.e., when a DPD signature matches only later). +## However, once the buffer is full, data is deleted and lost to analyzers that are +## activated afterwards. Then only analyzers that can deal with partial +## connections will be able to analyze the session. +## +## .. bro:see:: dpd_reassemble_first_packets dpd_config dpd_match_only_beginning +## dpd_ignore_ports const dpd_buffer_size = 1024 &redef; ## If true, stops signature matching if dpd_buffer_size has been reached. +## +## .. bro:see:: dpd_reassemble_first_packets dpd_buffer_size +## dpd_config dpd_ignore_ports +## +## .. note:: Despite the name, this option affects *all* signature matching, not +## only signatures used for dynamic protocol detection. const dpd_match_only_beginning = T &redef; -## If true, don't consider any ports for deciding which analyzer to use. +## If true, don't consider any ports for deciding which protocol analyzer to +## use. If so, the value of :bro:see:`dpd_config` is ignored. +## +## .. bro:see:: dpd_reassemble_first_packets dpd_buffer_size +## dpd_match_only_beginning dpd_config const dpd_ignore_ports = F &redef; -## Ports which the core considers being likely used by servers. +## Ports which the core considers being likely used by servers. For ports in +## this set, is may heuristically decide to flip the direction of the +## connection if it misses the initial handshake. const likely_server_ports: set[port] &redef; -## Set of all ports for which we know an analyzer. +## Deprated. Set of all ports for which we know an analyzer, built by +## :doc:`/scripts/base/frameworks/dpd/main`. +## +## .. todo::This should be defined by :doc:`/scripts/base/frameworks/dpd/main` +## itself we still need it. global dpd_analyzer_ports: table[port] of set[AnalyzerTag]; ## Per-incident timer managers are drained after this amount of inactivity. @@ -1474,37 +2294,41 @@ const time_machine_profiling = F &redef; ## If true, warns about unused event handlers at startup. const check_for_unused_event_handlers = F &redef; -## If true, dumps all invoked event handlers at startup. -const dump_used_event_handlers = F &redef; +# If true, dumps all invoked event handlers at startup. +# todo::Still used? +# const dump_used_event_handlers = F &redef; -## If true, we suppress prints to local files if we have a receiver for -## print_hook events. Ignored for files with a &disable_print_hook attribute. +## Deprecated. const suppress_local_output = F &redef; ## Holds the filename of the trace file given with -w (empty if none). +## +## .. bro:see:: record_all_packets const trace_output_file = ""; -## If a trace file is given, dump *all* packets seen by Bro into it. -## By default, Bro applies (very few) heuristics to reduce the volume. -## A side effect of setting this to true is that we can write the -## packets out before we actually process them, which can be helpful -## for debugging in case the analysis triggers a crash. +## If a trace file is given with ``-w``, dump *all* packets seen by Bro into it. By +## default, Bro applies (very few) heuristics to reduce the volume. A side effect +## of setting this to true is that we can write the packets out before we actually +## process them, which can be helpful for debugging in case the analysis triggers a +## crash. +## +## .. bro:see:: trace_output_file const record_all_packets = F &redef; -## Some connections (e.g., SSH) retransmit the acknowledged last -## byte to keep the connection alive. If ignore_keep_alive_rexmit -## is set to T, such retransmissions will be excluded in the rexmit -## counter in conn_stats. +## Ignore certain TCP retransmissions for :bro:see:`conn_stats`. Some connections +## (e.g., SSH) retransmit the acknowledged last byte to keep the connection alive. +## If *ignore_keep_alive_rexmit* is set to true, such retransmissions will be +## excluded in the rexmit counter in :bro:see:`conn_stats`. +## +## .. bro:see:: conn_stats const ignore_keep_alive_rexmit = F &redef; -## Skip HTTP data portions for performance considerations (the skipped -## portion will not go through TCP reassembly). -const skip_http_data = F &redef; - ## Whether the analysis engine parses IP packets encapsulated in -## UDP tunnels. See also: udp_tunnel_port, policy/udp-tunnel.bro. +## UDP tunnels. +## +## .. bro:see:: tunnel_port const parse_udp_tunnels = F &redef; -# Load the logging framework here because it uses fairly deep integration with +# Load the logging framework here because it uses fairly deep integration with # BiFs and script-land defined types. @load base/frameworks/logging diff --git a/src/event.bif b/src/event.bif index df6af21d66..8783e40b76 100644 --- a/src/event.bif +++ b/src/event.bif @@ -857,8 +857,8 @@ event icmp_time_exceeded%(c: connection, icmp: icmp_conn, code: count, context: event icmp_redirect%(c: connection, icmp: icmp_conn, a: addr%); ## Generated when a TCP connection terminated, passing on statistics about the -## two endpoints. This event is generated when Bro flushes the internal connection -## state, independent of how the connection gad terminated. +## two endpoints. This event is always generated when Bro flushes the internal +## connection state, independent of how a connection terminates. ## ## c: The connection. ## @@ -5311,6 +5311,7 @@ event software_unparsed_version_found%(c: connection, host: addr, str: string%); ## ## .. bro:see:: passive_fingerprint_file software_parse_error ## software_version_found software_unparsed_version_found +## generate_OS_version_event event OS_version_found%(c: connection, host: addr, OS: OS_version%); ## Generated when a connection to a remote Bro has been established. This event From 28c0733dcafac0c9cfe39b1a9f5114d706454f74 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 15 Dec 2011 06:31:51 -0800 Subject: [PATCH 185/964] Adding todo to all protocol events that aren't generated yet because 2.0 doesn't activate the analyzer. Seth, can you double-check whether I got the right events? --- src/event.bif | 535 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 535 insertions(+) diff --git a/src/event.bif b/src/event.bif index 8783e40b76..73c3bb0959 100644 --- a/src/event.bif +++ b/src/event.bif @@ -1001,6 +1001,11 @@ event arp_reply%(mac_src: string, mac_dst: string, SPA: addr, SHA: string, ## explanation: A short description of why the ARP packet is considered "bad". ## ## .. bro:see:: arp_reply arp_request +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event bad_arp%(SPA: addr, SHA: string, TPA: addr, THA: string, explanation: string%); ## TODO. @@ -1243,6 +1248,11 @@ event bt_tracker_weird%(c: connection, is_orig: bool, msg: string%); ## hostname: The request's host name. ## ## .. bro:see:: finger_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event finger_request%(c: connection, full: bool, username: string, hostname: string%); ## Generated for Finger replies. @@ -1255,6 +1265,11 @@ event finger_request%(c: connection, full: bool, username: string, hostname: str ## reply_line: The reply as returned by the server ## ## .. bro:see:: finger_request +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event finger_reply%(c: connection, reply_line: string%); @@ -1266,6 +1281,11 @@ event finger_reply%(c: connection, reply_line: string%); ## .. bro:see:: gnutella_binary_msg gnutella_establish gnutella_http_notify ## gnutella_not_establish gnutella_partial_binary_msg gnutella_signature_found ## +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event gnutella_text_msg%(c: connection, orig: bool, headers: string%); ## TODO. @@ -1275,6 +1295,11 @@ event gnutella_text_msg%(c: connection, orig: bool, headers: string%); ## ## .. bro:see:: gnutella_establish gnutella_http_notify gnutella_not_establish ## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event gnutella_binary_msg%(c: connection, orig: bool, msg_type: count, ttl: count, hops: count, msg_len: count, payload: string, payload_len: count, @@ -1287,6 +1312,11 @@ event gnutella_binary_msg%(c: connection, orig: bool, msg_type: count, ## ## .. bro:see:: gnutella_binary_msg gnutella_establish gnutella_http_notify ## gnutella_not_establish gnutella_signature_found gnutella_text_msg +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event gnutella_partial_binary_msg%(c: connection, orig: bool, msg: string, len: count%); @@ -1297,6 +1327,11 @@ event gnutella_partial_binary_msg%(c: connection, orig: bool, ## ## .. bro:see:: gnutella_binary_msg gnutella_http_notify gnutella_not_establish ## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event gnutella_establish%(c: connection%); ## TODO. @@ -1306,6 +1341,11 @@ event gnutella_establish%(c: connection%); ## ## .. bro:see:: gnutella_binary_msg gnutella_establish gnutella_http_notify ## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event gnutella_not_establish%(c: connection%); ## TODO. @@ -1315,6 +1355,11 @@ event gnutella_not_establish%(c: connection%); ## ## .. bro:see:: gnutella_binary_msg gnutella_establish gnutella_not_establish ## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event gnutella_http_notify%(c: connection%); ## Generated for Ident requests. @@ -1329,6 +1374,11 @@ event gnutella_http_notify%(c: connection%); ## rport: The request's remote port. ## ## .. bro:see:: ident_error ident_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event ident_request%(c: connection, lport: port, rport: port%); ## Generated for Ident replies. @@ -1347,6 +1397,11 @@ event ident_request%(c: connection, lport: port, rport: port%); ## system: The operating system returned by the reply. ## ## .. bro:see:: ident_error ident_request +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event ident_reply%(c: connection, lport: port, rport: port, user_id: string, system: string%); ## Generated for Ident error replies. @@ -1363,6 +1418,11 @@ event ident_reply%(c: connection, lport: port, rport: port, user_id: string, sys ## line: The error description returned by the reply. ## ## .. bro:see:: ident_reply ident_request +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event ident_error%(c: connection, lport: port, rport: port, line: string%); ## Generated for Telnet/Rlogin login failures. The *login* analyzer inspects @@ -1392,6 +1452,16 @@ event ident_error%(c: connection, lport: port, rport: port, line: string%); ## need to configured with patterns identifying login attempts. This configuration ## has not yet been ported over from Bro 1.5 to Bro 2.x, and the analyzer is ## therefore not directly usable at the moment. +## +## .. todo: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event login_failure%(c: connection, user: string, client_user: string, password: string, line: string%); ## Generated for successful Telnet/Rlogin logins. The *login* analyzer inspects @@ -1421,6 +1491,11 @@ event login_failure%(c: connection, user: string, client_user: string, password: ## need to configured with patterns identifying login attempts. This configuration ## has not yet been ported over from Bro 1.5 to Bro 2.x, and the analyzer is ## therefore not directly usable at the moment. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event login_success%(c: connection, user: string, client_user: string, password: string, line: string%); ## Generated for lines of input on Telnet/Rlogin sessions. The line will have @@ -1432,6 +1507,11 @@ event login_success%(c: connection, user: string, client_user: string, password: ## ## .. bro:see:: login_confused login_confused_text login_display login_failure ## login_output_line login_prompt login_success login_terminal rsh_request +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event login_input_line%(c: connection, line: string%); ## Generated for lines of output on Telnet/Rlogin sessions. The line will have @@ -1443,6 +1523,11 @@ event login_input_line%(c: connection, line: string%); ## ## .. bro:see:: login_confused login_confused_text login_display login_failure ## login_input_line login_prompt login_success login_terminal rsh_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event login_output_line%(c: connection, line: string%); ## Generated when tracking of Telnet/Rlogin authentication failed. As Bro's *login* @@ -1464,6 +1549,11 @@ event login_output_line%(c: connection, line: string%); ## login_prompt login_success login_terminal direct_login_prompts get_login_state ## login_failure_msgs login_non_failure_msgs login_prompts login_success_msgs ## login_timeouts set_login_state +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event login_confused%(c: connection, msg: string, line: string%); ## Generated after getting confused while tracking a Telnet/Rlogin authentication @@ -1478,6 +1568,11 @@ event login_confused%(c: connection, msg: string, line: string%); ## login_output_line login_prompt login_success login_terminal direct_login_prompts ## get_login_state login_failure_msgs login_non_failure_msgs login_prompts ## login_success_msgs login_timeouts set_login_state +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event login_confused_text%(c: connection, line: string%); ## Generated for clients transmitting a terminal type in an Telnet session. This @@ -1489,6 +1584,11 @@ event login_confused_text%(c: connection, line: string%); ## ## .. bro:see:: login_confused login_confused_text login_display login_failure ## login_input_line login_output_line login_prompt login_success +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event login_terminal%(c: connection, terminal: string%); ## Generated for clients transmitting a X11 DISPLAY in a Telnet session. This @@ -1500,6 +1600,11 @@ event login_terminal%(c: connection, terminal: string%); ## ## .. bro:see:: login_confused login_confused_text login_failure login_input_line ## login_output_line login_prompt login_success login_terminal +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event login_display%(c: connection, display: string%); ## Generated when a Telnet authentication has been successful. The Telnet protocol @@ -1518,6 +1623,11 @@ event login_display%(c: connection, display: string%); ## ## .. note:: This event inspects the corresponding Telnet option while :bro:id:`login_success` ## heuristically determines success by watching session data. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event authentication_accepted%(name: string, c: connection%); ## Generated when a Telnet authentication has been unsuccessful. The Telnet @@ -1537,6 +1647,11 @@ event authentication_accepted%(name: string, c: connection%); ## .. note:: This event inspects the corresponding Telnet option while :bro:id:`login_success` ## heuristically determines failure by watching session ## data. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event authentication_rejected%(name: string, c: connection%); ## Generated when for Telnet/Rlogin sessions when a pattern match indicates @@ -1555,6 +1670,11 @@ event authentication_rejected%(name: string, c: connection%); ## need to be configured with patterns identifying actvity. This configuration has ## not yet been ported over from Bro 1.5 to Bro 2.x, and the analyzer is therefore ## not directly usable at the moment. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event authentication_skipped%(c: connection%); ## Generated for clients transmitting a terminal prompt in a Telnet session. This @@ -1569,6 +1689,11 @@ event authentication_skipped%(c: connection%); ## ## .. bro:see:: login_confused login_confused_text login_display login_failure ## login_input_line login_output_line login_success login_terminal +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event login_prompt%(c: connection, prompt: string%); ## Generated for Telnet sessions when encryption is activated. The Telnet protoco; @@ -1615,6 +1740,11 @@ event inconsistent_option%(c: connection%); ## authentication_rejected authentication_skipped login_confused ## login_confused_text login_display login_failure login_input_line ## login_output_line login_prompt login_success login_terminal +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event bad_option%(c: connection%); ## Generated for a Telnet option that's incorrectly terminated. @@ -1626,6 +1756,11 @@ event bad_option%(c: connection%); ## authentication_rejected authentication_skipped login_confused ## login_confused_text login_display login_failure login_input_line ## login_output_line login_prompt login_success login_terminal +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event bad_option_termination%(c: connection%); ## Generated for client side commands on an RSH connection. @@ -1651,6 +1786,11 @@ event bad_option_termination%(c: connection%); ## ## .. note: For historical reasons, these events are separate from the ``login_`` ## events. Ideally, they would all be handled uniquely. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event rsh_request%(c: connection, client_user: string, server_user: string, line: string, new_session: bool%); ## Generated for client side commands on an RSH connection. @@ -1676,6 +1816,11 @@ event rsh_request%(c: connection, client_user: string, server_user: string, line ## ## .. note: For historical reasons, these events are separate from the ``login_`` ## events. Ideally, they would all be handled uniquely. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event rsh_reply%(c: connection, client_user: string, server_user: string, line: string%); ## Generated for client-side FTP commands. @@ -2031,6 +2176,11 @@ event mime_content_hash%(c: connection, content_len: count, hash_value: string%) ## ## .. bro:see:: rpc_call rpc_reply dce_rpc_bind dce_rpc_message dce_rpc_request ## dce_rpc_response rpc_timeout +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event rpc_dialogue%(c: connection, prog: count, ver: count, proc: count, status: rpc_status, start_time: time, call_len: count, reply_len: count%); ## Generated for RPC *call* messages. @@ -2052,6 +2202,11 @@ event rpc_dialogue%(c: connection, prog: count, ver: count, proc: count, status: ## ## .. bro:see:: rpc_dialogue rpc_reply dce_rpc_bind dce_rpc_message dce_rpc_request ## dce_rpc_response rpc_timeout +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event rpc_call%(c: connection, xid: count, prog: count, ver: count, proc: count, call_len: count%); ## Generated for RPC *reply* messages. @@ -2070,6 +2225,11 @@ event rpc_call%(c: connection, xid: count, prog: count, ver: count, proc: count, ## ## .. bro:see:: rpc_call rpc_dialogue dce_rpc_bind dce_rpc_message dce_rpc_request ## dce_rpc_response rpc_timeout +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event rpc_reply%(c: connection, xid: count, status: rpc_status, reply_len: count%); ## Generated for Portmapper requests of type *null*. @@ -2083,6 +2243,11 @@ event rpc_reply%(c: connection, xid: count, status: rpc_status, reply_len: count ## pm_attempt_null pm_attempt_set pm_attempt_unset pm_bad_port pm_request_callit ## pm_request_dump pm_request_getport pm_request_set pm_request_unset rpc_call ## rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pm_request_null%(r: connection%); ## Generated for Portmapper request/reply dialogues of type *set*. @@ -2101,6 +2266,11 @@ event pm_request_null%(r: connection%); ## pm_attempt_null pm_attempt_set pm_attempt_unset pm_bad_port pm_request_callit ## pm_request_dump pm_request_getport pm_request_null pm_request_unset rpc_call ## rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pm_request_set%(r: connection, m: pm_mapping, success: bool%); ## Generated for Portmapper request/reply dialogues of type *unset*. @@ -2119,6 +2289,11 @@ event pm_request_set%(r: connection, m: pm_mapping, success: bool%); ## pm_attempt_null pm_attempt_set pm_attempt_unset pm_bad_port pm_request_callit ## pm_request_dump pm_request_getport pm_request_null pm_request_set rpc_call ## rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pm_request_unset%(r: connection, m: pm_mapping, success: bool%); ## Generated for Portmapper request/reply dialogues of type *getport*. @@ -2139,6 +2314,11 @@ event pm_request_unset%(r: connection, m: pm_mapping, success: bool%); ## pm_attempt_null pm_attempt_set pm_attempt_unset pm_bad_port pm_request_callit ## pm_request_dump pm_request_null pm_request_set pm_request_unset rpc_call ## rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pm_request_getport%(r: connection, pr: pm_port_request, p: port%); ## Generated for Portmapper request/reply dialogues of type *dump*. @@ -2154,6 +2334,11 @@ event pm_request_getport%(r: connection, pr: pm_port_request, p: port%); ## pm_attempt_null pm_attempt_set pm_attempt_unset pm_bad_port pm_request_callit ## pm_request_getport pm_request_null pm_request_set pm_request_unset rpc_call ## rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pm_request_dump%(r: connection, m: pm_mappings%); ## Generated for Portmapper request/reply dialogues of type *callit*. @@ -2172,6 +2357,11 @@ event pm_request_dump%(r: connection, m: pm_mappings%); ## pm_attempt_null pm_attempt_set pm_attempt_unset pm_bad_port pm_request_dump ## pm_request_getport pm_request_null pm_request_set pm_request_unset rpc_call ## rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pm_request_callit%(r: connection, call: pm_callit_request, p: port%); ## Generated for failed Portmapper requests of type *null*. @@ -2189,6 +2379,11 @@ event pm_request_callit%(r: connection, call: pm_callit_request, p: port%); ## pm_attempt_set pm_attempt_unset pm_bad_port pm_request_callit pm_request_dump ## pm_request_getport pm_request_null pm_request_set pm_request_unset rpc_call ## rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pm_attempt_null%(r: connection, status: rpc_status%); ## Generated for failed Portmapper requests of type *set*. @@ -2208,6 +2403,11 @@ event pm_attempt_null%(r: connection, status: rpc_status%); ## pm_attempt_null pm_attempt_unset pm_bad_port pm_request_callit pm_request_dump ## pm_request_getport pm_request_null pm_request_set pm_request_unset rpc_call ## rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pm_attempt_set%(r: connection, status: rpc_status, m: pm_mapping%); ## Generated for failed Portmapper requests of type *unset*. @@ -2227,6 +2427,11 @@ event pm_attempt_set%(r: connection, status: rpc_status, m: pm_mapping%); ## pm_attempt_null pm_attempt_set pm_bad_port pm_request_callit pm_request_dump ## pm_request_getport pm_request_null pm_request_set pm_request_unset rpc_call ## rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pm_attempt_unset%(r: connection, status: rpc_status, m: pm_mapping%); ## Generated for failed Portmapper requests of type *getport*. @@ -2246,6 +2451,11 @@ event pm_attempt_unset%(r: connection, status: rpc_status, m: pm_mapping%); ## pm_attempt_set pm_attempt_unset pm_bad_port pm_request_callit pm_request_dump ## pm_request_getport pm_request_null pm_request_set pm_request_unset rpc_call ## rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pm_attempt_getport%(r: connection, status: rpc_status, pr: pm_port_request%); ## Generated for failed Portmapper requests of type *dump*. @@ -2263,6 +2473,11 @@ event pm_attempt_getport%(r: connection, status: rpc_status, pr: pm_port_request ## pm_attempt_set pm_attempt_unset pm_bad_port pm_request_callit pm_request_dump ## pm_request_getport pm_request_null pm_request_set pm_request_unset rpc_call ## rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pm_attempt_dump%(r: connection, status: rpc_status%); ## Generated for failed Portmapper requests of type *callit*. @@ -2282,6 +2497,11 @@ event pm_attempt_dump%(r: connection, status: rpc_status%); ## pm_attempt_set pm_attempt_unset pm_bad_port pm_request_callit pm_request_dump ## pm_request_getport pm_request_null pm_request_set pm_request_unset rpc_call ## rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pm_attempt_callit%(r: connection, status: rpc_status, call: pm_callit_request%); ## Generated for Portmapper requests or replies that include an invalid port @@ -2301,6 +2521,11 @@ event pm_attempt_callit%(r: connection, status: rpc_status, call: pm_callit_requ ## pm_attempt_null pm_attempt_set pm_attempt_unset pm_request_callit ## pm_request_dump pm_request_getport pm_request_null pm_request_set ## pm_request_unset rpc_call rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pm_bad_port%(r: connection, bad_p: count%); ## Generated for NFSv3 request/reply dialogues of type *null*. The event is @@ -2319,6 +2544,11 @@ event pm_bad_port%(r: connection, bad_p: count%); ## nfs_proc_not_implemented nfs_proc_read nfs_proc_readdir nfs_proc_readlink ## nfs_proc_remove nfs_proc_rmdir nfs_proc_write nfs_reply_status rpc_call ## rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event nfs_proc_null%(c: connection, info: NFS3::info_t%); ## Generated for NFSv3 request/reply dialogues of type *getattr*. The event is @@ -2340,6 +2570,11 @@ event nfs_proc_null%(c: connection, info: NFS3::info_t%); ## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readdir ## nfs_proc_readlink nfs_proc_remove nfs_proc_rmdir nfs_proc_write nfs_reply_status ## rpc_call rpc_dialogue rpc_reply NFS3::mode2string +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event nfs_proc_getattr%(c: connection, info: NFS3::info_t, fh: string, attrs: NFS3::fattr_t%); ## Generated for NFSv3 request/reply dialogues of type *lookup*. The event is @@ -2363,6 +2598,11 @@ event nfs_proc_getattr%(c: connection, info: NFS3::info_t, fh: string, attrs: NF ## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readdir ## nfs_proc_readlink nfs_proc_remove nfs_proc_rmdir nfs_proc_write nfs_reply_status ## rpc_call rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event nfs_proc_lookup%(c: connection, info: NFS3::info_t, req: NFS3::diropargs_t, rep: NFS3::lookup_reply_t%); ## Generated for NFSv3 request/reply dialogues of type *read*. The event is @@ -2386,6 +2626,11 @@ event nfs_proc_lookup%(c: connection, info: NFS3::info_t, req: NFS3::diropargs_t ## nfs_proc_not_implemented nfs_proc_null nfs_proc_remove nfs_proc_rmdir ## nfs_proc_write nfs_reply_status rpc_call rpc_dialogue rpc_reply ## NFS3::return_data NFS3::return_data_first_only NFS3::return_data_max +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event nfs_proc_read%(c: connection, info: NFS3::info_t, req: NFS3::readargs_t, rep: NFS3::read_reply_t%); ## Generated for NFSv3 request/reply dialogues of type *readlink*. The event is @@ -2409,6 +2654,11 @@ event nfs_proc_read%(c: connection, info: NFS3::info_t, req: NFS3::readargs_t, r ## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readdir ## nfs_proc_remove nfs_proc_rmdir nfs_proc_write nfs_reply_status rpc_call ## rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event nfs_proc_readlink%(c: connection, info: NFS3::info_t, fh: string, rep: NFS3::readlink_reply_t%); ## Generated for NFSv3 request/reply dialogues of type *write*. The event is @@ -2433,6 +2683,11 @@ event nfs_proc_readlink%(c: connection, info: NFS3::info_t, fh: string, rep: NFS ## nfs_proc_readlink nfs_proc_remove nfs_proc_rmdir nfs_reply_status rpc_call ## rpc_dialogue rpc_reply NFS3::return_data NFS3::return_data_first_only ## NFS3::return_data_max +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event nfs_proc_write%(c: connection, info: NFS3::info_t, req: NFS3::writeargs_t, rep: NFS3::write_reply_t%); ## Generated for NFSv3 request/reply dialogues of type *create*. The event is @@ -2456,6 +2711,11 @@ event nfs_proc_write%(c: connection, info: NFS3::info_t, req: NFS3::writeargs_t, ## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readdir ## nfs_proc_readlink nfs_proc_remove nfs_proc_rmdir nfs_proc_write nfs_reply_status ## rpc_call rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event nfs_proc_create%(c: connection, info: NFS3::info_t, req: NFS3::diropargs_t, rep: NFS3::newobj_reply_t%); ## Generated for NFSv3 request/reply dialogues of type *mkdir*. The event is @@ -2479,6 +2739,11 @@ event nfs_proc_create%(c: connection, info: NFS3::info_t, req: NFS3::diropargs_t ## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readdir ## nfs_proc_readlink nfs_proc_remove nfs_proc_rmdir nfs_proc_write nfs_reply_status ## rpc_call rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event nfs_proc_mkdir%(c: connection, info: NFS3::info_t, req: NFS3::diropargs_t, rep: NFS3::newobj_reply_t%); ## Generated for NFSv3 request/reply dialogues of type *remove*. The event is @@ -2502,6 +2767,11 @@ event nfs_proc_mkdir%(c: connection, info: NFS3::info_t, req: NFS3::diropargs_t, ## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readdir ## nfs_proc_readlink nfs_proc_rmdir nfs_proc_write nfs_reply_status rpc_call ## rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event nfs_proc_remove%(c: connection, info: NFS3::info_t, req: NFS3::diropargs_t, rep: NFS3::delobj_reply_t%); ## Generated for NFSv3 request/reply dialogues of type *rmdir*. The event is @@ -2525,6 +2795,11 @@ event nfs_proc_remove%(c: connection, info: NFS3::info_t, req: NFS3::diropargs_t ## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readdir ## nfs_proc_readlink nfs_proc_remove nfs_proc_write nfs_reply_status rpc_call ## rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event nfs_proc_rmdir%(c: connection, info: NFS3::info_t, req: NFS3::diropargs_t, rep: NFS3::delobj_reply_t%); ## Generated for NFSv3 request/reply dialogues of type *readdir*. The event is @@ -2548,6 +2823,11 @@ event nfs_proc_rmdir%(c: connection, info: NFS3::info_t, req: NFS3::diropargs_t, ## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readlink ## nfs_proc_remove nfs_proc_rmdir nfs_proc_write nfs_reply_status rpc_call ## rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event nfs_proc_readdir%(c: connection, info: NFS3::info_t, req: NFS3::readdirargs_t, rep: NFS3::readdir_reply_t%); ## Generated for NFS3 request/reply dialogues of a type that Bro's NFS3 analyzer @@ -2566,6 +2846,11 @@ event nfs_proc_readdir%(c: connection, info: NFS3::info_t, req: NFS3::readdirarg ## .. bro:see:: nfs_proc_create nfs_proc_getattr nfs_proc_lookup nfs_proc_mkdir ## nfs_proc_null nfs_proc_read nfs_proc_readdir nfs_proc_readlink nfs_proc_remove ## nfs_proc_rmdir nfs_proc_write nfs_reply_status rpc_call rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event nfs_proc_not_implemented%(c: connection, info: NFS3::info_t, proc: NFS3::proc_t%); ## Generated for each NFS3 reply message received, reporting just the @@ -2577,6 +2862,11 @@ event nfs_proc_not_implemented%(c: connection, info: NFS3::info_t, proc: NFS3::p ## nfs_proc_not_implemented nfs_proc_null nfs_proc_read nfs_proc_readdir ## nfs_proc_readlink nfs_proc_remove nfs_proc_rmdir nfs_proc_write rpc_call ## rpc_dialogue rpc_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event nfs_reply_status%(n: connection, info: NFS3::info_t%); ## Generated for all NTP messages. Different from many other of Bro's events, this @@ -2593,6 +2883,11 @@ event nfs_reply_status%(n: connection, info: NFS3::info_t%); ## further parse any optional fields. ## ## .. bro:see:: ntp_session_timeout +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event ntp_message%(u: connection, msg: ntp_msg, excess: string%); ## Generated for all NetBIOS SSN and DGM messages. Bro's NetBIOS analyzer processes @@ -2620,6 +2915,11 @@ event ntp_message%(u: connection, msg: ntp_msg, excess: string%); ## .. note:: These days, NetBIOS is primarily used as a transport mechanism for ## `SMB/CIFS `__. Bro's SMB ## anlyzer parses both SMB-over-NetBIOS and SMB-over-TCP on port 445. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event netbios_session_message%(c: connection, is_orig: bool, msg_type: count, data_len: count%); ## Generated for NetBIOS messages of type *session request*. Bro's NetBIOS analyzer @@ -2643,6 +2943,11 @@ event netbios_session_message%(c: connection, is_orig: bool, msg_type: count, da ## .. note:: These days, NetBIOS is primarily used as a transport mechanism for ## `SMB/CIFS `__. Bro's SMB ## anlyzer parses both SMB-over-NetBIOS and SMB-over-TCP on port 445. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event netbios_session_request%(c: connection, msg: string%); ## Generated for NetBIOS messages of type *positive session response*. Bro's @@ -2666,6 +2971,11 @@ event netbios_session_request%(c: connection, msg: string%); ## .. note:: These days, NetBIOS is primarily used as a transport mechanism for ## `SMB/CIFS `__. Bro's SMB ## anlyzer parses both SMB-over-NetBIOS and SMB-over-TCP on port 445. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event netbios_session_accepted%(c: connection, msg: string%); ## Generated for NetBIOS messages of type *negative session response*. Bro's @@ -2689,6 +2999,11 @@ event netbios_session_accepted%(c: connection, msg: string%); ## .. note:: These days, NetBIOS is primarily used as a transport mechanism for ## `SMB/CIFS `__. Bro's SMB ## anlyzer parses both SMB-over-NetBIOS and SMB-over-TCP on port 445. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event netbios_session_rejected%(c: connection, msg: string%); ## Generated for NetBIOS message of type *session message* that are not carrying @@ -2719,6 +3034,11 @@ event netbios_session_rejected%(c: connection, msg: string%); ## ## .. todo:: This is an oddly named event. In fact, it's probably an odd event to ## have to begin with. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event netbios_session_raw_message%(c: connection, is_orig: bool, msg: string%); ## Generated for NetBIOS messages of type *retarget response*. Bro's NetBIOS @@ -2743,6 +3063,11 @@ event netbios_session_raw_message%(c: connection, is_orig: bool, msg: string%); ## anlyzer parses both SMB-over-NetBIOS and SMB-over-TCP on port 445. ## ## .. todo: This is an oddly named event. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event netbios_session_ret_arg_resp%(c: connection, msg: string%); ## Generated for NetBIOS messages of type *keep-alive*. Bro's NetBIOS analyzer @@ -2765,6 +3090,11 @@ event netbios_session_ret_arg_resp%(c: connection, msg: string%); ## .. note:: These days, NetBIOS is primarily used as a transport mechanism for ## `SMB/CIFS `__. Bro's SMB ## anlyzer parses both SMB-over-NetBIOS and SMB-over-TCP on port 445. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event netbios_session_keepalive%(c: connection, msg: string%); ## Generated for all SMB/CIFS messages. @@ -2793,6 +3123,11 @@ event netbios_session_keepalive%(c: connection, msg: string%); ## smb_com_trans_pipe smb_com_trans_rap smb_com_transaction smb_com_transaction2 ## smb_com_tree_connect_andx smb_com_tree_disconnect smb_com_write_andx smb_error ## smb_get_dfs_referral +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_message%(c: connection, hdr: smb_hdr, is_orig: bool, cmd: string, body_length: count, body: string%); ## Generated for SMB/CIFS messages of type *tree connect andx*. @@ -2815,6 +3150,11 @@ event smb_message%(c: connection, hdr: smb_hdr, is_orig: bool, cmd: string, body ## smb_com_trans_pipe smb_com_trans_rap smb_com_transaction smb_com_transaction2 ## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral ## smb_message +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_com_tree_connect_andx%(c: connection, hdr: smb_hdr, path: string, service: string%); ## Generated for SMB/CIFS messages of type *tree disconnect*. @@ -2835,6 +3175,11 @@ event smb_com_tree_connect_andx%(c: connection, hdr: smb_hdr, path: string, serv ## smb_com_trans_pipe smb_com_trans_rap smb_com_transaction smb_com_transaction2 ## smb_com_tree_connect_andx smb_com_write_andx smb_error smb_get_dfs_referral ## smb_message +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_com_tree_disconnect%(c: connection, hdr: smb_hdr%); ## Generated for SMB/CIFS messages of type *nt create andx*. @@ -2855,6 +3200,11 @@ event smb_com_tree_disconnect%(c: connection, hdr: smb_hdr%); ## smb_com_transaction smb_com_transaction2 smb_com_tree_connect_andx ## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral ## smb_message +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_com_nt_create_andx%(c: connection, hdr: smb_hdr, name: string%); ## Generated for SMB/CIFS messages of type *nt transaction*. @@ -2879,6 +3229,11 @@ event smb_com_nt_create_andx%(c: connection, hdr: smb_hdr, name: string%); ## smb_com_trans_rap smb_com_transaction2 smb_com_tree_connect_andx ## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral ## smb_message +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_com_transaction%(c: connection, hdr: smb_hdr, trans: smb_trans, data: smb_trans_data, is_orig: bool%); ## Generated for SMB/CIFS messages of type *nt transaction 2*. @@ -2903,6 +3258,11 @@ event smb_com_transaction%(c: connection, hdr: smb_hdr, trans: smb_trans, data: ## smb_com_trans_rap smb_com_transaction smb_com_tree_connect_andx ## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral ## smb_message +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_com_transaction2%(c: connection, hdr: smb_hdr, trans: smb_trans, data: smb_trans_data, is_orig: bool%); ## Generated for SMB/CIFS messages of type *transaction mailslot*. @@ -2927,6 +3287,11 @@ event smb_com_transaction2%(c: connection, hdr: smb_hdr, trans: smb_trans, data: ## smb_com_transaction smb_com_transaction2 smb_com_tree_connect_andx ## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral ## smb_message +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_com_trans_mailslot%(c: connection, hdr: smb_hdr, trans: smb_trans, data: smb_trans_data, is_orig: bool%); ## Generated for SMB/CIFS messages of type *transaction rap*. @@ -2951,6 +3316,11 @@ event smb_com_trans_mailslot%(c: connection, hdr: smb_hdr, trans: smb_trans, dat ## smb_com_trans_pipe smb_com_transaction smb_com_transaction2 ## smb_com_tree_connect_andx smb_com_tree_disconnect smb_com_write_andx smb_error ## smb_get_dfs_referral smb_message +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_com_trans_rap%(c: connection, hdr: smb_hdr, trans: smb_trans, data: smb_trans_data, is_orig: bool%); ## Generated for SMB/CIFS messages of type *transaction pipe*. @@ -2975,6 +3345,11 @@ event smb_com_trans_rap%(c: connection, hdr: smb_hdr, trans: smb_trans, data: sm ## smb_com_transaction smb_com_transaction2 smb_com_tree_connect_andx ## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral ## smb_message +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_com_trans_pipe%(c: connection, hdr: smb_hdr, trans: smb_trans, data: smb_trans_data, is_orig: bool%); ## Generated for SMB/CIFS messages of type *read andx*. @@ -2997,6 +3372,11 @@ event smb_com_trans_pipe%(c: connection, hdr: smb_hdr, trans: smb_trans, data: s ## smb_com_transaction smb_com_transaction2 smb_com_tree_connect_andx ## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral ## smb_message +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_com_read_andx%(c: connection, hdr: smb_hdr, data: string%); ## Generated for SMB/CIFS messages of type *read andx*. @@ -3019,6 +3399,11 @@ event smb_com_read_andx%(c: connection, hdr: smb_hdr, data: string%); ## smb_com_trans_pipe smb_com_trans_rap smb_com_transaction smb_com_transaction2 ## smb_com_tree_connect_andx smb_com_tree_disconnect smb_error ## smb_get_dfs_referral smb_message +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_com_write_andx%(c: connection, hdr: smb_hdr, data: string%); ## Generated for SMB/CIFS messages of type *get dfs referral*. @@ -3042,6 +3427,11 @@ event smb_com_write_andx%(c: connection, hdr: smb_hdr, data: string%); ## smb_com_trans_pipe smb_com_trans_rap smb_com_transaction smb_com_transaction2 ## smb_com_tree_connect_andx smb_com_tree_disconnect smb_com_write_andx smb_error ## smb_message +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_get_dfs_referral%(c: connection, hdr: smb_hdr, max_referral_level: count, file_name: string%); ## Generated for SMB/CIFS messages of type *negotiate*. @@ -3059,6 +3449,11 @@ event smb_get_dfs_referral%(c: connection, hdr: smb_hdr, max_referral_level: cou ## smb_com_trans_mailslot smb_com_trans_pipe smb_com_trans_rap smb_com_transaction ## smb_com_transaction2 smb_com_tree_connect_andx smb_com_tree_disconnect ## smb_com_write_andx smb_error smb_get_dfs_referral smb_message +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_com_negotiate%(c: connection, hdr: smb_hdr%); ## Generated for SMB/CIFS messages of type *negotiate response*. @@ -3078,6 +3473,11 @@ event smb_com_negotiate%(c: connection, hdr: smb_hdr%); ## smb_com_trans_mailslot smb_com_trans_pipe smb_com_trans_rap smb_com_transaction ## smb_com_transaction2 smb_com_tree_connect_andx smb_com_tree_disconnect ## smb_com_write_andx smb_error smb_get_dfs_referral smb_message +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_com_negotiate_response%(c: connection, hdr: smb_hdr, dialect_index: count%); ## Generated for SMB/CIFS messages of type *setup andx*. @@ -3096,6 +3496,11 @@ event smb_com_negotiate_response%(c: connection, hdr: smb_hdr, dialect_index: co ## smb_com_transaction smb_com_transaction2 smb_com_tree_connect_andx ## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral ## smb_message +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_com_setup_andx%(c: connection, hdr: smb_hdr%); ## Generated for SMB/CIFS messages of type *generic andx*. @@ -3114,6 +3519,11 @@ event smb_com_setup_andx%(c: connection, hdr: smb_hdr%); ## smb_com_transaction smb_com_transaction2 smb_com_tree_connect_andx ## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral ## smb_message +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_com_generic_andx%(c: connection, hdr: smb_hdr%); ## Generated for SMB/CIFS messages of type *close*. @@ -3132,6 +3542,11 @@ event smb_com_generic_andx%(c: connection, hdr: smb_hdr%); ## smb_com_transaction smb_com_transaction2 smb_com_tree_connect_andx ## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral ## smb_message +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_com_close%(c: connection, hdr: smb_hdr%); ## Generated for SMB/CIFS messages of type *logoff andx*. @@ -3150,6 +3565,11 @@ event smb_com_close%(c: connection, hdr: smb_hdr%); ## smb_com_transaction smb_com_transaction2 smb_com_tree_connect_andx ## smb_com_tree_disconnect smb_com_write_andx smb_error smb_get_dfs_referral ## smb_message +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_com_logoff_andx%(c: connection, hdr: smb_hdr%); ## Generated for SMB/CIFS messages that indicate an error. This event is triggered @@ -3171,6 +3591,11 @@ event smb_com_logoff_andx%(c: connection, hdr: smb_hdr%); ## smb_com_trans_pipe smb_com_trans_rap smb_com_transaction smb_com_transaction2 ## smb_com_tree_connect_andx smb_com_tree_disconnect smb_com_write_andx ## smb_get_dfs_referral smb_message +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event smb_error%(c: connection, hdr: smb_hdr, cmd: count, cmd_str: string, data: string%); ## Generated for all DNS messages. @@ -3647,6 +4072,11 @@ event dns_end%(c: connection, msg: dns_msg%) &group="dns"; ## .. note: Bro does not support broadcast packets (as used by the DHCP protocol). ## It treats broadcast addresses just like any other and associates packets into ## transport-level flows in the same way as usual. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event dhcp_discover%(c: connection, msg: dhcp_msg, req_addr: addr%); ## Generated for DHCP messages of type *offer*. @@ -3675,6 +4105,11 @@ event dhcp_discover%(c: connection, msg: dhcp_msg, req_addr: addr%); ## .. note: Bro does not support broadcast packets (as used by the DHCP protocol). ## It treats broadcast addresses just like any other and associates packets into ## transport-level flows in the same way as usual. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event dhcp_offer%(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr%); ## Generated for DHCP messages of type *request*. @@ -3701,6 +4136,11 @@ event dhcp_offer%(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_ ## .. note: Bro does not support broadcast packets (as used by the DHCP protocol). ## It treats broadcast addresses just like any other and associates packets into ## transport-level flows in the same way as usual. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event dhcp_request%(c: connection, msg: dhcp_msg, req_addr: addr, serv_addr: addr%); ## Generated for DHCP messages of type *decline*. @@ -3723,6 +4163,11 @@ event dhcp_request%(c: connection, msg: dhcp_msg, req_addr: addr, serv_addr: add ## .. note: Bro does not support broadcast packets (as used by the DHCP protocol). ## It treats broadcast addresses just like any other and associates packets into ## transport-level flows in the same way as usual. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event dhcp_decline%(c: connection, msg: dhcp_msg%); ## Generated for DHCP messages of type *acknowledgment*. @@ -3753,6 +4198,11 @@ event dhcp_decline%(c: connection, msg: dhcp_msg%); ## .. note: Bro does not support broadcast packets (as used by the DHCP protocol). ## It treats broadcast addresses just like any other and associates packets into ## transport-level flows in the same way as usual. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event dhcp_ack%(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr%); ## Generated for DHCP messages of type *negative acknowledgment*. @@ -3775,6 +4225,11 @@ event dhcp_ack%(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_li ## .. note: Bro does not support broadcast packets (as used by the DHCP protocol). ## It treats broadcast addresses just like any other and associates packets into ## transport-level flows in the same way as usual. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event dhcp_nak%(c: connection, msg: dhcp_msg%); ## Generated for DHCP messages of type *release*. @@ -3797,6 +4252,11 @@ event dhcp_nak%(c: connection, msg: dhcp_msg%); ## .. note: Bro does not support broadcast packets (as used by the DHCP protocol). ## It treats broadcast addresses just like any other and associates packets into ## transport-level flows in the same way as usual. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event dhcp_release%(c: connection, msg: dhcp_msg%); ## Generated for DHCP messages of type *inform*. @@ -3819,6 +4279,11 @@ event dhcp_release%(c: connection, msg: dhcp_msg%); ## .. note: Bro does not support broadcast packets (as used by the DHCP protocol). ## It treats broadcast addresses just like any other and associates packets into ## transport-level flows in the same way as usual. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event dhcp_inform%(c: connection, msg: dhcp_msg%); ## Generated for HTTP requests. Bro supports persistent and pipelined HTTP sessions @@ -4263,30 +4728,55 @@ event x509_error%(c: connection, err: count%); ## ## .. bro:see:: rpc_call rpc_dialogue rpc_reply dce_rpc_bind dce_rpc_request ## dce_rpc_response rpc_timeout +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event dce_rpc_message%(c: connection, is_orig: bool, ptype: dce_rpc_ptype, msg: string%); ## TODO. ## ## .. bro:see:: rpc_call rpc_dialogue rpc_reply dce_rpc_message dce_rpc_request ## dce_rpc_response rpc_timeout +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event dce_rpc_bind%(c: connection, uuid: string%); ## TODO. ## ## .. bro:see:: rpc_call rpc_dialogue rpc_reply dce_rpc_bind dce_rpc_message ## dce_rpc_response rpc_timeout +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event dce_rpc_request%(c: connection, opnum: count, stub: string%); ## TODO. ## ## .. bro:see:: rpc_call rpc_dialogue rpc_reply dce_rpc_bind dce_rpc_message ## dce_rpc_request rpc_timeout +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event dce_rpc_response%(c: connection, opnum: count, stub: string%); ## TODO. ## ## .. bro:see:: rpc_call rpc_dialogue rpc_reply dce_rpc_bind dce_rpc_message ## dce_rpc_request dce_rpc_response rpc_timeout +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event epm_map_response%(c: connection, uuid: string, p: port, h: addr%); ## Generated for NCP requests (Netware Core Protocol). @@ -4303,6 +4793,11 @@ event epm_map_response%(c: connection, uuid: string, p: port, h: addr%); ## func: The requested function, as specified by the protocol. ## ## .. bro:see:: ncp_reply +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event ncp_request%(c: connection, frame_type: count, length: count, func: count%); ## Generated for NCP replies (Netware Core Protocol). @@ -4323,6 +4818,11 @@ event ncp_request%(c: connection, frame_type: count, length: count, func: count% ## completion_code: The replie's completion code, as specified by the protocol. ## ## .. bro:see:: ncp_request +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event ncp_reply%(c: connection, frame_type: count, length: count, req_frame: count, req_func: count, completion_code: count%); ## Generated for client-side commands on POP3 connections. @@ -4340,6 +4840,11 @@ event ncp_reply%(c: connection, frame_type: count, length: count, req_frame: cou ## ## .. bro:see:: pop3_data pop3_login_failure pop3_login_success pop3_reply ## pop3_terminate pop3_unexpected +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pop3_request%(c: connection, is_orig: bool, command: string, arg: string%); @@ -4363,6 +4868,11 @@ event pop3_request%(c: connection, is_orig: bool, ## pop3_terminate pop3_unexpected ## ## .. todo: This event is receiving odd parameters, should unify. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pop3_reply%(c: connection, is_orig: bool, cmd: string, msg: string%); ## Generated for server-side multi-lines responses on POP3 connections. POP3 @@ -4381,6 +4891,11 @@ event pop3_reply%(c: connection, is_orig: bool, cmd: string, msg: string%); ## ## .. bro:see:: pop3_login_failure pop3_login_success pop3_reply pop3_request ## pop3_terminate pop3_unexpected +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pop3_data%(c: connection, is_orig: bool, data: string%); ## Generated for errors encountered on POP3 sessions. If the POP3 analyzers finds @@ -4400,6 +4915,11 @@ event pop3_data%(c: connection, is_orig: bool, data: string%); ## ## .. bro:see:: pop3_data pop3_login_failure pop3_login_success pop3_reply pop3_request ## pop3_terminate +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pop3_unexpected%(c: connection, is_orig: bool, msg: string, detail: string%); @@ -4422,6 +4942,11 @@ event pop3_unexpected%(c: connection, is_orig: bool, ## ## .. note:: Currently, only the ``STARTLS`` command is recognized and ## triggers this. +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pop3_terminate%(c: connection, is_orig: bool, msg: string%); ## Generated for successful authentications on POP3 connections. @@ -4440,6 +4965,11 @@ event pop3_terminate%(c: connection, is_orig: bool, msg: string%); ## ## .. bro:see:: pop3_data pop3_login_failure pop3_reply pop3_request pop3_terminate ## pop3_unexpected +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pop3_login_success%(c: connection, is_orig: bool, user: string, password: string%); @@ -4459,6 +4989,11 @@ event pop3_login_success%(c: connection, is_orig: bool, ## ## .. bro:see:: pop3_data pop3_login_success pop3_reply pop3_request pop3_terminate ## pop3_unexpected +## +## .. todo:: Bro's current default configuration does not activate the protocol +## analyzer that generates this event; the corresponding script has not yet +## been ported to Bro 2.x. To still enable this event, one needs to add a +## corresponding entry to :bro:see:`dpd_config` or a DPD payload signature. event pop3_login_failure%(c: connection, is_orig: bool, user: string, password: string%); From 2d97e25eeb572cfef8e1c6089bfb06e926903fab Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 15 Dec 2011 12:27:41 -0500 Subject: [PATCH 186/964] Initial fixes for the problem of async actions with notice email extensions. --- .../notice/extend-email/hostnames.bro | 13 +++++++-- scripts/base/frameworks/notice/main.bro | 29 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/scripts/base/frameworks/notice/extend-email/hostnames.bro b/scripts/base/frameworks/notice/extend-email/hostnames.bro index a73810c726..9e0f4c612b 100644 --- a/scripts/base/frameworks/notice/extend-email/hostnames.bro +++ b/scripts/base/frameworks/notice/extend-email/hostnames.bro @@ -2,7 +2,6 @@ module Notice; -# This probably doesn't actually work due to the async lookup_addr. event Notice::notice(n: Notice::Info) &priority=10 { if ( ! n?$src && ! n?$dst ) @@ -11,22 +10,32 @@ event Notice::notice(n: Notice::Info) &priority=10 # This should only be done for notices that are being sent to email. if ( ACTION_EMAIL !in n$actions ) return; - + + # I'm not recovering gracefully from the when statements because I want + # the notice framework to detect that something has exceeded the maximum + # allowed email delay and tell the user. + local output = ""; if ( n?$src ) { + add n$email_delay_tokens["hostnames-src"]; when ( local src_name = lookup_addr(n$src) ) { output = string_cat("orig_h/src hostname: ", src_name, "\n"); n$email_body_sections[|n$email_body_sections|] = output; + delete n$email_delay_tokens["hostnames-src"]; } + timeout max_email_delay+5secs { } } if ( n?$dst ) { + add n$email_delay_tokens["hostnames-dst"]; when ( local dst_name = lookup_addr(n$dst) ) { output = string_cat("resp_h/dst hostname: ", dst_name, "\n"); n$email_body_sections[|n$email_body_sections|] = output; + delete n$email_delay_tokens["hostnames-dst"]; } + timeout max_email_delay+5secs { } } } diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index 7d98c6464c..41a44774bc 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -98,6 +98,12 @@ export { ## event and modifying the notice in place. email_body_sections: vector of string &default=vector(); + ## Adding a string "token" to this set will cause the notice framework's + ## built-in emailing functionality to delay sending the email until + ## either the token has been removed or the email has been delayed + ## for :bro:id:`max_email_delay`. + email_delay_tokens: set[string] &default=set(); + ## This field is to be provided when a notice is generated for the ## purpose of deduplicating notices. The identifier string should ## be unique for a single instance of the notice. This field should be @@ -203,6 +209,8 @@ export { const reply_to = "" &redef; ## Text string prefixed to the subject of all emails sent out. const mail_subject_prefix = "[Bro]" &redef; + ## The maximum amount of time a plugin can delay email from being sent. + const max_email_delay = 15secs &redef; ## A log postprocessing function that implements emailing the contents ## of a log upon rotation to any configured :bro:id:`Notice::mail_dest`. @@ -347,10 +355,31 @@ function email_headers(subject_desc: string, dest: string): string return header_text; } +event delay_sending_email(n: Notice::Info, dest: string, extend: bool) + { + email_notice_to(n, dest, extend); + } + function email_notice_to(n: Notice::Info, dest: string, extend: bool) { if ( reading_traces() || dest == "" ) return; + + if ( extend ) + { + if ( |n$email_delay_tokens| > 0 ) + { + # If we still are within the max_email_delay, keep delaying + if ( n$ts + max_email_delay > network_time() ) + schedule 1sec { delay_sending_email(n, dest, extend) }; + else + { + event reporter_info(network_time(), + fmt("Notice email delay tokens weren't released in time (%s).", n$email_delay_tokens), + ""); + } + } + } local email_text = email_headers(fmt("%s", n$note), dest); From f1f5719f8302285d884d3f2b44f00025828e1fb4 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 15 Dec 2011 12:41:05 -0500 Subject: [PATCH 187/964] Fixed a small bug major problem with email delay timeout catching. --- scripts/base/frameworks/notice/extend-email/hostnames.bro | 4 ++-- scripts/base/frameworks/notice/main.bro | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/base/frameworks/notice/extend-email/hostnames.bro b/scripts/base/frameworks/notice/extend-email/hostnames.bro index 9e0f4c612b..89e3a3997e 100644 --- a/scripts/base/frameworks/notice/extend-email/hostnames.bro +++ b/scripts/base/frameworks/notice/extend-email/hostnames.bro @@ -25,7 +25,7 @@ event Notice::notice(n: Notice::Info) &priority=10 n$email_body_sections[|n$email_body_sections|] = output; delete n$email_delay_tokens["hostnames-src"]; } - timeout max_email_delay+5secs { } + timeout max_email_delay { } } if ( n?$dst ) { @@ -36,6 +36,6 @@ event Notice::notice(n: Notice::Info) &priority=10 n$email_body_sections[|n$email_body_sections|] = output; delete n$email_delay_tokens["hostnames-dst"]; } - timeout max_email_delay+5secs { } + timeout max_email_delay { } } } diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index 41a44774bc..ec2f0133b5 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -369,7 +369,7 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool) { if ( |n$email_delay_tokens| > 0 ) { - # If we still are within the max_email_delay, keep delaying + # If we still are within the max_email_delay, keep delaying. if ( n$ts + max_email_delay > network_time() ) schedule 1sec { delay_sending_email(n, dest, extend) }; else From cb904cec4fd908c0d9b7a8f88722947cf0a3e11d Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 15 Dec 2011 12:46:15 -0500 Subject: [PATCH 188/964] Ugh, still major failure. I'm just cutting the timeout handling for now. --- scripts/base/frameworks/notice/extend-email/hostnames.bro | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/base/frameworks/notice/extend-email/hostnames.bro b/scripts/base/frameworks/notice/extend-email/hostnames.bro index 89e3a3997e..945729846c 100644 --- a/scripts/base/frameworks/notice/extend-email/hostnames.bro +++ b/scripts/base/frameworks/notice/extend-email/hostnames.bro @@ -25,7 +25,6 @@ event Notice::notice(n: Notice::Info) &priority=10 n$email_body_sections[|n$email_body_sections|] = output; delete n$email_delay_tokens["hostnames-src"]; } - timeout max_email_delay { } } if ( n?$dst ) { @@ -36,6 +35,5 @@ event Notice::notice(n: Notice::Info) &priority=10 n$email_body_sections[|n$email_body_sections|] = output; delete n$email_delay_tokens["hostnames-dst"]; } - timeout max_email_delay { } } } From 667dcb251a540d41e6f6631dc13f88732ee89534 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 15 Dec 2011 12:51:14 -0500 Subject: [PATCH 189/964] Working around a problem with setting default container types. --- scripts/base/frameworks/notice/main.bro | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index ec2f0133b5..f6800b7697 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -102,7 +102,7 @@ export { ## built-in emailing functionality to delay sending the email until ## either the token has been removed or the email has been delayed ## for :bro:id:`max_email_delay`. - email_delay_tokens: set[string] &default=set(); + email_delay_tokens: set[string] &optional; ## This field is to be provided when a notice is generated for the ## purpose of deduplicating notices. The identifier string should @@ -503,6 +503,9 @@ function apply_policy(n: Notice::Info) if ( ! n?$actions ) n$actions = set(); + + if ( ! n?$email_delay_tokens ) + n$email_delay_tokens = set(); if ( ! n?$policy_items ) n$policy_items = set(); From f302f2f3f26eb5b2bebb83d27a849f262c82e513 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 15 Dec 2011 12:16:42 -0600 Subject: [PATCH 190/964] Fix &default fields in records not being initialized in coerced assignments. Addresses #722 --- src/Expr.cc | 9 ++++++++- testing/btest/Baseline/bifs.records_fields/out | 4 ++-- .../language.record-default-coercion/out | 4 ++++ .../output | 4 ++-- .../test.failure.log | 2 +- .../test.success.log | 2 +- .../sender.test.failure.log | 6 +++--- .../sender.test.log | 10 +++++----- .../sender.test.success.log | 4 ++-- .../manager-1.notice.log | 2 +- .../notice.log | 4 ++-- .../manager-1.notice.log | 2 +- .../manager-1.notice.log | 2 +- .../notice.log | 2 +- .../btest/language/record-default-coercion.bro | 18 ++++++++++++++++++ 15 files changed, 52 insertions(+), 23 deletions(-) create mode 100644 testing/btest/Baseline/language.record-default-coercion/out create mode 100644 testing/btest/language/record-default-coercion.bro diff --git a/src/Expr.cc b/src/Expr.cc index f6d1fc568e..7e833f8397 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -4053,7 +4053,14 @@ Val* RecordCoerceExpr::Fold(Val* v) const val->Assign(i, rhs); } else - val->Assign(i, 0); + { + const Attr* def = + Type()->AsRecordType()->FieldDecl(i)->FindAttr(ATTR_DEFAULT); + if ( def ) + val->Assign(i, def->AttrExpr()->Eval(0)); + else + val->Assign(i, 0); + } } return val; diff --git a/testing/btest/Baseline/bifs.records_fields/out b/testing/btest/Baseline/bifs.records_fields/out index b221230fc0..0d52e64255 100644 --- a/testing/btest/Baseline/bifs.records_fields/out +++ b/testing/btest/Baseline/bifs.records_fields/out @@ -1,6 +1,6 @@ -[a=42, b=, c=, d=Bar] +[a=42, b=Foo, c=, d=Bar] { -[b] = [type_name=record, log=F, value=, default_val=Foo], +[b] = [type_name=record, log=F, value=Foo, default_val=Foo], [d] = [type_name=record, log=T, value=Bar, default_val=], [c] = [type_name=record, log=F, value=, default_val=], [a] = [type_name=record, log=F, value=42, default_val=] diff --git a/testing/btest/Baseline/language.record-default-coercion/out b/testing/btest/Baseline/language.record-default-coercion/out new file mode 100644 index 0000000000..2f0e6cd17d --- /dev/null +++ b/testing/btest/Baseline/language.record-default-coercion/out @@ -0,0 +1,4 @@ +[a=13, c=13, v=[]] +0 +[a=13, c=13, v=[test]] +1 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.events/output b/testing/btest/Baseline/scripts.base.frameworks.logging.events/output index c3dbf607a6..297b3dabd2 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.events/output +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.events/output @@ -1,2 +1,2 @@ -[t=1299718502.96511, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=success, country=] -[t=1299718502.96511, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=failure, country=US] +[t=1323970492.986366, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=success, country=unknown] +[t=1323970492.986366, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=failure, country=US] diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log index ba688d7843..566c8ac0c0 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log @@ -2,4 +2,4 @@ #path test.failure #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.923545 1.2.3.4 1234 2.3.4.5 80 failure US +1323970640.453742 1.2.3.4 1234 2.3.4.5 80 failure US diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log index 7a91b1a2d9..c88f37a20e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log @@ -2,4 +2,4 @@ #path test.success #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.923545 1.2.3.4 1234 2.3.4.5 80 success - +1323970640.453742 1.2.3.4 1234 2.3.4.5 80 success unknown diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log index aba9fdddd9..d93021273b 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log @@ -2,6 +2,6 @@ #path test.failure #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure US -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure UK -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure MX +1323970700.744867 1.2.3.4 1234 2.3.4.5 80 failure US +1323970700.744867 1.2.3.4 1234 2.3.4.5 80 failure UK +1323970700.744867 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log index b928c37685..60b91e524b 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log @@ -2,8 +2,8 @@ #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 success - -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure US -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure UK -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 success BR -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure MX +1323970700.744867 1.2.3.4 1234 2.3.4.5 80 success unknown +1323970700.744867 1.2.3.4 1234 2.3.4.5 80 failure US +1323970700.744867 1.2.3.4 1234 2.3.4.5 80 failure UK +1323970700.744867 1.2.3.4 1234 2.3.4.5 80 success BR +1323970700.744867 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log index a951c6ed1a..7fd2d100ec 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log @@ -2,5 +2,5 @@ #path test.success #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 success - -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 success BR +1323970700.744867 1.2.3.4 1234 2.3.4.5 80 success unknown +1323970700.744867 1.2.3.4 1234 2.3.4.5 80 success BR diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log index f5df2e96f3..b1454abdb0 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log @@ -2,4 +2,4 @@ #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network #types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet -1316952194.679491 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 6 3600.000000 - - - - - - 1.2.3.4 - - +1323971217.669520 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log index 33745500e0..5c02d541de 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log @@ -2,5 +2,5 @@ #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network #types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet -1316952223.891502 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 3/2 - 1.2.3.4 - - 3 bro Notice::ACTION_LOG 6 3600.000000 - - - - - - 1.2.3.4 - - -1316952223.891502 - - - - - Test_Notice Threshold crossed by metric_index(host=6.5.4.3) 2/2 - 6.5.4.3 - - 2 bro Notice::ACTION_LOG 6 3600.000000 - - - - - - 6.5.4.3 - - +1323971243.445213 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 3/2 - 1.2.3.4 - - 3 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - - +1323971243.445213 - - - - - Test_Notice Threshold crossed by metric_index(host=6.5.4.3) 2/2 - 6.5.4.3 - - 2 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - 6.5.4.3 - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log index 0662c13294..07f3ff0f33 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log @@ -2,4 +2,4 @@ #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network #types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet -1316952264.931290 - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 6 3600.000000 - - - - - - - - - +1323971344.565540 - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log index 6e0214b7d3..99f828368d 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log @@ -2,4 +2,4 @@ #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network #types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet -1316950574.408256 - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 6 3600.000000 - - - - - - - - - +1323971387.261655 - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log index 6b4c925e0f..ad749ed9cd 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log @@ -2,4 +2,4 @@ #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude #types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double -1316950497.513136 - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 6 3600.000000 - - - - - - +1323971422.944442 - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 6 3600.000000 F - - - - - diff --git a/testing/btest/language/record-default-coercion.bro b/testing/btest/language/record-default-coercion.bro new file mode 100644 index 0000000000..7e717c39e2 --- /dev/null +++ b/testing/btest/language/record-default-coercion.bro @@ -0,0 +1,18 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +type MyRecord: record { + a: count &default=13; + c: count; + v: vector of string &default=vector(); +}; + +event bro_init() + { + local r: MyRecord = [$c=13]; + print r; + print |r$v|; + r$v[|r$v|] = "test"; + print r; + print |r$v|; + } From d04558dc45468e77846d1639795722ddf9f4ecb3 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 15 Dec 2011 13:00:29 -0600 Subject: [PATCH 191/964] Add search box to Broxygen docs (fixes #726). --- doc/_templates/layout.html | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html index cb6181361d..a4775f5870 100644 --- a/doc/_templates/layout.html +++ b/doc/_templates/layout.html @@ -73,6 +73,19 @@ {% endif %} + {%- if pagename != "search" %} + + + {%- endif %} + From fc9a38a796ad2d5444e44d7eaef0b9069786d262 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 15 Dec 2011 13:01:23 -0600 Subject: [PATCH 192/964] Cleanup some misc Broxygen css/js stuff. --- doc/_static/broxygen-extra.css | 9 +- doc/_static/default.css_t | 309 --------------------------------- doc/_static/showhide.js | 64 ------- 3 files changed, 5 insertions(+), 377 deletions(-) delete mode 100644 doc/_static/default.css_t delete mode 100644 doc/_static/showhide.js diff --git a/doc/_static/broxygen-extra.css b/doc/_static/broxygen-extra.css index d873e286bc..ec240cec7b 100644 --- a/doc/_static/broxygen-extra.css +++ b/doc/_static/broxygen-extra.css @@ -1,7 +1,3 @@ -.highlight { - background-color: #ffffff; -} - h1 { font-weight: bold; font-size: 32px; @@ -13,3 +9,8 @@ h1 { color: #000; border-bottom: 0px; } + +th.field-name +{ + white-space:nowrap; +} diff --git a/doc/_static/default.css_t b/doc/_static/default.css_t deleted file mode 100644 index 34c2157b25..0000000000 --- a/doc/_static/default.css_t +++ /dev/null @@ -1,309 +0,0 @@ -/* - * default.css_t - * ~~~~~~~~~~~~~ - * - * Sphinx stylesheet -- default theme. - * - * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. - * :license: BSD, see LICENSE for details. - * - */ - -@import url("basic.css"); - -/* -- page layout ----------------------------------------------------------- */ - -body { - font-family: {{ theme_bodyfont }}; - font-size: 100%; - background-color: {{ theme_footerbgcolor }}; - color: #000; - margin: 0; - padding: 0; -} - -div.document { - background-color: {{ theme_sidebarbgcolor }}; -} - -div.documentwrapper { - float: left; - width: 100%; -} - -div.bodywrapper { - margin: 0 0 0 {{ theme_sidebarwidth|toint }}px; -} - -div.body { - background-color: {{ theme_bgcolor }}; - color: {{ theme_textcolor }}; - padding: 0 20px 30px 20px; -} - -{%- if theme_rightsidebar|tobool %} -div.bodywrapper { - margin: 0 {{ theme_sidebarwidth|toint }}px 0 0; -} -{%- endif %} - -div.footer { - color: {{ theme_footertextcolor }}; - background-color: {{ theme_footerbgcolor }}; - width: 100%; - padding: 9px 0 9px 0; - text-align: center; - font-size: 75%; -} - -div.footer a { - color: {{ theme_footertextcolor }}; - text-decoration: underline; -} - -div.related { - background-color: {{ theme_relbarbgcolor }}; - line-height: 30px; - color: {{ theme_relbartextcolor }}; -} - -div.related a { - color: {{ theme_relbarlinkcolor }}; -} - -div.sphinxsidebar { - {%- if theme_stickysidebar|tobool %} - top: 30px; - bottom: 0; - margin: 0; - position: fixed; - overflow: auto; - height: auto; - {%- endif %} - {%- if theme_rightsidebar|tobool %} - float: right; - {%- if theme_stickysidebar|tobool %} - right: 0; - {%- endif %} - {%- endif %} -} - -{%- if theme_stickysidebar|tobool %} -/* this is nice, but it it leads to hidden headings when jumping - to an anchor */ -/* -div.related { - position: fixed; -} - -div.documentwrapper { - margin-top: 30px; -} -*/ -{%- endif %} - -div.sphinxsidebar h3 { - font-family: {{ theme_bodyfont }}; - color: {{ theme_sidebartextcolor }}; - font-size: 1.4em; - font-weight: normal; - margin: 0; - padding: 0; -} - -div.sphinxsidebar h3 a { - color: {{ theme_sidebartextcolor }}; -} - -div.sphinxsidebar h4 { - font-family: {{ theme_bodyfont }}; - color: {{ theme_sidebartextcolor }}; - font-size: 1.3em; - font-weight: normal; - margin: 5px 0 0 0; - padding: 0; -} - -div.sphinxsidebar p { - color: {{ theme_sidebartextcolor }}; -} - -div.sphinxsidebar p.topless { - margin: 5px 10px 10px 10px; -} - -div.sphinxsidebar ul { - margin: 10px; - padding: 0; - color: {{ theme_sidebartextcolor }}; -} - -div.sphinxsidebar a { - color: {{ theme_sidebarlinkcolor }}; -} - -div.sphinxsidebar input { - border: 1px solid {{ theme_sidebarlinkcolor }}; - font-family: sans-serif; - font-size: 1em; -} - -{% if theme_collapsiblesidebar|tobool %} -/* for collapsible sidebar */ -div#sidebarbutton { - background-color: {{ theme_sidebarbtncolor }}; -} -{% endif %} - -/* -- hyperlink styles ------------------------------------------------------ */ - -a { - color: {{ theme_linkcolor }}; - text-decoration: none; -} - -a:visited { - color: {{ theme_visitedlinkcolor }}; - text-decoration: none; -} - -{% if theme_externalrefs|tobool %} -a.external { - text-decoration: none; - border-bottom: 1px dashed {{ theme_linkcolor }}; -} - -a.external:hover { - text-decoration: none; - border-bottom: none; -} - -a.external:visited { - text-decoration: none; - border-bottom: 1px dashed {{ theme_visitedlinkcolor }}; -} -{% endif %} - -/* -- body styles ----------------------------------------------------------- */ - -div.body h1, -div.body h2, -div.body h3, -div.body h4, -div.body h5, -div.body h6 { - font-family: {{ theme_bodyfont }}; - background-color: #ffffff; - font-weight: normal; - color: {{ theme_headtextcolor }}; - border-bottom: 1px solid #aaa; - margin: 20px -20px 10px -20px; - padding: 3px 0 3px 10px; -} - -div.body h1 { - font-family: {{ theme_headfont }}; - text-align: center; - border-bottom: none; -} - -div.body h1 { margin-top: 0; font-size: 200%; } -div.body h2 { font-size: 160%; } -div.body h3 { font-size: 140%; } -div.body h4 { font-size: 120%; } -div.body h5 { font-size: 110%; } -div.body h6 { font-size: 100%; } - -a.headerlink { - color: {{ theme_headlinkcolor }}; - font-size: 0.8em; - padding: 0 4px 0 4px; - text-decoration: none; -} - -a.headerlink:hover { - background-color: {{ theme_headlinkcolor }}; - color: white; -} - -div.admonition p.admonition-title + p { - display: inline; -} - -div.admonition p { - margin-bottom: 5px; -} - -div.admonition pre { - margin-bottom: 5px; -} - -div.admonition ul, div.admonition ol { - margin-bottom: 5px; -} - -div.note { - background-color: #eee; - border: 1px solid #ccc; -} - -div.seealso { - background-color: #ffc; - border: 1px solid #ff6; -} - -div.warning { - background-color: #ffe4e4; - border: 1px solid #f66; -} - -p.admonition-title { - display: inline; -} - -p.admonition-title:after { - content: ":"; -} - -pre { - padding: 5px; - background-color: {{ theme_codebgcolor }}; - color: {{ theme_codetextcolor }}; - line-height: 120%; - border: 1px solid #ac9; - border-left: none; - border-right: none; -} - -tt { - background-color: #ecf0f3; - padding: 0 1px 0 1px; - font-size: 0.95em; -} - -th { - background-color: #ede; -} - -.warning tt { - background: #efc2c2; -} - -.note tt { - background: #d6d6d6; -} - -.viewcode-back { - font-family: {{ theme_bodyfont }}; -} - -div.viewcode-block:target { - background-color: #f4debf; - border-top: 1px solid #ac9; - border-bottom: 1px solid #ac9; -} - -th.field-name -{ - white-space:nowrap; -} diff --git a/doc/_static/showhide.js b/doc/_static/showhide.js deleted file mode 100644 index d6a8923143..0000000000 --- a/doc/_static/showhide.js +++ /dev/null @@ -1,64 +0,0 @@ -// make literal blocks corresponding to identifier initial values -// hidden by default -$(document).ready(function() { - - var showText='(Show Value)'; - var hideText='(Hide Value)'; - - var is_visible = false; - - // select field-list tables that come before a literal block - tables = $('.highlight-python').prev('table.docutils.field-list'); - - tables.find('th.field-name').filter(function(index) { - return $(this).html() == "Default :"; - }).next().append(''+showText+''); - - // hide all literal blocks that follow a field-list table - tables.next('.highlight-python').hide(); - - // register handler for clicking a "toggle" link - $('a.toggleLink').click(function() { - is_visible = !is_visible; - - $(this).html( (!is_visible) ? showText : hideText); - - // the link is inside a
and the next - // literal block after the table is the literal block that we want - // to show/hide - $(this).parent().parent().parent().parent().next('.highlight-python').slideToggle('fast'); - - // override default link behavior - return false; - }); -}); - -// make "Private Interface" sections hidden by default -$(document).ready(function() { - - var showText='Show Private Interface (for internal use)'; - var hideText='Hide Private Interface'; - - var is_visible = false; - - // insert show/hide links - $('#private-interface').children(":first-child").after(''+showText+''); - - // wrap all sub-sections in a new div that can be hidden/shown - $('#private-interface').children(".section").wrapAll('
'); - - // hide the given class - $('.private').hide(); - - // register handler for clicking a "toggle" link - $('a.privateToggle').click(function() { - is_visible = !is_visible; - - $(this).html( (!is_visible) ? showText : hideText); - - $('.private').slideToggle('fast'); - - // override default link behavior - return false; - }); -}); From b66c73baaa271a715e79a6e816383add06a16138 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 15 Dec 2011 15:57:42 -0500 Subject: [PATCH 193/964] Fixed more bugs with delayed emails. --- .../frameworks/notice/extend-email/hostnames.bro | 16 ++++++++++++---- scripts/base/frameworks/notice/main.bro | 9 +++++++-- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/scripts/base/frameworks/notice/extend-email/hostnames.bro b/scripts/base/frameworks/notice/extend-email/hostnames.bro index 945729846c..8970ef8d92 100644 --- a/scripts/base/frameworks/notice/extend-email/hostnames.bro +++ b/scripts/base/frameworks/notice/extend-email/hostnames.bro @@ -2,6 +2,14 @@ module Notice; +function lookup_addr_wrapper(n: Info, a: addr): string + { + return when ( local name = lookup_addr(a) ) + { + return name; + } + } + event Notice::notice(n: Notice::Info) &priority=10 { if ( ! n?$src && ! n?$dst ) @@ -19,9 +27,9 @@ event Notice::notice(n: Notice::Info) &priority=10 if ( n?$src ) { add n$email_delay_tokens["hostnames-src"]; - when ( local src_name = lookup_addr(n$src) ) + when ( local src_name = lookup_addr_wrapper(n, n$src) ) { - output = string_cat("orig_h/src hostname: ", src_name, "\n"); + output = string_cat("orig/src hostname: ", src_name, "\n"); n$email_body_sections[|n$email_body_sections|] = output; delete n$email_delay_tokens["hostnames-src"]; } @@ -29,9 +37,9 @@ event Notice::notice(n: Notice::Info) &priority=10 if ( n?$dst ) { add n$email_delay_tokens["hostnames-dst"]; - when ( local dst_name = lookup_addr(n$dst) ) + when ( local dst_name = lookup_addr_wrapper(n, n$dst) ) { - output = string_cat("resp_h/dst hostname: ", dst_name, "\n"); + output = string_cat("resp/dst hostname: ", dst_name, "\n"); n$email_body_sections[|n$email_body_sections|] = output; delete n$email_delay_tokens["hostnames-dst"]; } diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index f6800b7697..1e687e3ce9 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -96,7 +96,7 @@ export { ## expand on notices that are being emailed. The normal way to add text ## is to extend the vector by handling the :bro:id:`Notice::notice` ## event and modifying the notice in place. - email_body_sections: vector of string &default=vector(); + email_body_sections: vector of string &optional; ## Adding a string "token" to this set will cause the notice framework's ## built-in emailing functionality to delay sending the email until @@ -371,7 +371,10 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool) { # If we still are within the max_email_delay, keep delaying. if ( n$ts + max_email_delay > network_time() ) + { schedule 1sec { delay_sending_email(n, dest, extend) }; + return; + } else { event reporter_info(network_time(), @@ -503,7 +506,9 @@ function apply_policy(n: Notice::Info) if ( ! n?$actions ) n$actions = set(); - + + if ( ! n?$email_body_sections ) + n$email_body_sections = vector(); if ( ! n?$email_delay_tokens ) n$email_delay_tokens = set(); From 0b8b14a0ed7c5fbf3ba408ab5f6cd89e4c68f2d7 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 15 Dec 2011 15:59:51 -0500 Subject: [PATCH 194/964] Fixed major bug with cluster synchronization (it was broken!) --- scripts/base/frameworks/cluster/setup-connections.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base/frameworks/cluster/setup-connections.bro b/scripts/base/frameworks/cluster/setup-connections.bro index 059b984d61..84d9767875 100644 --- a/scripts/base/frameworks/cluster/setup-connections.bro +++ b/scripts/base/frameworks/cluster/setup-connections.bro @@ -41,7 +41,7 @@ event bro_init() &priority=9 { if ( n$node_type == WORKER && n$proxy == node ) Communication::nodes[i] = - [$host=n$ip, $connect=F, $class=i, $events=worker2proxy_events]; + [$host=n$ip, $connect=F, $class=i, $sync=T, $auth=T, $events=worker2proxy_events]; # accepts connections from the previous one. # (This is not ideal for setups with many proxies) From 41a68933ebd84c6bf75487f589da3e0af3c83b1e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 16 Dec 2011 02:59:39 -0800 Subject: [PATCH 195/964] Updating submodule(s). --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- cmake | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aux/binpac b/aux/binpac index 0f99acfbf6..a29ccf131c 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 0f99acfbf6205830f0db699a75554262c26427f9 +Subproject commit a29ccf131cc44da73b8e8c8d78f7939b34d154be diff --git a/aux/bro-aux b/aux/bro-aux index 1a7a9357fb..3ee84004b3 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 1a7a9357fba88a43c90a39d8d72b42fa53b89b75 +Subproject commit 3ee84004b351755d8156378093abdf01a6541c68 diff --git a/aux/broccoli b/aux/broccoli index a1a03c6868..a287654121 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit a1a03c686866bd30ee086ff933128055a20ebd56 +Subproject commit a287654121c83cf0a0b2845fb558be07afe1391a diff --git a/aux/broctl b/aux/broctl index 03d39aa5d4..c4405333fd 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 03d39aa5d4ab24cd9b8e404a9ceb583d5270444c +Subproject commit c4405333fdb684925168ecd32659395c9a6f47e9 diff --git a/cmake b/cmake index 44f2985475..0c0a469768 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 44f2985475e5ff6cc9061683e21ef4b184bdfc7e +Subproject commit 0c0a4697687df7f17c09391a1d0d95b25297a662 From 8399d28c2ef9bc1268bece67f92b67e152ab9d40 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Fri, 16 Dec 2011 10:59:30 -0500 Subject: [PATCH 196/964] The hostname notice email extension works now. --- .../notice/extend-email/hostnames.bro | 26 +++++++++---------- scripts/base/frameworks/notice/main.bro | 3 ++- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/scripts/base/frameworks/notice/extend-email/hostnames.bro b/scripts/base/frameworks/notice/extend-email/hostnames.bro index 8970ef8d92..d0f52e56ca 100644 --- a/scripts/base/frameworks/notice/extend-email/hostnames.bro +++ b/scripts/base/frameworks/notice/extend-email/hostnames.bro @@ -2,13 +2,11 @@ module Notice; -function lookup_addr_wrapper(n: Info, a: addr): string - { - return when ( local name = lookup_addr(a) ) - { - return name; - } - } +# We have to store references to the notices here because the when statement +# clones the frame which doesn't give us access to modify values outside +# of it's execution scope. (we get a clone of the notice instead of a +# reference to the original notice) +global tmp_notice_storage: table[string] of Notice::Info &create_expire=max_email_delay+10secs; event Notice::notice(n: Notice::Info) &priority=10 { @@ -22,26 +20,28 @@ event Notice::notice(n: Notice::Info) &priority=10 # I'm not recovering gracefully from the when statements because I want # the notice framework to detect that something has exceeded the maximum # allowed email delay and tell the user. + local uid = unique_id(""); + tmp_notice_storage[uid] = n; local output = ""; if ( n?$src ) { add n$email_delay_tokens["hostnames-src"]; - when ( local src_name = lookup_addr_wrapper(n, n$src) ) + when ( local src_name = lookup_addr(n$src) ) { output = string_cat("orig/src hostname: ", src_name, "\n"); - n$email_body_sections[|n$email_body_sections|] = output; - delete n$email_delay_tokens["hostnames-src"]; + tmp_notice_storage[uid]$email_body_sections[|tmp_notice_storage[uid]$email_body_sections|] = output; + delete tmp_notice_storage[uid]$email_delay_tokens["hostnames-src"]; } } if ( n?$dst ) { add n$email_delay_tokens["hostnames-dst"]; - when ( local dst_name = lookup_addr_wrapper(n, n$dst) ) + when ( local dst_name = lookup_addr(n$dst) ) { output = string_cat("resp/dst hostname: ", dst_name, "\n"); - n$email_body_sections[|n$email_body_sections|] = output; - delete n$email_delay_tokens["hostnames-dst"]; + tmp_notice_storage[uid]$email_body_sections[|tmp_notice_storage[uid]$email_body_sections|] = output; + delete tmp_notice_storage[uid]$email_delay_tokens["hostnames-dst"]; } } } diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index 1e687e3ce9..21e565cbea 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -409,9 +409,10 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool) # Add the extended information if it's requested. if ( extend ) { + email_text = string_cat(email_text, "\nEmail Extensions\n"); + email_text = string_cat(email_text, "----------------\n"); for ( i in n$email_body_sections ) { - email_text = string_cat(email_text, "******************\n"); email_text = string_cat(email_text, n$email_body_sections[i], "\n"); } } From 366a5de60679f7ae1f5f766bff37648916a0cd3a Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 16 Dec 2011 11:13:20 -0600 Subject: [PATCH 197/964] Minor doc tweaks to init-bare.bro. --- scripts/base/init-bare.bro | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 3ad114d6dc..7c4029cd7c 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -321,7 +321,7 @@ type packet: record { ## .. todo:: We need this type definition only for declaring builtin functions via ## ``bifcl``. We should extend ``bifcl`` to understand composite types directly and ## then remove this alias. -type var_sizes: table[string] of count; ##< indexed by var's name, returns size +type var_sizes: table[string] of count; ## Meta-information about a script-level identifier. ## @@ -375,8 +375,9 @@ type record_field_table: table[string] of record_field; ## Set of BPF capture filters to use for capturing, indexed by a user-definable ## ID (which must be unique). If Bro is *not* configured to examine -## :bro:id:`all_packets`, all packets matching at least one of the filters in this -## table (and all in :bro:id:`restrict_filters`) will be analyzed. +## :bro:id:`PacketFilter::all_packets`, all packets matching at least +## one of the filters in this table (and all in :bro:id:`restrict_filters`) +## will be analyzed. ## ## .. bro:see:: PacketFilter PacketFilter::all_packets ## PacketFilter::unrestricted_filter restrict_filters @@ -530,7 +531,7 @@ type geo_location: record { ## Computed entropy values. The record captures a number of measures that are ## computed in parallel. See `A Pseudorandom Number Sequence Test Program -## ` for more information, Bro uses the same +## `_ for more information, Bro uses the same ## code. ## ## .. bro:see:: entropy_test_add entropy_test_finish entropy_test_init find_entropy @@ -1736,8 +1737,8 @@ type X509: record { not_valid_after: time; ##< Timestamp after when certificate is not valid. }; -## This is indexed with the CA's name and yields a DER (binary) encoded certificate. -## todo::Is this still used? +# This is indexed with the CA's name and yields a DER (binary) encoded certificate. +# todo::Is this still used? # const root_ca_certs: table[string] of string = {} &redef; ## HTTP session statistics. @@ -2207,7 +2208,7 @@ const remote_trace_sync_interval = 0 secs &redef; ## .. bro:see:: remote_trace_sync_interval const remote_trace_sync_peers = 0 &redef; -## Whether for :bro:attr:`synchronized` state to send the old value as a +## Whether for :bro:attr:`&synchronized` state to send the old value as a ## consistency check. const remote_check_sync_consistency = F &redef; From 3b91df8cf54fd477d4963dc30e2495892b678401 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 16 Dec 2011 11:21:49 -0600 Subject: [PATCH 198/964] Allow Broxygen markup "##<" for more general use. --- doc/scripts/example.bro | 10 +++++----- src/BroDocObj.cc | 3 +++ src/BroDocObj.h | 14 ++++++++++++++ src/scan.l | 5 +++++ .../Baseline/doc.autogen-reST-example/example.rst | 8 +++++--- 5 files changed, 32 insertions(+), 8 deletions(-) diff --git a/doc/scripts/example.bro b/doc/scripts/example.bro index e7c86f4bd3..20591072f2 100644 --- a/doc/scripts/example.bro +++ b/doc/scripts/example.bro @@ -1,5 +1,5 @@ -##! This is an example script that demonstrates how to document. Comments -##! of the form ``##!`` are for the script summary. The contents of +##! This is an example script that demonstrates documentation features. +##! Comments of the form ``##!`` are for the script summary. The contents of ##! these comments are transferred directly into the auto-generated ##! `reStructuredText `_ ##! (reST) document's summary section. @@ -22,8 +22,8 @@ # field comments, it's necessary to disambiguate the field with # which a comment associates: e.g. "##<" can be used on the same line # as a field to signify the comment relates to it and not the -# following field. "##<" is not meant for general use, just -# record/enum fields. +# following field. "##<" can also be used more generally in any +# variable declarations to associate with the last-declared identifier. # # Generally, the auto-doc comments (##) are associated with the # next declaration/identifier found in the script, but the doc framework @@ -151,7 +151,7 @@ export { const an_option: set[addr, addr, string] &redef; # default initialization will be self-documenting - const option_with_init = 0.01 secs &redef; + const option_with_init = 0.01 secs &redef; ##< More docs can be added here. ############## state variables ############ # right now, I'm defining this as any global diff --git a/src/BroDocObj.cc b/src/BroDocObj.cc index d9fe16632b..12753ea15d 100644 --- a/src/BroDocObj.cc +++ b/src/BroDocObj.cc @@ -4,9 +4,12 @@ #include "ID.h" #include "BroDocObj.h" +BroDocObj* BroDocObj::last = 0; + BroDocObj::BroDocObj(const ID* id, std::list*& reST, bool is_fake) { + last = this; broID = id; reST_doc_strings = reST; reST = 0; diff --git a/src/BroDocObj.h b/src/BroDocObj.h index 0ad96afa86..cb512f8cda 100644 --- a/src/BroDocObj.h +++ b/src/BroDocObj.h @@ -103,6 +103,20 @@ public: */ int LongestShortDescLen() const; + /** + * Adds a reST documentation string to this BroDocObj's list. + * @param s the documentation string to append. + */ + void AddDocString(const std::string& s) + { + if ( ! reST_doc_strings ) + reST_doc_strings = new std::list(); + reST_doc_strings->push_back(s); + FormulateShortDesc(); + } + + static BroDocObj* last; + protected: std::list* reST_doc_strings; std::list short_desc; diff --git a/src/scan.l b/src/scan.l index b167d69943..933eecec15 100644 --- a/src/scan.l +++ b/src/scan.l @@ -201,6 +201,11 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) } } +##<.* { + if ( generate_documentation && BroDocObj::last ) + BroDocObj::last->AddDocString(canon_doc_comment(yytext + 3)); +} + ##.* { if ( generate_documentation && (yytext[2] != '#') ) { diff --git a/testing/btest/Baseline/doc.autogen-reST-example/example.rst b/testing/btest/Baseline/doc.autogen-reST-example/example.rst index fbda81771b..1902e2754b 100644 --- a/testing/btest/Baseline/doc.autogen-reST-example/example.rst +++ b/testing/btest/Baseline/doc.autogen-reST-example/example.rst @@ -7,8 +7,8 @@ example.bro Overview -------- -This is an example script that demonstrates how to document. Comments -of the form ``##!`` are for the script summary. The contents of +This is an example script that demonstrates documentation features. +Comments of the form ``##!`` are for the script summary. The contents of these comments are transferred directly into the auto-generated `reStructuredText `_ (reST) document's summary section. @@ -34,7 +34,7 @@ Options ============================================================================ ====================================== :bro:id:`Example::an_option`: :bro:type:`set` :bro:attr:`&redef` add documentation for "an_option" here -:bro:id:`Example::option_with_init`: :bro:type:`interval` :bro:attr:`&redef` +:bro:id:`Example::option_with_init`: :bro:type:`interval` :bro:attr:`&redef` More docs can be added here. ============================================================================ ====================================== State Variables @@ -128,6 +128,8 @@ Options :Attributes: :bro:attr:`&redef` :Default: ``10.0 msecs`` + More docs can be added here. + State Variables ~~~~~~~~~~~~~~~ .. bro:id:: Example::a_var From 8394829fb1b1004b0c86225347ffbf2032c85ef5 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 16 Dec 2011 11:59:55 -0600 Subject: [PATCH 199/964] Broxygen fix for function parameter recognition; better than 80b2451. --- src/scan.l | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scan.l b/src/scan.l index 933eecec15..623e0d2ed6 100644 --- a/src/scan.l +++ b/src/scan.l @@ -167,7 +167,7 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) return TOK_POST_DOC; } -##{OWS}{ID}:[^:].* { +##{OWS}{ID}:{WS}.* { const char* id_start = skip_whitespace(yytext + 2); yylval.str = copy_string(canon_doc_func_param(id_start).c_str()); return TOK_DOC; @@ -181,7 +181,7 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) } } -##{OWS}{ID}:[^:].* { +##{OWS}{ID}:{WS}.* { if ( generate_documentation ) { // Comment is documenting either a function parameter or return type, From bd5dadf427b3dcbd5832274617addfd4d4a21068 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 16 Dec 2011 11:24:52 -0800 Subject: [PATCH 200/964] change software framework interface again. At the moment everything should worl. --- scripts/base/frameworks/software/main.bro | 63 +++++++++---------- scripts/policy/protocols/ftp/software.bro | 2 +- .../policy/protocols/http/detect-webapps.bro | 3 +- .../http/software-browser-plugins.bro | 4 +- scripts/policy/protocols/http/software.bro | 8 +-- scripts/policy/protocols/smtp/software.bro | 2 +- scripts/policy/protocols/ssh/software.bro | 4 +- 7 files changed, 43 insertions(+), 43 deletions(-) diff --git a/scripts/base/frameworks/software/main.bro b/scripts/base/frameworks/software/main.bro index 5cfd249982..e451ebd218 100644 --- a/scripts/base/frameworks/software/main.bro +++ b/scripts/base/frameworks/software/main.bro @@ -36,9 +36,10 @@ export { unparsed_version: string; }; + ## Record that is used to add and log software information. type Info: record { ## The time at which the software was first detected. - ts: time &log; + ts: time &log &optional; ## The IP address detected running the software. host: addr &log; ## The Port on which the software is running. Only sensible for server software. @@ -48,27 +49,12 @@ export { ## The type of software detected (e.g. WEB_SERVER) software_type: Type &log &default=UNKNOWN; ## Name of the software (e.g. Apache) - name: string &log; + name: string &log &optional; ## Version of the software - version: Version &log; + version: Version &log &optional; ## The full unparsed version string found because the version parsing ## doesn't work 100% reliably and this acts as a fall back in the logs. unparsed_version: string &log &optional; - }; - - type AddItem: record { - ## The connection - id: conn_id; - ## The unparsed string representing the software version - banner: string; - ## Pre-parsed version. If this field is present, banner should only contain the name of the software - version: Version &optional; - ## The IP address detected running the software. - host: addr; - ## The port on which the software is running (if applicable). - host_p: port &optional; - ## The type of software detected (e.g. WEB_SERVER) - sw_type: Type; ## This can indicate that this software being detected should ## definitely be sent onward to the logging framework. By @@ -89,7 +75,7 @@ export { ## unparsed_version: This is the full string from which the ## :bro:type:`Software::Info` was extracted. ## Returns: T if the software was logged, F otherwise. - global found: function(i: AddItem): bool; + global found: function(id: conn_id, info: Info): bool; ## This function can take many software version strings and parse them ## into a sensible :bro:type:`Software::Version` record. There are @@ -382,7 +368,7 @@ function software_fmt(i: Info): string # Insert a mapping into the table # Overides old entries for the same software and generates events if needed. -event software_register(id: conn_id, force_log: bool, info: Info) +event software_register(id: conn_id, info: Info) { # Host already known? if ( info$host !in tracked ) @@ -398,7 +384,7 @@ event software_register(id: conn_id, force_log: bool, info: Info) # If the version hasn't changed, then we're just redetecting the # same thing, then we don't care. This results in no extra logging. # But if the $force_log value is set then we'll continue. - if ( ! force_log && cmp_versions(old$version, info$version) == 0 ) + if ( ! info$force_log && cmp_versions(old$version, info$version) == 0 ) return; } ts[info$name] = info; @@ -406,25 +392,38 @@ event software_register(id: conn_id, force_log: bool, info: Info) Log::write(Software::LOG, info); } -function found(i: AddItem): bool +function found(id: conn_id, info: Info): bool { - if ( i$force_log || addr_matches_host(i$host, asset_tracking) ) + if ( info$force_log || addr_matches_host(info$host, asset_tracking) ) { - local sw: SoftwareDescription; - if ( i?$version ) # already fully parsed, banner should contain the software name + if ( !info?$ts ) + info$ts=network_time(); + + if ( info?$version ) # we have a version number and don't have to parse. check if the name is also set... { - sw = [$version=i$version, $name=i$banner, $unparsed_version=i$banner]; - } - else + if ( !info?$name ) + { + Reporter::error("Required field name not present in Software::found"); + return F; + } + } + else # no version present, we have to parse... { - sw = parse(i$banner); + if ( !info?$unparsed_version ) + { + Reporter::error("No unparsed version string present in Info record with version in Software::found"); + return F; + } + local sw = parse(info$unparsed_version); + info$unparsed_version = sw$unparsed_version; + info$name = sw$name; + info$version = sw$version; + } - event software_register(i$id, i$force_log, [$ts=network_time(), $host=i$host, $host_p=i$host_p, $name=sw$name, - $version=sw$version, $unparsed_version=sw$unparsed_version, - $software_type=i$sw_type] ); + event software_register(id, info); return T; } diff --git a/scripts/policy/protocols/ftp/software.bro b/scripts/policy/protocols/ftp/software.bro index 24e7ff0b0a..1f5262fcab 100644 --- a/scripts/policy/protocols/ftp/software.bro +++ b/scripts/policy/protocols/ftp/software.bro @@ -21,6 +21,6 @@ event ftp_request(c: connection, command: string, arg: string) &priority=4 { if ( command == "CLNT" ) { - Software::found([$id=c$id, $banner=arg, $host=c$id$orig_h, $sw_type=FTP_CLIENT]); + Software::found(c$id, [$unparsed_version=arg, $host=c$id$orig_h, $software_type=FTP_CLIENT]); } } diff --git a/scripts/policy/protocols/http/detect-webapps.bro b/scripts/policy/protocols/http/detect-webapps.bro index 63f481422a..b9cc309069 100644 --- a/scripts/policy/protocols/http/detect-webapps.bro +++ b/scripts/policy/protocols/http/detect-webapps.bro @@ -23,7 +23,8 @@ event signature_match(state: signature_state, msg: string, data: string) &priori if ( /^webapp-/ !in state$sig_id ) return; local c = state$conn; - local si = Software::parse_with_port(msg, c$id$resp_h, c$id$resp_p, WEB_APPLICATION); + local si = Software::Info; + si = [$unparsed_version=msg, $host=c$id$resp_h, $host_p=c$id$resp_p, $software_type=WEB_APPLICATION]; si$url = build_url_http(c$http); if ( c$id$resp_h in Software::tracked && si$name in Software::tracked[c$id$resp_h] ) diff --git a/scripts/policy/protocols/http/software-browser-plugins.bro b/scripts/policy/protocols/http/software-browser-plugins.bro index 21bc8d07cb..7316595e7f 100644 --- a/scripts/policy/protocols/http/software-browser-plugins.bro +++ b/scripts/policy/protocols/http/software-browser-plugins.bro @@ -26,7 +26,7 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr # Flash doesn't include it's name so we'll add it here since it # simplifies the version parsing. value = cat("Flash/", value); - Software::found([$id=c$id, $banner=flash_version, $host=c$id$orig_h, $sw_type=BROWSER_PLUGIN]); + Software::found(c$id, [$unparsed_version=value, $host=c$id$orig_h, $software_type=BROWSER_PLUGIN]); } } else @@ -53,7 +53,7 @@ event log_http(rec: Info) local plugins = split(sw, /[[:blank:]]*;[[:blank:]]*/); for ( i in plugins ) - Software::found([$id=rec$id, $banner=plugins[i], $host=rec$id$orig_h, $sw_type=BROWSER_PLUGIN]); + Software::found(rec$id, [$unparsed_version=plugins[i], $host=rec$id$orig_h, $software_type=BROWSER_PLUGIN]); } } } diff --git a/scripts/policy/protocols/http/software.bro b/scripts/policy/protocols/http/software.bro index 43552798e6..99b9a534f7 100644 --- a/scripts/policy/protocols/http/software.bro +++ b/scripts/policy/protocols/http/software.bro @@ -20,18 +20,18 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr if ( is_orig ) { if ( name == "USER-AGENT" && ignored_user_agents !in value ) - Software::found([$id=c$id, $banner=value, $host=c$id$orig_h, $sw_type=BROWSER]); + Software::found(c$id, [$unparsed_version=value, $host=c$id$orig_h, $software_type=BROWSER]); } else { if ( name == "SERVER" ) - Software::found([$id=c$id, $banner=value, $host=c$id$resp_h, $host_p=c$id$resp_p, $sw_type=SERVER]); + Software::found(c$id, [$unparsed_version=value, $host=c$id$resp_h, $host_p=c$id$resp_p, $software_type=SERVER]); else if ( name == "X-POWERED-BY" ) - Software::found([$id=c$id, $banner=value, $host=c$id$resp_h, $host_p=c$id$resp_p, $sw_type=APPSERVER]); + Software::found(c$id, [$unparsed_version=value, $host=c$id$resp_h, $host_p=c$id$resp_p, $software_type=APPSERVER]); else if ( name == "MICROSOFTSHAREPOINTTEAMSERVICES" ) { value = cat("SharePoint/", value); - Software::found([$id=c$id, $banner=value, $host=c$id$resp_h, $host_p=c$id$resp_p, $sw_type=APPSERVER]); + Software::found(c$id, [$unparsed_version=value, $host=c$id$resp_h, $host_p=c$id$resp_p, $software_type=APPSERVER]); } } } diff --git a/scripts/policy/protocols/smtp/software.bro b/scripts/policy/protocols/smtp/software.bro index 881848abc2..f520485338 100644 --- a/scripts/policy/protocols/smtp/software.bro +++ b/scripts/policy/protocols/smtp/software.bro @@ -75,7 +75,7 @@ event log_smtp(rec: Info) if ( addr_matches_host(rec$id$orig_h, detect_clients_in_messages_from) ) { - Software::found([$id=rec$id, $banner=rec$user_agent, $host=client_ip, $sw_type=s_type]); + Software::found(rec$id, [$unparsed_version=rec$user_agent, $host=client_ip, $software_type=s_type]); } } } diff --git a/scripts/policy/protocols/ssh/software.bro b/scripts/policy/protocols/ssh/software.bro index a0160ee7ba..a47f198c56 100644 --- a/scripts/policy/protocols/ssh/software.bro +++ b/scripts/policy/protocols/ssh/software.bro @@ -16,12 +16,12 @@ event ssh_client_version(c: connection, version: string) &priority=4 { # Get rid of the protocol information when passing to the software framework. local cleaned_version = sub(version, /^SSH[0-9\.\-]+/, ""); - Software::found([$id=c$id, $banner=cleaned_version, $host=c$id$orig_h, $sw_type=CLIENT]); + Software::found(c$id, [$unparsed_version=cleaned_version, $host=c$id$orig_h, $software_type=CLIENT]); } event ssh_server_version(c: connection, version: string) &priority=4 { # Get rid of the protocol information when passing to the software framework. local cleaned_version = sub(version, /SSH[0-9\.\-]{2,}/, ""); - Software::found([$id=c$id, $banner=cleaned_version, $host=c$id$resp_h, $host_p=c$id$resp_p, $sw_type=SERVER]); + Software::found(c$id, [$unparsed_version=cleaned_version, $host=c$id$resp_h, $host_p=c$id$resp_p, $software_type=SERVER]); } From cc1459ef35205dc9d36987a5aa863d1a00e5a066 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 16 Dec 2011 14:30:36 -0600 Subject: [PATCH 201/964] Fix some malformed Broxygen xref roles. --- doc/logging.rst | 2 +- doc/scripts/builtins.rst | 9 +++++ scripts/base/frameworks/control/main.bro | 10 +++-- scripts/base/frameworks/intel/main.bro | 4 +- scripts/base/frameworks/metrics/cluster.bro | 4 +- scripts/base/frameworks/metrics/main.bro | 3 +- .../frameworks/notice/actions/email_admin.bro | 4 +- .../base/frameworks/notice/actions/page.bro | 2 +- scripts/base/frameworks/software/main.bro | 6 +-- scripts/base/init-bare.bro | 17 +++++---- scripts/base/protocols/irc/main.bro | 4 +- scripts/base/utils/site.bro | 2 +- scripts/policy/integration/barnyard2/main.bro | 2 +- scripts/policy/misc/capture-loss.bro | 6 +-- scripts/policy/misc/trim-trace-file.bro | 3 +- scripts/policy/protocols/http/detect-sqli.bro | 6 +-- .../protocols/ssh/detect-bruteforcing.bro | 2 +- .../protocols/ssh/interesting-hostnames.bro | 2 +- .../policy/protocols/ssl/expiring-certs.bro | 2 +- src/bro.bif | 8 ++-- src/event.bif | 38 +++++++++---------- src/strings.bif | 10 ++--- 22 files changed, 80 insertions(+), 66 deletions(-) diff --git a/doc/logging.rst b/doc/logging.rst index e7734915da..2b5a7167b3 100644 --- a/doc/logging.rst +++ b/doc/logging.rst @@ -163,7 +163,7 @@ quickly ... .. sidebar:: A More Generic Path Function The ``split_log`` method has one draw-back: it can be used - only with the :bro:enum:`Conn::Log` stream as the record type is hardcoded + only with the :bro:enum:`Conn::LOG` stream as the record type is hardcoded into its argument list. However, Bro allows to do a more generic variant: diff --git a/doc/scripts/builtins.rst b/doc/scripts/builtins.rst index 4a3316c04f..ef6738a1a6 100644 --- a/doc/scripts/builtins.rst +++ b/doc/scripts/builtins.rst @@ -6,6 +6,11 @@ Types The Bro scripting language supports the following built-in types. +.. bro:type:: void + + An internal Bro type representing an absence of a type. Should + most often be seen as a possible function return type. + .. bro:type:: bool Reflects a value with one of two meanings: true or false. The two @@ -621,6 +626,10 @@ scripting language supports the following built-in attributes. Writes a record field to the associated log stream. +.. bro:attr:: &error_handler + +.. TODO: needs documented + .. bro:attr:: (&tracked) .. TODO: needs documented or removed if it's not used anywhere. diff --git a/scripts/base/frameworks/control/main.bro b/scripts/base/frameworks/control/main.bro index 5aabaa4bac..497936c8f1 100644 --- a/scripts/base/frameworks/control/main.bro +++ b/scripts/base/frameworks/control/main.bro @@ -1,6 +1,6 @@ ##! This is a utility script that sends the current values of all &redef'able -##! consts to a remote Bro then sends the :bro:id:`configuration_update` event -##! and terminates processing. +##! consts to a remote Bro then sends the +##! :bro:id:`Control::configuration_update` event and terminates processing. ##! ##! Intended to be used from the command line like this when starting a controller:: ##! @@ -53,7 +53,8 @@ export { ## Event for requesting the value of an ID (a variable). global id_value_request: event(id: string); - ## Event for returning the value of an ID after an :bro:id:`id_request` event. + ## Event for returning the value of an ID after an + ## :bro:id:`Control::id_value_request` event. global id_value_response: event(id: string, val: string); ## Requests the current communication status. @@ -68,7 +69,8 @@ export { ## Inform the remote Bro instance that it's configuration may have been updated. global configuration_update_request: event(); - ## This event is a wrapper and alias for the :bro:id:`configuration_update_request` event. + ## This event is a wrapper and alias for the + ## :bro:id:`Control::configuration_update_request` event. ## This event is also a primary hooking point for the control framework. global configuration_update: event(); ## Message in response to a configuration update request. diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro index 648c32bc57..6ee4435831 100644 --- a/scripts/base/frameworks/intel/main.bro +++ b/scripts/base/frameworks/intel/main.bro @@ -76,8 +76,8 @@ export { and_tags: set[string] &optional; ## The predicate can be given when searching for a match. It will - ## be tested against every :bro:type:`MetaData` item associated with - ## the data being matched on. If it returns T a single time, the + ## be tested against every :bro:type:`Intel::MetaData` item associated + ## with the data being matched on. If it returns T a single time, the ## matcher will consider that the item has matched. pred: function(meta: Intel::MetaData): bool &optional; }; diff --git a/scripts/base/frameworks/metrics/cluster.bro b/scripts/base/frameworks/metrics/cluster.bro index 6835c5bb9b..a3a2b8b53c 100644 --- a/scripts/base/frameworks/metrics/cluster.bro +++ b/scripts/base/frameworks/metrics/cluster.bro @@ -40,12 +40,12 @@ export { global cluster_index_request: event(uid: string, id: ID, filter_name: string, index: Index); ## This event is sent by nodes in response to a - ## :bro:id:`cluster_index_request` event. + ## :bro:id:`Metrics::cluster_index_request` event. global cluster_index_response: event(uid: string, id: ID, filter_name: string, index: Index, val: count); ## This is sent by workers to indicate that they crossed the percent of the ## current threshold by the percentage defined globally in - ## :bro:id:`cluster_request_global_view_percent` + ## :bro:id:`Metrics::cluster_request_global_view_percent` global cluster_index_intermediate_response: event(id: Metrics::ID, filter_name: string, index: Metrics::Index, val: count); ## This event is scheduled internally on workers to send result chunks. diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro index f53a86a977..c30807e179 100644 --- a/scripts/base/frameworks/metrics/main.bro +++ b/scripts/base/frameworks/metrics/main.bro @@ -86,6 +86,8 @@ export { ## threshold in every break interval. notice_freq: interval &optional; }; + + type MetricTable: table[Index] of count &default=0; global add_filter: function(id: ID, filter: Filter); global add_data: function(id: ID, index: Index, increment: count); @@ -105,7 +107,6 @@ redef record Notice::Info += { global metric_filters: table[ID] of vector of Filter = table(); global filter_store: table[ID, string] of Filter = table(); -type MetricTable: table[Index] of count &default=0; # This is indexed by metric ID and stream filter name. global store: table[ID, string] of MetricTable = table() &default=table(); diff --git a/scripts/base/frameworks/notice/actions/email_admin.bro b/scripts/base/frameworks/notice/actions/email_admin.bro index 044deb192a..7484a1c606 100644 --- a/scripts/base/frameworks/notice/actions/email_admin.bro +++ b/scripts/base/frameworks/notice/actions/email_admin.bro @@ -11,8 +11,8 @@ module Notice; export { redef enum Action += { ## Indicate that the generated email should be addressed to the - ## appropriate email addresses as found in the - ## :bro:id:`Site::addr_to_emails` variable based on the relevant + ## appropriate email addresses as found by the + ## :bro:id:`Site::get_emails` function based on the relevant ## address or addresses indicated in the notice. ACTION_EMAIL_ADMIN }; diff --git a/scripts/base/frameworks/notice/actions/page.bro b/scripts/base/frameworks/notice/actions/page.bro index 8002566a1a..16a3463126 100644 --- a/scripts/base/frameworks/notice/actions/page.bro +++ b/scripts/base/frameworks/notice/actions/page.bro @@ -7,7 +7,7 @@ module Notice; export { redef enum Action += { ## Indicates that the notice should be sent to the pager email address - ## configured in the :bro:id:`mail_page_dest` variable. + ## configured in the :bro:id:`Notice::mail_page_dest` variable. ACTION_PAGE }; diff --git a/scripts/base/frameworks/software/main.bro b/scripts/base/frameworks/software/main.bro index 574886288a..e244915077 100644 --- a/scripts/base/frameworks/software/main.bro +++ b/scripts/base/frameworks/software/main.bro @@ -81,9 +81,9 @@ export { global cmp_versions: function(v1: Version, v2: Version): int; ## This type represents a set of software. It's used by the - ## :bro:id:`tracked` variable to store all known pieces of software - ## for a particular host. It's indexed with the name of a piece of - ## software such as "Firefox" and it yields a + ## :bro:id:`Software::tracked` variable to store all known pieces of + ## software for a particular host. It's indexed with the name of a piece + ## of software such as "Firefox" and it yields a ## :bro:type:`Software::Info` record with more information about the ## software. type SoftwareSet: table[string] of Info; diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 7c4029cd7c..de05f93543 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -75,7 +75,7 @@ type conn_id: record { } &log; ## Specifics about an ICMP conversation. ICMP events typically pass this in -## addition to :bro:id:`conn_id`. +## addition to :bro:type:`conn_id`. ## ## .. bro:see:: icmp_echo_reply icmp_echo_request icmp_redirect icmp_sent ## icmp_time_exceeded icmp_unreachable @@ -331,7 +331,7 @@ type script_id: record { exported: bool; ##< True if the identifier is exported. constant: bool; ##< True if the identifier is a constant. enum_constant: bool; ##< True if the identifier is an enum value. - redefinable: bool; ##< True if the identifier is declared with the :bro:attr:`redef` attribute. + redefinable: bool; ##< True if the identifier is declared with the :bro:attr:`&redef` attribute. value: any &optional; ##< The current value of the identifier. }; @@ -350,11 +350,11 @@ type id_table: table[string] of script_id; ## .. bro:see:: record_fields record_field_table type record_field: record { type_name: string; ##< The name of the field's type. - log: bool; ##< True of the field is declared with :bro:attr:`log` attribute. + log: bool; ##< True of the field is declared with :bro:attr:`&log` attribute. ## The current value of the field in the record instance passed into ## :bro:see:`record_fields` (if it has one). value: any &optional; - default_val: any &optional; ##< The value of the :bro:attr:`default` attribute if defined. + default_val: any &optional; ##< The value of the :bro:attr:`&default` attribute if defined. }; ## Table type used to map record field declarations to meta-information describing @@ -384,9 +384,10 @@ type record_field_table: table[string] of record_field; global capture_filters: table[string] of string &redef; ## Set of BPF filters to restrict capturing, indexed by a user-definable ID (which -## must be unique). If Bro is *not* configured to examine :bro:id:`all_packets`, -## only packets matching *all* of the filters in this table (and -## any in :bro:id:`capture_filters`) will be analyzed. +## must be unique). If Bro is *not* configured to examine +## :bro:id:`PacketFilter::all_packets`, only packets matching *all* of the +## filters in this table (and any in :bro:id:`capture_filters`) will be +## analyzed. ## ## .. bro:see:: PacketFilter PacketFilter::all_packets ## PacketFilter::unrestricted_filter capture_filters @@ -555,7 +556,7 @@ global log_file_name: function(tag: string): string &redef; global open_log_file: function(tag: string): file &redef; ## Specifies a directory for Bro store its persistent state. All globals can -## be declared persistent via the :bro:attr:`persistent` attribute. +## be declared persistent via the :bro:attr:`&persistent` attribute. const state_dir = ".state" &redef; ## Length of the delays inserted when storing state incrementally. To avoid diff --git a/scripts/base/protocols/irc/main.bro b/scripts/base/protocols/irc/main.bro index e74eea21f1..3736569910 100644 --- a/scripts/base/protocols/irc/main.bro +++ b/scripts/base/protocols/irc/main.bro @@ -1,8 +1,8 @@ ##! This is the script that implements the core IRC analysis support. It only ##! logs a very limited subset of the IRC protocol by default. The points ##! that it logs at are NICK commands, USER commands, and JOIN commands. It -##! log various bits of meta data as indicated in the :bro:type:`Info` record -##! along with the command at the command arguments. +##! log various bits of meta data as indicated in the :bro:type:`IRC::Info` +##! record along with the command at the command arguments. module IRC; diff --git a/scripts/base/utils/site.bro b/scripts/base/utils/site.bro index 536c891572..4aeb70fe3f 100644 --- a/scripts/base/utils/site.bro +++ b/scripts/base/utils/site.bro @@ -18,7 +18,7 @@ export { const local_nets: set[subnet] &redef; ## This is used for retrieving the subnet when you multiple - ## :bro:id:`local_nets`. A membership query can be done with an + ## :bro:id:`Site::local_nets`. A membership query can be done with an ## :bro:type:`addr` and the table will yield the subnet it was found ## within. global local_nets_table: table[subnet] of subnet = {}; diff --git a/scripts/policy/integration/barnyard2/main.bro b/scripts/policy/integration/barnyard2/main.bro index c2f1c790d3..1d38d80809 100644 --- a/scripts/policy/integration/barnyard2/main.bro +++ b/scripts/policy/integration/barnyard2/main.bro @@ -15,7 +15,7 @@ export { alert: AlertData &log; }; - ## This can convert a Barnyard :bro:type:`PacketID` value to a + ## This can convert a Barnyard :bro:type:`Barnyard2::PacketID` value to a ## :bro:type:`conn_id` value in the case that you might need to index ## into an existing data structure elsewhere within Bro. global pid2cid: function(p: PacketID): conn_id; diff --git a/scripts/policy/misc/capture-loss.bro b/scripts/policy/misc/capture-loss.bro index d966708762..10d0c9519d 100644 --- a/scripts/policy/misc/capture-loss.bro +++ b/scripts/policy/misc/capture-loss.bro @@ -42,9 +42,9 @@ export { const watch_interval = 15mins &redef; ## The percentage of missed data that is considered "too much" - ## when the :bro:enum:`Too_Much_Loss` notice should be generated. - ## The value is expressed as a double between 0 and 1 with 1 being - ## 100% + ## when the :bro:enum:`CaptureLoss::Too_Much_Loss` notice should be + ## generated. The value is expressed as a double between 0 and 1 with 1 + ## being 100% const too_much_loss: double = 0.1 &redef; } diff --git a/scripts/policy/misc/trim-trace-file.bro b/scripts/policy/misc/trim-trace-file.bro index 3caa41c06b..8a7781b628 100644 --- a/scripts/policy/misc/trim-trace-file.bro +++ b/scripts/policy/misc/trim-trace-file.bro @@ -10,7 +10,8 @@ export { ## This event can be generated externally to this script if on-demand ## tracefile rotation is required with the caveat that the script doesn't ## currently attempt to get back on schedule automatically and the next - ## trim will likely won't happen on the :bro:id:`trim_interval`. + ## trim will likely won't happen on the + ## :bro:id:`TrimTraceFile::trim_interval`. global go: event(first_trim: bool); } diff --git a/scripts/policy/protocols/http/detect-sqli.bro b/scripts/policy/protocols/http/detect-sqli.bro index c4ba7ee74e..6be7ae2c88 100644 --- a/scripts/policy/protocols/http/detect-sqli.bro +++ b/scripts/policy/protocols/http/detect-sqli.bro @@ -35,9 +35,9 @@ export { ## injection attacks. const sqli_requests_threshold = 50 &redef; - ## Interval at which to watch for the :bro:id:`sqli_requests_threshold` - ## variable to be crossed. At the end of each interval the counter is - ## reset. + ## Interval at which to watch for the + ## :bro:id:`HTTP::sqli_requests_threshold` variable to be crossed. + ## At the end of each interval the counter is reset. const sqli_requests_interval = 5min &redef; ## This regular expression is used to match URI based SQL injections diff --git a/scripts/policy/protocols/ssh/detect-bruteforcing.bro b/scripts/policy/protocols/ssh/detect-bruteforcing.bro index 3abe185d58..d6c848f42e 100644 --- a/scripts/policy/protocols/ssh/detect-bruteforcing.bro +++ b/scripts/policy/protocols/ssh/detect-bruteforcing.bro @@ -9,7 +9,7 @@ module SSH; export { redef enum Notice::Type += { ## Indicates that a host has been identified as crossing the - ## :bro:id:`password_guesses_limit` threshold with heuristically + ## :bro:id:`SSH::password_guesses_limit` threshold with heuristically ## determined failed logins. Password_Guessing, ## Indicates that a host previously identified as a "password guesser" diff --git a/scripts/policy/protocols/ssh/interesting-hostnames.bro b/scripts/policy/protocols/ssh/interesting-hostnames.bro index 29886d0eb0..0f3af164f8 100644 --- a/scripts/policy/protocols/ssh/interesting-hostnames.bro +++ b/scripts/policy/protocols/ssh/interesting-hostnames.bro @@ -12,7 +12,7 @@ export { redef enum Notice::Type += { ## Generated if a login originates or responds with a host and the ## reverse hostname lookup resolves to a name matched by the - ## :bro:id:`interesting_hostnames` regular expression. + ## :bro:id:`SSH::interesting_hostnames` regular expression. Interesting_Hostname_Login, }; diff --git a/scripts/policy/protocols/ssl/expiring-certs.bro b/scripts/policy/protocols/ssl/expiring-certs.bro index 50480b3a09..eaefd6740e 100644 --- a/scripts/policy/protocols/ssl/expiring-certs.bro +++ b/scripts/policy/protocols/ssl/expiring-certs.bro @@ -29,7 +29,7 @@ export { const notify_certs_expiration = LOCAL_HOSTS &redef; ## The time before a certificate is going to expire that you would like to - ## start receiving :bro:enum:`Certificate_Expires_Soon` notices. + ## start receiving :bro:enum:`SSL::Certificate_Expires_Soon` notices. const notify_when_cert_expiring_in = 30days &redef; } diff --git a/src/bro.bif b/src/bro.bif index 1afd98fddb..c941d19c3b 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -657,7 +657,7 @@ function md5_hash_init%(index: any%): bool %} ## Update the MD5 value associated with a given index. It is required to -## call :bro:id:`md5_hash_init(index)` once before calling this +## call :bro:id:`md5_hash_init` once before calling this ## function. ## ## index: The unique identifier to associate with this hash computation. @@ -3025,11 +3025,11 @@ function connection_exists%(c: conn_id%): bool return new Val(0, TYPE_BOOL); %} -## Returns the :bro:id:`connection` record for a given connection identifier. +## Returns the :bro:type:`connection` record for a given connection identifier. ## ## cid: The connection ID. ## -## Returns: The :bro:id:`connection` record for *cid*. If *cid* does not point +## Returns: The :bro:type:`connection` record for *cid*. If *cid* does not point ## to an existing connection, the function generates a run-time error ## and returns a dummy value. ## @@ -4833,7 +4833,7 @@ function enable_communication%(%): any return 0; %} -## Flushes in-memory state tagged with the :bro:attr:`&persistence` attribute +## Flushes in-memory state tagged with the :bro:attr:`&persistent` attribute ## to disk. The function writes the state to the file ``.state/state.bst`` in ## the directory where Bro was started. ## diff --git a/src/event.bif b/src/event.bif index 73c3bb0959..ae0ef07ade 100644 --- a/src/event.bif +++ b/src/event.bif @@ -383,7 +383,7 @@ event connection_timeout%(c: connection%); event connection_reused%(c: connection%); ## Generated in regular intervals during the life time of a connection. The -## events is raised each :bro:id:`connection_status_update_interval` seconds +## events is raised each ``connection_status_update_interval`` seconds ## and can be used to check conditions on a regular basis. ## ## c: The connection. @@ -428,7 +428,7 @@ event connection_external%(c: connection, tag: string%); ## a: The analyzer that was scheduled for the connection with the ## :bro:id:`expect_connection` call. When the event is raised, that ## analyzer will already have been activated to process the connection. The -## ``count`` is one of the ``ANALYZER_*`` constants, e.g., ``ANALYZER_HTTP``. +## ``count`` is one of the ``ANALYZER_*`` constants, e.g., ``ANALYZER_HTTP``. ## ## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt ## connection_established connection_external connection_finished @@ -515,8 +515,8 @@ event tcp_option%(c: connection, is_orig: bool, opt: count, optlen: count%); ## Generated for each chunk of reassembled TCP payload. When content delivery is ## enabled for a TCP connection (via :bro:id:`tcp_content_delivery_ports_orig`, ## :bro:id:`tcp_content_delivery_ports_resp`, -## :bro:id:`tcp_content_delivery_all_orig`, -## :bro:id:`tcp_content_delivery_all_resp`), this event is raised for each chunk +## :bro:id:`tcp_content_deliver_all_orig`, +## :bro:id:`tcp_content_deliver_all_resp`), this event is raised for each chunk ## of in-order payload reconstructed from the packet stream. Note that this event ## is potentially expensive if many connections carry signficant amounts of data as ## then all that needs to be passed on to the scripting layer. @@ -643,7 +643,7 @@ event gap_report%(dt: interval, info: gap_info%); ## ## atype: The type of the analyzer confirming that its protocol is in ## use. The value is one of the ``ANALYZER_*`` constants. For example, -## :bro:id:`ANALYZER_HTTP` means the HTTP analyzers determined that it's indeed +## ``ANALYZER_HTTP`` means the HTTP analyzers determined that it's indeed ## parsing an HTTP connection. ## ## aid: A unique integer ID identifying the specific *instance* of the @@ -656,8 +656,8 @@ event gap_report%(dt: interval, info: gap_info%); ## .. note:: ## ## Bro's default scripts use this event to determine the ``service`` column of -## :bro:id:`Conn::Info`: once confirmed, the protocol will be listed there (and -## thus in ``conn.log``). +## :bro:type:`Conn::Info`: once confirmed, the protocol will be listed there +## (and thus in ``conn.log``). event protocol_confirmation%(c: connection, atype: count, aid: count%); ## Generated when a protocol analyzer determines that a connection it is parsing @@ -671,7 +671,7 @@ event protocol_confirmation%(c: connection, atype: count, aid: count%); ## ## atype: The type of the analyzer confirming that its protocol is in ## use. The value is one of the ``ANALYZER_*`` constants. For example, -## :bro:id:`ANALYZER_HTTP` means the HTTP analyzers determined that it's indeed +## ``ANALYZER_HTTP`` means the HTTP analyzers determined that it's indeed ## parsing an HTTP connection. ## ## aid: A unique integer ID identifying the specific *instance* of the @@ -5965,13 +5965,13 @@ event remote_state_inconsistency%(operation: string, id: string, ## intended primarily for use by Bro's communication framework, it can also trigger ## additional code if helpful. ## -## level: The log level, which is either :bro:enum:`REMOTE_LOG_INFO` or -## :bro:enum:`REMOTE_LOG_ERROR`. +## level: The log level, which is either :bro:id:`REMOTE_LOG_INFO` or +## :bro:id:`REMOTE_LOG_ERROR`. ## ## src: The component of the comminication system that logged the message. -## Currently, this will be one of :bro:enum:`REMOTE_SRC_CHILD` (Bro's -## child process), :bro:enum:`REMOTE_SRC_PARENT` (Bro's main process), or -## :bro:enum:`REMOTE_SRC_SCRIPT` (the script level). +## Currently, this will be one of :bro:id:`REMOTE_SRC_CHILD` (Bro's +## child process), :bro:id:`REMOTE_SRC_PARENT` (Bro's main process), or +## :bro:id:`REMOTE_SRC_SCRIPT` (the script level). ## ## msg: The message logged. ## @@ -5988,13 +5988,13 @@ event remote_log%(level: count, src: count, msg: string%); ## ## p: A record describing the remote peer. ## -## level: The log level, which is either :bro:enum:`REMOTE_LOG_INFO` or -## :bro:enum:`REMOTE_LOG_ERROR`. +## level: The log level, which is either :bro:id:`REMOTE_LOG_INFO` or +## :bro:id:`REMOTE_LOG_ERROR`. ## ## src: The component of the comminication system that logged the message. -## Currently, this will be one of :bro:enum:`REMOTE_SRC_CHILD` (Bro's -## child process), :bro:enum:`REMOTE_SRC_PARENT` (Bro's main process), or -## :bro:enum:`REMOTE_SRC_SCRIPT` (the script level). +## Currently, this will be one of :bro:id:`REMOTE_SRC_CHILD` (Bro's +## child process), :bro:id:`REMOTE_SRC_PARENT` (Bro's main process), or +## :bro:id:`REMOTE_SRC_SCRIPT` (the script level). ## ## msg: The message logged. ## @@ -6051,7 +6051,7 @@ event remote_state_access_performed%(id: string, v: any%); ## f: The profiling file. ## ## expensive: True if this event corresponds to heavier-weight profiling as -## indicated by the :bro:enum:`expensive_profiling_multiple` variable. +## indicated by the :bro:id:`expensive_profiling_multiple` variable. ## ## .. bro:see:: profiling_interval expensive_profiling_multiple event profiling_update%(f: file, expensive: bool%); diff --git a/src/strings.bif b/src/strings.bif index 3fec92cd7a..5301dfcf5e 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -83,7 +83,7 @@ BroString* cat_string_array_n(TableVal* tbl, int start, int end) ## Concatenates all elements in an array of strings. ## -## a: The :bro:id:`string_array` (``table[count] of string``). +## a: The :bro:type:`string_array` (``table[count] of string``). ## ## Returns: The concatenation of all elements in *a*. ## @@ -98,7 +98,7 @@ function cat_string_array%(a: string_array%): string ## Concatenates a specific range of elements in an array of strings. ## -## a: The :bro:id:`string_array` (``table[count] of string``). +## a: The :bro:type:`string_array` (``table[count] of string``). ## ## start: The array index of the first element of the range. ## @@ -120,7 +120,7 @@ function cat_string_array_n%(a: string_array, start: count, end: count%): string ## ## sep: The separator to place between each element. ## -## a: The :bro:id:`string_array` (``table[count] of string``). +## a: The :bro:type:`string_array` (``table[count] of string``). ## ## Returns: The concatenation of all elements in *a*, with *sep* placed ## between each element. @@ -156,7 +156,7 @@ function join_string_array%(sep: string, a: string_array%): string ## ## sep: The separator to place between each element. ## -## a: The :bro:id:`string_vec` (``vector of string``). +## a: The :bro:type:`string_vec` (``vector of string``). ## ## Returns: The concatenation of all elements in *a*, with *sep* placed ## between each element. @@ -185,7 +185,7 @@ function join_string_vec%(vec: string_vec, sep: string%): string ## Sorts an array of strings. ## -## a: The :bro:id:`string_array` (``table[count] of string``). +## a: The :bro:type:`string_array` (``table[count] of string``). ## ## Returns: A sorted copy of *a*. ## From 23177aa0813198432ad2adaadd87833befecae07 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 16 Dec 2011 14:36:15 -0600 Subject: [PATCH 202/964] Enable warnings for malformed Broxygen xref roles. --- doc/ext/bro.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/ext/bro.py b/doc/ext/bro.py index a4f0142ce3..9bdd86bd9a 100644 --- a/doc/ext/bro.py +++ b/doc/ext/bro.py @@ -257,6 +257,9 @@ class BroDomain(Domain): objects[objtype, target], objtype + '-' + target, contnode, target + ' ' + objtype) + else: + self.env.warn(fromdocname, + 'unknown target for ":bro:%s:`%s`"' % (typ, target)) def get_objects(self): for (typ, name), docname in self.data['objects'].iteritems(): From 3220bbce555f05baef12ebff35d42a45eeac6ccc Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sun, 18 Dec 2011 16:42:58 -0800 Subject: [PATCH 203/964] Merge remote branch 'origin/topic/jsiwek/log-escaping' * origin/topic/jsiwek/log-escaping: Add missing ascii writer options to log header. Escape the ASCII log's set separator (addresses #712) Rewrite ODesc character escaping functionality. (addresses #681) Closes #712. --- src/Desc.cc | 80 ++++++++++--------- src/Desc.h | 28 ++++++- src/LogWriterAscii.cc | 46 +++++------ src/util.cc | 21 +++++ src/util.h | 2 + .../Baseline/core.expr-exception/reporter.log | 21 ++--- .../core.print-bpf-filters-ipv4/conn.log | 3 + .../core.print-bpf-filters-ipv4/output | 20 ++++- .../btest/Baseline/core.vlan-mpls/conn.log | 3 + .../canonified_loaded_scripts.log | 3 + .../canonified_loaded_scripts.log | 3 + .../istate.events-ssl/receiver.http.log | 5 +- .../istate.events-ssl/sender.http.log | 5 +- .../Baseline/istate.events/receiver.http.log | 5 +- .../Baseline/istate.events/sender.http.log | 5 +- .../send.log | 27 ++++--- .../ssh-new-default.log | 7 +- .../ssh.log | 3 + .../ssh.log | 13 +-- .../test.log | 3 + .../http.log | 3 + .../test.log | 8 ++ .../ssh.log | 13 +-- .../test.log | 3 + .../ssh.log | 3 + .../ssh.log | 3 + .../ssh.log | 13 +-- .../ssh.log | 3 + .../ssh.log | 5 +- .../ssh.log | 13 +-- .../local.log | 3 + .../remote.log | 3 + .../output | 35 ++++++-- .../test.failure.log | 5 +- .../test.success.log | 5 +- .../receiver.test.log | 5 +- .../sender.test.failure.log | 9 ++- .../sender.test.log | 13 +-- .../sender.test.success.log | 7 +- .../ssh.failure.log | 7 +- .../ssh.log | 9 ++- .../out | 3 + .../out | 30 +++++++ .../output | 13 +-- .../ssh.log | 13 +-- .../ssh.log | 5 +- .../testing.log | 3 + .../ssh.log | 3 + .../manager-1.metrics.log | 9 ++- .../metrics.log | 9 ++- .../manager-1.notice.log | 5 +- .../notice.log | 7 +- .../manager-1.notice.log | 5 +- .../manager-1.notice.log | 5 +- .../notice.log | 5 +- .../http.log | 3 + .../http.log | 3 + .../http.log | 3 + .../http.log | 3 + .../scripts.base.protocols.irc.basic/irc.log | 3 + .../irc.log | 3 + .../smtp.log | 3 + .../smtp_entities.log | 3 + .../smtp_entities.log | 3 + .../knownhosts-all.log | 3 + .../knownhosts-local.log | 3 + .../knownhosts-remote.log | 3 + .../knownservices-all.log | 3 + .../knownservices-local.log | 3 + .../knownservices-remote.log | 3 + .../dns.log | 3 + .../logging/ascii-escape-set-separator.bro | 21 +++++ 72 files changed, 487 insertions(+), 168 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log create mode 100644 testing/btest/scripts/base/frameworks/logging/ascii-escape-set-separator.bro diff --git a/src/Desc.cc b/src/Desc.cc index c70878de34..269af07b3b 100644 --- a/src/Desc.cc +++ b/src/Desc.cc @@ -41,8 +41,7 @@ ODesc::ODesc(desc_type t, BroFile* arg_f) do_flush = 1; include_stats = 0; indent_with_spaces = 0; - escape = 0; - escape_len = 0; + escape = false; } ODesc::~ODesc() @@ -56,10 +55,9 @@ ODesc::~ODesc() free(base); } -void ODesc::SetEscape(const char* arg_escape, int len) +void ODesc::EnableEscaping() { - escape = arg_escape; - escape_len = len; + escape = true; } void ODesc::PushIndent() @@ -228,6 +226,25 @@ static const char* find_first_unprintable(ODesc* d, const char* bytes, unsigned return 0; } +pair ODesc::FirstEscapeLoc(const char* bytes, size_t n) + { + pair p(find_first_unprintable(this, bytes, n), 1); + + string str(bytes, n); + list::const_iterator it; + for ( it = escape_sequences.begin(); it != escape_sequences.end(); ++it ) + { + size_t pos = str.find(*it); + if ( pos != string::npos && (p.first == 0 || bytes + pos < p.first) ) + { + p.first = bytes + pos; + p.second = it->size(); + } + } + + return p; + } + void ODesc::AddBytes(const void* bytes, unsigned int n) { if ( ! escape ) @@ -241,45 +258,30 @@ void ODesc::AddBytes(const void* bytes, unsigned int n) while ( s < e ) { - const char* t1 = (const char*) memchr(s, escape[0], e - s); - - if ( ! t1 ) - t1 = e; - - const char* t2 = find_first_unprintable(this, s, t1 - s); - - if ( t2 && t2 < t1 ) + pair p = FirstEscapeLoc(s, e - s); + if ( p.first ) { - AddBytesRaw(s, t2 - s); - - char hex[6] = "\\x00"; - hex[2] = hex_chars[((*t2) & 0xf0) >> 4]; - hex[3] = hex_chars[(*t2) & 0x0f]; - AddBytesRaw(hex, 4); - - s = t2 + 1; - continue; + AddBytesRaw(s, p.first - s); + if ( p.second == 1 ) + { + char hex[6] = "\\x00"; + hex[2] = hex_chars[((*p.first) & 0xf0) >> 4]; + hex[3] = hex_chars[(*p.first) & 0x0f]; + AddBytesRaw(hex, 4); + } + else + { + string esc_str = get_escaped_string(string(p.first, p.second)); + AddBytesRaw(esc_str.c_str(), esc_str.size()); + } + s = p.first + p.second; } - - if ( memcmp(t1, escape, escape_len) != 0 ) - break; - - AddBytesRaw(s, t1 - s); - - for ( int i = 0; i < escape_len; ++i ) + else { - char hex[5] = "\\x00"; - hex[2] = hex_chars[((*t1) & 0xf0) >> 4]; - hex[3] = hex_chars[(*t1) & 0x0f]; - AddBytesRaw(hex, 4); - ++t1; + AddBytesRaw(s, e - s); + break; } - - s = t1; } - - if ( s < e ) - AddBytesRaw(s, e - s); } void ODesc::AddBytesRaw(const void* bytes, unsigned int n) diff --git a/src/Desc.h b/src/Desc.h index 4ed05c1763..27cbd4fa01 100644 --- a/src/Desc.h +++ b/src/Desc.h @@ -4,6 +4,8 @@ #define descriptor_h #include +#include +#include #include "BroString.h" typedef enum { @@ -48,8 +50,13 @@ public: void SetFlush(int arg_do_flush) { do_flush = arg_do_flush; } - // The string passed in must remain valid as long as this object lives. - void SetEscape(const char* escape, int len); + void EnableEscaping(); + void AddEscapeSequence(const char* s) { escape_sequences.push_back(s); } + void AddEscapeSequence(const char* s, size_t n) + { escape_sequences.push_back(string(s, n)); } + void RemoveEscapeSequence(const char* s) { escape_sequences.remove(s); } + void RemoveEscapeSequence(const char* s, size_t n) + { escape_sequences.remove(string(s, n)); } void PushIndent(); void PopIndent(); @@ -133,6 +140,19 @@ protected: void OutOfMemory(); + /** + * Returns the location of the first place in the bytes to be hex-escaped. + * + * @param bytes the starting memory address to start searching for + * escapable character. + * @param n the maximum number of bytes to search. + * @return a pair whose first element represents a starting memory address + * to be escaped up to the number of characters indicated by the + * second element. The first element may be 0 if nothing is + * to be escaped. + */ + pair FirstEscapeLoc(const char* bytes, size_t n); + desc_type type; desc_style style; @@ -140,8 +160,8 @@ protected: unsigned int offset; // where we are in the buffer unsigned int size; // size of buffer in bytes - int escape_len; // number of bytes in to escape sequence - const char* escape; // bytes to escape on output + bool escape; // escape unprintable characters in output? + list escape_sequences; // additional sequences of chars to escape BroFile* f; // or the file we're using. diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 9b1fda3b62..ff9bd77ad6 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -6,27 +6,6 @@ #include "LogWriterAscii.h" #include "NetVar.h" -/** - * Takes a string, escapes each character into its equivalent hex code (\x##), and - * returns a string containing all escaped values. - * - * @param str string to escape - * @return A std::string containing a list of escaped hex values of the form \x## - */ -static string get_escaped_string(const std::string& str) -{ - char tbuf[16]; - string esc = ""; - - for ( size_t i = 0; i < str.length(); ++i ) - { - snprintf(tbuf, sizeof(tbuf), "\\x%02x", str[i]); - esc += tbuf; - } - - return esc; -} - LogWriterAscii::LogWriterAscii() { file = 0; @@ -59,7 +38,8 @@ LogWriterAscii::LogWriterAscii() memcpy(header_prefix, BifConst::LogAscii::header_prefix->Bytes(), header_prefix_len); - desc.SetEscape(separator, separator_len); + desc.EnableEscaping(); + desc.AddEscapeSequence(separator, separator_len); } LogWriterAscii::~LogWriterAscii() @@ -108,7 +88,13 @@ bool LogWriterAscii::DoInit(string path, int num_fields, if( fwrite(str.c_str(), str.length(), 1, file) != 1 ) goto write_error; - if ( ! WriteHeaderField("path", path) ) + if ( ! (WriteHeaderField("set_separator", get_escaped_string( + string(set_separator, set_separator_len))) && + WriteHeaderField("empty_field", get_escaped_string( + string(empty_field, empty_field_len))) && + WriteHeaderField("unset_field", get_escaped_string( + string(unset_field, unset_field_len))) && + WriteHeaderField("path", path)) ) goto write_error; string names; @@ -238,14 +224,19 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) break; } + desc->AddEscapeSequence(set_separator, set_separator_len); for ( int j = 0; j < val->val.set_val.size; j++ ) { if ( j > 0 ) - desc->AddN(set_separator, set_separator_len); + desc->AddRaw(set_separator, set_separator_len); if ( ! DoWriteOne(desc, val->val.set_val.vals[j], field) ) + { + desc->RemoveEscapeSequence(set_separator, set_separator_len); return false; + } } + desc->RemoveEscapeSequence(set_separator, set_separator_len); break; } @@ -258,14 +249,19 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) break; } + desc->AddEscapeSequence(set_separator, set_separator_len); for ( int j = 0; j < val->val.vector_val.size; j++ ) { if ( j > 0 ) - desc->AddN(set_separator, set_separator_len); + desc->AddRaw(set_separator, set_separator_len); if ( ! DoWriteOne(desc, val->val.vector_val.vals[j], field) ) + { + desc->RemoveEscapeSequence(set_separator, set_separator_len); return false; + } } + desc->RemoveEscapeSequence(set_separator, set_separator_len); break; } diff --git a/src/util.cc b/src/util.cc index f81eff8f22..01632a5a97 100644 --- a/src/util.cc +++ b/src/util.cc @@ -41,6 +41,27 @@ #include "Net.h" #include "Reporter.h" +/** + * Takes a string, escapes each character into its equivalent hex code (\x##), and + * returns a string containing all escaped values. + * + * @param str string to escape + * @return A std::string containing a list of escaped hex values of the form \x## + */ +std::string get_escaped_string(const std::string& str) +{ + char tbuf[16]; + string esc = ""; + + for ( size_t i = 0; i < str.length(); ++i ) + { + snprintf(tbuf, sizeof(tbuf), "\\x%02x", str[i]); + esc += tbuf; + } + + return esc; +} + char* copy_string(const char* s) { char* c = new char[strlen(s)+1]; diff --git a/src/util.h b/src/util.h index 6e76b0f61f..83986b59c3 100644 --- a/src/util.h +++ b/src/util.h @@ -89,6 +89,8 @@ void delete_each(T* t) delete *it; } +std::string get_escaped_string(const std::string& str); + extern char* copy_string(const char* s); extern int streq(const char* s1, const char* s2); diff --git a/testing/btest/Baseline/core.expr-exception/reporter.log b/testing/btest/Baseline/core.expr-exception/reporter.log index 2dfe6b7b8e..b2a412d1d3 100644 --- a/testing/btest/Baseline/core.expr-exception/reporter.log +++ b/testing/btest/Baseline/core.expr-exception/reporter.log @@ -1,13 +1,16 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path reporter #fields ts level message location #types time enum string string -1300475168.783842 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.915940 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.916118 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.918295 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.952193 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.952228 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.954761 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.962628 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475169.780331 Reporter::ERROR field value missing [c$ftp] /home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.783842 Reporter::ERROR field value missing [c$ftp] /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.915940 Reporter::ERROR field value missing [c$ftp] /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.916118 Reporter::ERROR field value missing [c$ftp] /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.918295 Reporter::ERROR field value missing [c$ftp] /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.952193 Reporter::ERROR field value missing [c$ftp] /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.952228 Reporter::ERROR field value missing [c$ftp] /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.954761 Reporter::ERROR field value missing [c$ftp] /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.962628 Reporter::ERROR field value missing [c$ftp] /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475169.780331 Reporter::ERROR field value missing [c$ftp] /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 diff --git a/testing/btest/Baseline/core.print-bpf-filters-ipv4/conn.log b/testing/btest/Baseline/core.print-bpf-filters-ipv4/conn.log index 3736748484..915c356a1f 100644 --- a/testing/btest/Baseline/core.print-bpf-filters-ipv4/conn.log +++ b/testing/btest/Baseline/core.print-bpf-filters-ipv4/conn.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path conn #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes #types time string addr port addr port enum string interval count count string bool count string count count count count diff --git a/testing/btest/Baseline/core.print-bpf-filters-ipv4/output b/testing/btest/Baseline/core.print-bpf-filters-ipv4/output index 4f6230b768..bfcf8318c9 100644 --- a/testing/btest/Baseline/core.print-bpf-filters-ipv4/output +++ b/testing/btest/Baseline/core.print-bpf-filters-ipv4/output @@ -1,20 +1,32 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path packet_filter #fields ts node filter init success #types time string string bool bool -1320367155.152502 - not ip6 T T +1323275491.966719 - not ip6 T T #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path packet_filter #fields ts node filter init success #types time string string bool bool -1320367155.379066 - (((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)) and (not ip6) T T +1323275492.165829 - (((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)) and (not ip6) T T #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path packet_filter #fields ts node filter init success #types time string string bool bool -1320367155.601980 - port 42 T T +1323275492.362403 - port 42 T T #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path packet_filter #fields ts node filter init success #types time string string bool bool -1320367155.826539 - port 56730 T T +1323275492.563649 - port 56730 T T diff --git a/testing/btest/Baseline/core.vlan-mpls/conn.log b/testing/btest/Baseline/core.vlan-mpls/conn.log index 69e23f3875..48be03014c 100644 --- a/testing/btest/Baseline/core.vlan-mpls/conn.log +++ b/testing/btest/Baseline/core.vlan-mpls/conn.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path conn #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes #types time string addr port addr port enum string interval count count string bool count string count count count count diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 6819dc0813..49f3d79365 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path loaded_scripts #fields name #types string diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 7a461a3903..6587b1ac97 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path loaded_scripts #fields name #types string diff --git a/testing/btest/Baseline/istate.events-ssl/receiver.http.log b/testing/btest/Baseline/istate.events-ssl/receiver.http.log index 06d453c241..f98864b0b1 100644 --- a/testing/btest/Baseline/istate.events-ssl/receiver.http.log +++ b/testing/btest/Baseline/istate.events-ssl/receiver.http.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table string string table string string file -1319568535.914761 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - +1323276411.786237 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - diff --git a/testing/btest/Baseline/istate.events-ssl/sender.http.log b/testing/btest/Baseline/istate.events-ssl/sender.http.log index 06d453c241..f98864b0b1 100644 --- a/testing/btest/Baseline/istate.events-ssl/sender.http.log +++ b/testing/btest/Baseline/istate.events-ssl/sender.http.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table string string table string string file -1319568535.914761 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - +1323276411.786237 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - diff --git a/testing/btest/Baseline/istate.events/receiver.http.log b/testing/btest/Baseline/istate.events/receiver.http.log index d85d560b6d..028f33db42 100644 --- a/testing/btest/Baseline/istate.events/receiver.http.log +++ b/testing/btest/Baseline/istate.events/receiver.http.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table string string table string string file -1319568558.542142 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - +1323276438.655853 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - diff --git a/testing/btest/Baseline/istate.events/sender.http.log b/testing/btest/Baseline/istate.events/sender.http.log index d85d560b6d..028f33db42 100644 --- a/testing/btest/Baseline/istate.events/sender.http.log +++ b/testing/btest/Baseline/istate.events/sender.http.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table string string table string string file -1319568558.542142 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - +1323276438.655853 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log index 7f71757ca0..665d21c153 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log +++ b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log @@ -1,16 +1,19 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path communication #fields ts peer src_name connected_peer_desc connected_peer_addr connected_peer_port level message #types time string string string addr port string string -1322788789.351248 bro parent - - - info [#1/127.0.0.1:47757] added peer -1322788789.354851 bro child - - - info [#1/127.0.0.1:47757] connected -1322788789.354956 bro parent - - - info [#1/127.0.0.1:47757] peer connected -1322788789.354956 bro parent - - - info [#1/127.0.0.1:47757] phase: version -1322788789.355429 bro script - - - info connection established -1322788789.355429 bro script - - - info requesting events matching /^?(NOTHING)$?/ -1322788789.355429 bro script - - - info accepting state -1322788789.355967 bro parent - - - info [#1/127.0.0.1:47757] phase: handshake -1322788789.355967 bro parent - - - info warning: no events to request -1322788789.355967 bro parent - - - info terminating... -1322788789.355967 bro parent - - - info [#1/127.0.0.1:47757] peer_description is bro -1322788789.355967 bro parent - - - info [#1/127.0.0.1:47757] closing connection +1323275566.293849 bro parent - - - info [#1/127.0.0.1:47757] added peer +1323275566.300180 bro child - - - info [#1/127.0.0.1:47757] connected +1323275566.300467 bro parent - - - info [#1/127.0.0.1:47757] peer connected +1323275566.300467 bro parent - - - info [#1/127.0.0.1:47757] phase: version +1323275566.300936 bro script - - - info connection established +1323275566.300936 bro script - - - info requesting events matching /^?(NOTHING)$?/ +1323275566.300936 bro script - - - info accepting state +1323275566.302043 bro parent - - - info [#1/127.0.0.1:47757] phase: handshake +1323275566.302043 bro parent - - - info warning: no events to request +1323275566.302043 bro parent - - - info terminating... +1323275566.302043 bro parent - - - info [#1/127.0.0.1:47757] peer_description is bro +1323275566.302043 bro parent - - - info [#1/127.0.0.1:47757] closing connection diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log b/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log index fc2c133dc6..76edd50f26 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log @@ -1,6 +1,9 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path ssh-new-default #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167052.603186 1.2.3.4 1234 2.3.4.5 80 success unknown -1315167052.603186 1.2.3.4 1234 2.3.4.5 80 failure US +1323275589.577486 1.2.3.4 1234 2.3.4.5 80 success unknown +1323275589.577486 1.2.3.4 1234 2.3.4.5 80 failure US diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log index b236cb818b..f0006dbe37 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log @@ -1,4 +1,7 @@ #separator \x7c +#set_separator|\x2c +#empty_field|\x2d +#unset_field|\x2d #path|ssh #fields|data|data2 #types|string|string diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log index e1ba48cf8e..b11fd30678 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log @@ -1,9 +1,12 @@ PREFIX<>separator \x7c +PREFIX<>set_separator|\x2c +PREFIX<>empty_field|\x45\x4d\x50\x54\x59 +PREFIX<>unset_field|\x4e\x4f\x54\x2d\x53\x45\x54 PREFIX<>path|ssh PREFIX<>fields|t|id.orig_h|id.orig_p|id.resp_h|id.resp_p|status|country|b PREFIX<>types|time|addr|port|addr|port|string|string|bool -1315167052.828457|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET -1315167052.828457|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET -1315167052.828457|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET -1315167052.828457|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET -1315167052.828457|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T +1323275635.348361|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET +1323275635.348361|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET +1323275635.348361|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET +1323275635.348361|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET +1323275635.348361|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log index 683fed60f2..8c6fbee126 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path test #fields x y z #types string string string diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log index db9ce497ed..4e06ce3e2c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table string string table string string file diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log new file mode 100644 index 0000000000..34f8668cbf --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log @@ -0,0 +1,8 @@ +#separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d +#path test +#fields ss +#types table +CC,AA,\x2c,\x2c\x2c diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log index 3100fa0cb2..84e2991a47 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log @@ -1,9 +1,12 @@ #separator \x7c\x7c +#set_separator||\x2c +#empty_field||\x2d +#unset_field||\x2d #path||ssh #fields||t||id.orig_h||id.orig_p||id.resp_h||id.resp_p||status||country #types||time||addr||port||addr||port||string||string -1315802040.006123||1.2.3.4||1234||2.3.4.5||80||success||unknown -1315802040.006123||1.2.3.4||1234||2.3.4.5||80||failure||US -1315802040.006123||1.2.3.4||1234||2.3.4.5||80||fa\x7c\x7cure||UK -1315802040.006123||1.2.3.4||1234||2.3.4.5||80||su\x7c\x7cess||BR -1315802040.006123||1.2.3.4||1234||2.3.4.5||80||failure||MX +1323275761.036351||1.2.3.4||1234||2.3.4.5||80||success||unknown +1323275761.036351||1.2.3.4||1234||2.3.4.5||80||failure||US +1323275761.036351||1.2.3.4||1234||2.3.4.5||80||fa\x7c\x7cure||UK +1323275761.036351||1.2.3.4||1234||2.3.4.5||80||su\x7c\x7cess||BR +1323275761.036351||1.2.3.4||1234||2.3.4.5||80||failure||MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log index 7f512c15d9..7a05376b43 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path test #fields data #types time diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log index c2c32c5c6a..e79320b415 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path ssh #fields status country a1 b1 b2 #types string string count count count diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log index 18e4d5cbad..73266386c8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path ssh #fields status country #types string string diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log index 49272bfd53..1e25a5b664 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log @@ -1,9 +1,12 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path ssh #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.369918 1.2.3.4 1234 2.3.4.5 80 success unknown -1315167053.369918 1.2.3.4 1234 2.3.4.5 80 failure US -1315167053.369918 1.2.3.4 1234 2.3.4.5 80 failure UK -1315167053.369918 1.2.3.4 1234 2.3.4.5 80 success BR -1315167053.369918 1.2.3.4 1234 2.3.4.5 80 failure MX +1323275824.696040 1.2.3.4 1234 2.3.4.5 80 success unknown +1323275824.696040 1.2.3.4 1234 2.3.4.5 80 failure US +1323275824.696040 1.2.3.4 1234 2.3.4.5 80 failure UK +1323275824.696040 1.2.3.4 1234 2.3.4.5 80 success BR +1323275824.696040 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log index b078b4746a..b70046bd15 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path ssh #fields id.orig_p id.resp_h id.resp_p status country #types port addr port string string diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log index 0a988ff9b9..47d0f93f2e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path ssh #fields t f #types time file -1315167053.585834 Foo.log +1323275842.508479 Foo.log diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log index 5675ef6632..e1b1d18ddc 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log @@ -1,9 +1,12 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path ssh #fields t id.orig_h #types time addr -1315167053.694473 1.2.3.4 -1315167053.694473 1.2.3.4 -1315167053.694473 1.2.3.4 -1315167053.694473 1.2.3.4 -1315167053.694473 1.2.3.4 +1323275846.507507 1.2.3.4 +1323275846.507507 1.2.3.4 +1323275846.507507 1.2.3.4 +1323275846.507507 1.2.3.4 +1323275846.507507 1.2.3.4 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log index d8d90cf1fa..1cc0c681fb 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path local #fields ts id.orig_h #types time addr diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log index a17c2821f5..20e096f399 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path remote #fields ts id.orig_h #types time addr diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output index 2c196340cc..d62ae42069 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output @@ -6,37 +6,58 @@ static-prefix-1-US.log static-prefix-2-MX2.log static-prefix-2-UK.log #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path static-prefix-0-BR #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.803346 1.2.3.4 1234 2.3.4.5 80 success BR +1323275860.153895 1.2.3.4 1234 2.3.4.5 80 success BR #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path static-prefix-0-MX3 #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.803346 1.2.3.4 1234 2.3.4.5 80 failure MX3 +1323275860.153895 1.2.3.4 1234 2.3.4.5 80 failure MX3 #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path static-prefix-0-unknown #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.803346 1.2.3.4 1234 2.3.4.5 80 success unknown +1323275860.153895 1.2.3.4 1234 2.3.4.5 80 success unknown #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path static-prefix-1-MX #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.803346 1.2.3.4 1234 2.3.4.5 80 failure MX +1323275860.153895 1.2.3.4 1234 2.3.4.5 80 failure MX #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path static-prefix-1-US #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.803346 1.2.3.4 1234 2.3.4.5 80 failure US +1323275860.153895 1.2.3.4 1234 2.3.4.5 80 failure US #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path static-prefix-2-MX2 #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.803346 1.2.3.4 1234 2.3.4.5 80 failure MX2 +1323275860.153895 1.2.3.4 1234 2.3.4.5 80 failure MX2 #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path static-prefix-2-UK #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.803346 1.2.3.4 1234 2.3.4.5 80 failure UK +1323275860.153895 1.2.3.4 1234 2.3.4.5 80 failure UK diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log index ba688d7843..f6a0779ff3 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path test.failure #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.923545 1.2.3.4 1234 2.3.4.5 80 failure US +1323275882.725518 1.2.3.4 1234 2.3.4.5 80 failure US diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log index 7a91b1a2d9..4744d0df37 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path test.success #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167053.923545 1.2.3.4 1234 2.3.4.5 80 success - +1323275882.725518 1.2.3.4 1234 2.3.4.5 80 success - diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log index c00e7765d5..efd68918de 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator \x2c +#empty_field \x45\x4d\x50\x54\x59 +#unset_field \x2d #path test #fields b i e c p sn a d t iv s sc ss se vc ve #types bool int enum count port subnet addr double time interval string table table table vector vector -T -42 Test::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315167054.320958 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY +T -42 Test::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1323275900.286451 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log index aba9fdddd9..f6fdc8d4a7 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log @@ -1,7 +1,10 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path test.failure #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure US -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure UK -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure MX +1323276013.684540 1.2.3.4 1234 2.3.4.5 80 failure US +1323276013.684540 1.2.3.4 1234 2.3.4.5 80 failure UK +1323276013.684540 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log index b928c37685..48c2ebe139 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log @@ -1,9 +1,12 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 success - -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure US -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure UK -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 success BR -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 failure MX +1323276013.684540 1.2.3.4 1234 2.3.4.5 80 success - +1323276013.684540 1.2.3.4 1234 2.3.4.5 80 failure US +1323276013.684540 1.2.3.4 1234 2.3.4.5 80 failure UK +1323276013.684540 1.2.3.4 1234 2.3.4.5 80 success BR +1323276013.684540 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log index a951c6ed1a..37e2798856 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log @@ -1,6 +1,9 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path test.success #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 success - -1315167059.502670 1.2.3.4 1234 2.3.4.5 80 success BR +1323276013.684540 1.2.3.4 1234 2.3.4.5 80 success - +1323276013.684540 1.2.3.4 1234 2.3.4.5 80 success BR diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log index 6185e86028..9256f309ce 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log @@ -1,6 +1,9 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path ssh.failure #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167066.575996 1.2.3.4 1234 2.3.4.5 80 failure US -1315167066.575996 1.2.3.4 1234 2.3.4.5 80 failure UK +1323276050.103643 1.2.3.4 1234 2.3.4.5 80 failure US +1323276050.103643 1.2.3.4 1234 2.3.4.5 80 failure UK diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log index a4ec2dc7de..2e70ef44f3 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log @@ -1,7 +1,10 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path ssh #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167066.575996 1.2.3.4 1234 2.3.4.5 80 failure US -1315167066.575996 1.2.3.4 1234 2.3.4.5 80 failure UK -1315167066.575996 1.2.3.4 1234 2.3.4.5 80 failure BR +1323276050.103643 1.2.3.4 1234 2.3.4.5 80 failure US +1323276050.103643 1.2.3.4 1234 2.3.4.5 80 failure UK +1323276050.103643 1.2.3.4 1234 2.3.4.5 80 failure BR diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out index 337ed3ca32..71d0413464 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out @@ -18,11 +18,14 @@ custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_11.00.05.log, pat custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_11.59.55.log, path=test2, open=1299499195.0, close=1299499205.0, terminating=F] custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_12.00.05.log, path=test2, open=1299499205.0, close=1299502795.0, terminating=F] custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_12.59.55.log, path=test2, open=1299502795.0, close=1299502795.0, terminating=T] +#empty_field \x2d #fields t id.orig_h id.orig_p id.resp_h id.resp_p #path test #path test2 #separator \x09 +#set_separator \x2c #types time addr port addr port +#unset_field \x2d 1299466805.000000 10.0.0.1 20 10.0.0.2 1024 1299470395.000000 10.0.0.2 20 10.0.0.3 0 1299470405.000000 10.0.0.1 20 10.0.0.2 1025 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out index 74ce45023a..a5f9ae758b 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out @@ -10,6 +10,9 @@ test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0 test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 > test.2011-03-07-03-00-05.log #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -17,6 +20,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299470395.000000 10.0.0.2 20 10.0.0.3 0 > test.2011-03-07-04-00-05.log #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -24,6 +30,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299473995.000000 10.0.0.2 20 10.0.0.3 1 > test.2011-03-07-05-00-05.log #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -31,6 +40,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299477595.000000 10.0.0.2 20 10.0.0.3 2 > test.2011-03-07-06-00-05.log #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -38,6 +50,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299481195.000000 10.0.0.2 20 10.0.0.3 3 > test.2011-03-07-07-00-05.log #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -45,6 +60,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299484795.000000 10.0.0.2 20 10.0.0.3 4 > test.2011-03-07-08-00-05.log #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -52,6 +70,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299488395.000000 10.0.0.2 20 10.0.0.3 5 > test.2011-03-07-09-00-05.log #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -59,6 +80,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299491995.000000 10.0.0.2 20 10.0.0.3 6 > test.2011-03-07-10-00-05.log #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -66,6 +90,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299495595.000000 10.0.0.2 20 10.0.0.3 7 > test.2011-03-07-11-00-05.log #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -73,6 +100,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299499195.000000 10.0.0.2 20 10.0.0.3 8 > test.2011-03-07-12-00-05.log #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output b/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output index 84521cb645..65c24b8752 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output @@ -1,9 +1,12 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path /dev/stdout #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167067.393739 1.2.3.4 1234 2.3.4.5 80 success unknown -1315167067.393739 1.2.3.4 1234 2.3.4.5 80 failure US -1315167067.393739 1.2.3.4 1234 2.3.4.5 80 failure UK -1315167067.393739 1.2.3.4 1234 2.3.4.5 80 success BR -1315167067.393739 1.2.3.4 1234 2.3.4.5 80 failure MX +1323276116.980214 1.2.3.4 1234 2.3.4.5 80 success unknown +1323276116.980214 1.2.3.4 1234 2.3.4.5 80 failure US +1323276116.980214 1.2.3.4 1234 2.3.4.5 80 failure UK +1323276116.980214 1.2.3.4 1234 2.3.4.5 80 success BR +1323276116.980214 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log index 5b93b6e23b..b283ad8856 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log @@ -1,9 +1,12 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path ssh #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1315167067.507542 1.2.3.4 1234 2.3.4.5 80 success unknown -1315167067.507542 1.2.3.4 1234 2.3.4.5 80 failure US -1315167067.507542 1.2.3.4 1234 2.3.4.5 80 failure UK -1315167067.507542 1.2.3.4 1234 2.3.4.5 80 success BR -1315167067.507542 1.2.3.4 1234 2.3.4.5 80 failure MX +1323276164.251500 1.2.3.4 1234 2.3.4.5 80 success unknown +1323276164.251500 1.2.3.4 1234 2.3.4.5 80 failure US +1323276164.251500 1.2.3.4 1234 2.3.4.5 80 failure UK +1323276164.251500 1.2.3.4 1234 2.3.4.5 80 success BR +1323276164.251500 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log index ffd579c224..4ed3876d43 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator \x2c +#empty_field \x45\x4d\x50\x54\x59 +#unset_field \x2d #path ssh #fields b i e c p sn a d t iv s sc ss se vc ve f #types bool int enum count port subnet addr double time interval string table table table vector vector func -T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1323276169.782634 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log b/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log index 12bb1d1704..a9acb91cd3 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path testing #fields a.val1 a.val2 b #types count count count diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log index b9a54404ed..b1b07c3ae2 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path ssh #fields vec #types vector diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log index 1677297ecc..f3485833d2 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log @@ -1,7 +1,10 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path metrics #fields ts metric_id filter_name index.host index.str index.network value #types time enum string addr string subnet count -1317950616.401733 TEST_METRIC foo-bar 6.5.4.3 - - 4 -1317950616.401733 TEST_METRIC foo-bar 1.2.3.4 - - 6 -1317950616.401733 TEST_METRIC foo-bar 7.2.1.5 - - 2 +1323276206.622034 TEST_METRIC foo-bar 6.5.4.3 - - 4 +1323276206.622034 TEST_METRIC foo-bar 1.2.3.4 - - 6 +1323276206.622034 TEST_METRIC foo-bar 7.2.1.5 - - 2 diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log index 45334cf3d7..ef8fe73611 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log @@ -1,7 +1,10 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path metrics #fields ts metric_id filter_name index.host index.str index.network value #types time enum string addr string subnet count -1315167083.455574 TEST_METRIC foo-bar 6.5.4.3 - - 2 -1315167083.455574 TEST_METRIC foo-bar 1.2.3.4 - - 3 -1315167083.455574 TEST_METRIC foo-bar 7.2.1.5 - - 1 +1323276222.644659 TEST_METRIC foo-bar 6.5.4.3 - - 2 +1323276222.644659 TEST_METRIC foo-bar 1.2.3.4 - - 3 +1323276222.644659 TEST_METRIC foo-bar 7.2.1.5 - - 1 diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log index f5df2e96f3..9fadaa76d9 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network #types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet -1316952194.679491 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 6 3600.000000 - - - - - - 1.2.3.4 - - +1323276259.751377 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 6 3600.000000 - - - - - - 1.2.3.4 - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log index 33745500e0..b2a18dd011 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log @@ -1,6 +1,9 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network #types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet -1316952223.891502 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 3/2 - 1.2.3.4 - - 3 bro Notice::ACTION_LOG 6 3600.000000 - - - - - - 1.2.3.4 - - -1316952223.891502 - - - - - Test_Notice Threshold crossed by metric_index(host=6.5.4.3) 2/2 - 6.5.4.3 - - 2 bro Notice::ACTION_LOG 6 3600.000000 - - - - - - 6.5.4.3 - - +1323276275.255136 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 3/2 - 1.2.3.4 - - 3 bro Notice::ACTION_LOG 6 3600.000000 - - - - - - 1.2.3.4 - - +1323276275.255136 - - - - - Test_Notice Threshold crossed by metric_index(host=6.5.4.3) 2/2 - 6.5.4.3 - - 2 bro Notice::ACTION_LOG 6 3600.000000 - - - - - - 6.5.4.3 - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log index 0662c13294..be6c90d3da 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network #types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet -1316952264.931290 - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 6 3600.000000 - - - - - - - - - +1323276288.745044 - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 6 3600.000000 - - - - - - - - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log index 6e0214b7d3..c8cfaac901 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network #types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet -1316950574.408256 - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 6 3600.000000 - - - - - - - - - +1323276310.879512 - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 6 3600.000000 - - - - - - - - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log index 6b4c925e0f..b939a49cc3 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude #types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double -1316950497.513136 - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 6 3600.000000 - - - - - - +1323276329.733314 - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 6 3600.000000 - - - - - - diff --git a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log index 812b4bc151..26d355908d 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table string string table string string file diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log index 386eaf8901..2a56bc65c7 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table string string table string string file diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log index 9515eb8168..f236bdff34 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table string string table string string file diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log index 01d62b3981..f8d9689918 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table string string table string file diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log index d224556632..64fc0135b5 100644 --- a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log +++ b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path irc #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user channels command value addl tags dcc_file_name dcc_file_size extraction_file #types time string addr port addr port string string table string string string table string count file diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log index a692d2dd4d..d1727c0f13 100644 --- a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log +++ b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path irc #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user channels command value addl tags dcc_file_name dcc_file_size dcc_mime_type extraction_file #types time string addr port addr port string string table string string string table string count string file diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log b/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log index b93720cfe6..2ebe4482b3 100644 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path smtp #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth helo mailfrom rcptto date from to reply_to msg_id in_reply_to subject x_originating_ip first_received second_received last_reply path user_agent #types time string addr port addr port count string string table string string table string string string string addr string string string vector string diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log index 63b287a791..13ec1b02d5 100644 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path smtp_entities #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth filename content_len mime_type md5 extraction_file excerpt #types time string addr port addr port count string count string string file string diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log index e45d8dc757..26e0c2dc01 100644 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path smtp_entities #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth filename content_len mime_type md5 extraction_file excerpt #types time string addr port addr port count string count string string file string diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log index cde5156594..13646617a9 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path known_hosts #fields ts host #types time addr diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log index 008eb364ed..3ab2a480b0 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path known_hosts #fields ts host #types time addr diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log index 43b28ded8a..cc06e288aa 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path known_hosts #fields ts host #types time addr diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log index ad9fa52e1c..3d2fa81d65 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path known_services #fields ts host port_num port_proto service #types time addr port enum table diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log index 1607d69f24..50caa79184 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path known_services #fields ts host port_num port_proto service #types time addr port enum table diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log index 0d1210c941..8276470858 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path known_services #fields ts host port_num port_proto service #types time addr port enum table diff --git a/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log b/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log index 78e61070d7..6839aa25e1 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log +++ b/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log @@ -1,4 +1,7 @@ #separator \x09 +#set_separator \x2c +#empty_field \x2d +#unset_field \x2d #path dns #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id query qclass qclass_name qtype qtype_name rcode rcode_name QR AA TC RD RA Z answers TTLs auth addl #types time string addr port addr port enum count string count string count string count string bool bool bool bool bool count vector vector table table diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-escape-set-separator.bro b/testing/btest/scripts/base/frameworks/logging/ascii-escape-set-separator.bro new file mode 100644 index 0000000000..f5fb7a6259 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/ascii-escape-set-separator.bro @@ -0,0 +1,21 @@ +# @TEST-EXEC: bro -b %INPUT +# @TEST-EXEC: btest-diff test.log + +module Test; + +export { + redef enum Log::ID += { LOG }; + + type Log: record { + ss: set[string]; + } &log; +} + +event bro_init() +{ + Log::create_stream(Test::LOG, [$columns=Log]); + + + Log::write(Test::LOG, [$ss=set("AA", ",", ",,", "CC")]); +} + From a9f0b10e2e45473c99e81e00252307455a3c3f02 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 19 Dec 2011 07:44:29 -0800 Subject: [PATCH 204/964] Updating baselines for recent commits. --- .../btest/Baseline/istate.events-ssl/receiver.http.log | 4 ++-- .../btest/Baseline/istate.events-ssl/sender.http.log | 4 ++-- testing/btest/Baseline/istate.events/receiver.http.log | 4 ++-- testing/btest/Baseline/istate.events/sender.http.log | 4 ++-- .../test.log | 2 +- .../test.success.log | 2 +- .../receiver.test.log | 5 ++++- .../sender.test.log | 10 +++++----- .../sender.test.success.log | 4 ++-- .../scripts.base.frameworks.logging.types/ssh.log | 5 ++++- .../manager-1.notice.log | 4 ++-- .../scripts.base.frameworks.metrics.notice/notice.log | 6 +++--- .../manager-1.notice.log | 4 ++-- .../manager-1.notice.log | 4 ++-- .../notice.log | 4 ++-- .../scripts.base.protocols.http.100-continue/http.log | 2 +- .../http.log | 2 +- .../http.log | 2 +- .../http.log | 2 +- .../Baseline/scripts.base.protocols.irc.basic/irc.log | 2 +- .../scripts.base.protocols.irc.dcc-extract/irc.log | 2 +- .../scripts.base.protocols.smtp.basic/smtp.log | 2 +- .../knownservices-all.log | 2 +- .../knownservices-local.log | 2 +- .../knownservices-remote.log | 2 +- .../dns.log | 2 +- 26 files changed, 47 insertions(+), 41 deletions(-) diff --git a/testing/btest/Baseline/istate.events-ssl/receiver.http.log b/testing/btest/Baseline/istate.events-ssl/receiver.http.log index f98864b0b1..d4b71d5d53 100644 --- a/testing/btest/Baseline/istate.events-ssl/receiver.http.log +++ b/testing/btest/Baseline/istate.events-ssl/receiver.http.log @@ -4,5 +4,5 @@ #unset_field \x2d #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file -1323276411.786237 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file +1324308802.436269 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - diff --git a/testing/btest/Baseline/istate.events-ssl/sender.http.log b/testing/btest/Baseline/istate.events-ssl/sender.http.log index f98864b0b1..d4b71d5d53 100644 --- a/testing/btest/Baseline/istate.events-ssl/sender.http.log +++ b/testing/btest/Baseline/istate.events-ssl/sender.http.log @@ -4,5 +4,5 @@ #unset_field \x2d #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file -1323276411.786237 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file +1324308802.436269 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - diff --git a/testing/btest/Baseline/istate.events/receiver.http.log b/testing/btest/Baseline/istate.events/receiver.http.log index 028f33db42..efb76a9af1 100644 --- a/testing/btest/Baseline/istate.events/receiver.http.log +++ b/testing/btest/Baseline/istate.events/receiver.http.log @@ -4,5 +4,5 @@ #unset_field \x2d #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file -1323276438.655853 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file +1324308826.107003 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - diff --git a/testing/btest/Baseline/istate.events/sender.http.log b/testing/btest/Baseline/istate.events/sender.http.log index 028f33db42..efb76a9af1 100644 --- a/testing/btest/Baseline/istate.events/sender.http.log +++ b/testing/btest/Baseline/istate.events/sender.http.log @@ -4,5 +4,5 @@ #unset_field \x2d #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file -1323276438.655853 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file +1324308826.107003 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log index 34f8668cbf..caf19240e0 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log @@ -4,5 +4,5 @@ #unset_field \x2d #path test #fields ss -#types table +#types table[string] CC,AA,\x2c,\x2c\x2c diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log index 4744d0df37..18f73a3d6c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log @@ -5,4 +5,4 @@ #path test.success #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1323275882.725518 1.2.3.4 1234 2.3.4.5 80 success - +1324308566.444800 1.2.3.4 1234 2.3.4.5 80 success unknown diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log index 320e1a66d8..938bf9ab1a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator \x2c +#empty_field \x45\x4d\x50\x54\x59 +#unset_field \x2d #path test #fields b i e c p sn a d t iv s sc ss se vc ve #types bool int enum count port subnet addr double time interval string table[count] table[string] table[string] vector[count] vector[string] -T -42 Test::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1323292199.700588 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY +T -42 Test::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1324308572.066737 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log index 48c2ebe139..8c4d930c4e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log @@ -5,8 +5,8 @@ #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1323276013.684540 1.2.3.4 1234 2.3.4.5 80 success - -1323276013.684540 1.2.3.4 1234 2.3.4.5 80 failure US -1323276013.684540 1.2.3.4 1234 2.3.4.5 80 failure UK -1323276013.684540 1.2.3.4 1234 2.3.4.5 80 success BR -1323276013.684540 1.2.3.4 1234 2.3.4.5 80 failure MX +1324308589.020941 1.2.3.4 1234 2.3.4.5 80 success unknown +1324308589.020941 1.2.3.4 1234 2.3.4.5 80 failure US +1324308589.020941 1.2.3.4 1234 2.3.4.5 80 failure UK +1324308589.020941 1.2.3.4 1234 2.3.4.5 80 success BR +1324308589.020941 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log index 37e2798856..ac0ce01ee0 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log @@ -5,5 +5,5 @@ #path test.success #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1323276013.684540 1.2.3.4 1234 2.3.4.5 80 success - -1323276013.684540 1.2.3.4 1234 2.3.4.5 80 success BR +1324308589.020941 1.2.3.4 1234 2.3.4.5 80 success unknown +1324308589.020941 1.2.3.4 1234 2.3.4.5 80 success BR diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log index 9b21b85800..7d80a93fd5 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log @@ -1,5 +1,8 @@ #separator \x09 +#set_separator \x2c +#empty_field \x45\x4d\x50\x54\x59 +#unset_field \x2d #path ssh #fields b i e c p sn a d t iv s sc ss se vc ve f #types bool int enum count port subnet addr double time interval string table[count] table[string] table[string] vector[count] vector[string] func -T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1323292210.836187 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1324308607.500960 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log index 9fadaa76d9..cc95398121 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log @@ -4,5 +4,5 @@ #unset_field \x2d #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet -1323276259.751377 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 6 3600.000000 - - - - - - 1.2.3.4 - - +#types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet +1324308631.319990 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log index b2a18dd011..c96078f654 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log @@ -4,6 +4,6 @@ #unset_field \x2d #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet -1323276275.255136 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 3/2 - 1.2.3.4 - - 3 bro Notice::ACTION_LOG 6 3600.000000 - - - - - - 1.2.3.4 - - -1323276275.255136 - - - - - Test_Notice Threshold crossed by metric_index(host=6.5.4.3) 2/2 - 6.5.4.3 - - 2 bro Notice::ACTION_LOG 6 3600.000000 - - - - - - 6.5.4.3 - - +#types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet +1324308665.314874 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 3/2 - 1.2.3.4 - - 3 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - - +1324308665.314874 - - - - - Test_Notice Threshold crossed by metric_index(host=6.5.4.3) 2/2 - 6.5.4.3 - - 2 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - 6.5.4.3 - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log index be6c90d3da..91ee4eb823 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log @@ -4,5 +4,5 @@ #unset_field \x2d #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet -1323276288.745044 - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 6 3600.000000 - - - - - - - - - +#types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet +1324308679.119923 - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log index c8cfaac901..e99bb9ee65 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log @@ -4,5 +4,5 @@ #unset_field \x2d #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double addr string subnet -1323276310.879512 - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 6 3600.000000 - - - - - - - - - +#types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet +1324308705.683375 - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log index b939a49cc3..ce93bb617d 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log @@ -4,5 +4,5 @@ #unset_field \x2d #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude -#types time string addr port addr port enum string string addr addr port count string table table interval bool string string string double double -1323276329.733314 - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 6 3600.000000 - - - - - - +#types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double +1324308722.344582 - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 6 3600.000000 F - - - - - diff --git a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log index 26d355908d..3d711d430c 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log @@ -4,5 +4,5 @@ #unset_field \x2d #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file 1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 1 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 60731 200 OK 100 Continue - - - - - text/html - - diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log index 2a56bc65c7..e063bcbaf5 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log @@ -4,5 +4,5 @@ #unset_field \x2d #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file 1128727435.634189 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log index f236bdff34..c2e74fcae1 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log @@ -4,7 +4,7 @@ #unset_field \x2d #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table string string table string string file +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file 1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - - - - - FAKE_MIME - - 1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - - - - - FAKE_MIME - - 1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - - - - - FAKE_MIME - - diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log index f8d9689918..0246cf450a 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log @@ -4,7 +4,7 @@ #unset_field \x2d #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table string string table string file +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string file 1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - - - - - - - 1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - - - - - - - 1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - - - - - - - diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log index 64fc0135b5..6a200a9e41 100644 --- a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log +++ b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log @@ -4,7 +4,7 @@ #unset_field \x2d #path irc #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user channels command value addl tags dcc_file_name dcc_file_size extraction_file -#types time string addr port addr port string string table string string string table string count file +#types time string addr port addr port string string table[string] string string string table[enum] string count file 1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - - NICK bloed - - - - - 1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - - 1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - JOIN #easymovies - - - - - diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log index d1727c0f13..f3d0b2ff32 100644 --- a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log +++ b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log @@ -4,7 +4,7 @@ #unset_field \x2d #path irc #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user channels command value addl tags dcc_file_name dcc_file_size dcc_mime_type extraction_file -#types time string addr port addr port string string table string string string table string count string file +#types time string addr port addr port string string table[string] string string string table[enum] string count string file 1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - - NICK bloed - - - - - - 1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - - - 1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - JOIN #easymovies - - - - - - diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log b/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log index 2ebe4482b3..4d1b8cca84 100644 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log @@ -4,5 +4,5 @@ #unset_field \x2d #path smtp #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth helo mailfrom rcptto date from to reply_to msg_id in_reply_to subject x_originating_ip first_received second_received last_reply path user_agent -#types time string addr port addr port count string string table string string table string string string string addr string string string vector string +#types time string addr port addr port count string string table[string] string string table[string] string string string string addr string string string vector[addr] string 1254722768.219663 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 GP Mon, 5 Oct 2009 11:36:07 +0530 "Gurpartap Singh" - <000301ca4581$ef9e57f0$cedb07d0$@in> - SMTP - - - 250 OK id=1Mugho-0003Dg-Un 74.53.140.153,10.10.1.4 Microsoft Office Outlook 12.0 diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log index 3d2fa81d65..9e73167aba 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log @@ -4,7 +4,7 @@ #unset_field \x2d #path known_services #fields ts host port_num port_proto service -#types time addr port enum table +#types time addr port enum table[string] 1308930691.049431 172.16.238.131 22 tcp SSH 1308930694.550308 172.16.238.131 80 tcp HTTP 1308930716.462556 74.125.225.81 80 tcp HTTP diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log index 50caa79184..04928321c5 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log @@ -4,7 +4,7 @@ #unset_field \x2d #path known_services #fields ts host port_num port_proto service -#types time addr port enum table +#types time addr port enum table[string] 1308930691.049431 172.16.238.131 22 tcp SSH 1308930694.550308 172.16.238.131 80 tcp HTTP 1308930718.361665 172.16.238.131 21 tcp FTP diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log index 8276470858..edfe160a7d 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log @@ -4,6 +4,6 @@ #unset_field \x2d #path known_services #fields ts host port_num port_proto service -#types time addr port enum table +#types time addr port enum table[string] 1308930716.462556 74.125.225.81 80 tcp HTTP 1308930726.872485 141.142.192.39 22 tcp SSH diff --git a/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log b/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log index 6839aa25e1..c50278a533 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log +++ b/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log @@ -4,5 +4,5 @@ #unset_field \x2d #path dns #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id query qclass qclass_name qtype qtype_name rcode rcode_name QR AA TC RD RA Z answers TTLs auth addl -#types time string addr port addr port enum count string count string count string count string bool bool bool bool bool count vector vector table table +#types time string addr port addr port enum count string count string count string count string bool bool bool bool bool count vector[string] vector[interval] table[string] table[string] 930613226.529070 UWkUyAuUGXf 212.180.42.100 25000 131.243.64.3 53 tcp 34798 - - - - - 0 NOERROR F F F F T 0 4.3.2.1 31337.000000 - - From c1e656d89e554dcf5d809338b18ab8ddbeaac63a Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 19 Dec 2011 08:44:41 -0800 Subject: [PATCH 205/964] In log headers, only escape information when necessary. --- src/Desc.cc | 2 +- src/LogWriterAscii.cc | 10 +++++----- src/util.cc | 24 +++++++++++++++++------- src/util.h | 2 +- 4 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/Desc.cc b/src/Desc.cc index 269af07b3b..12b4a524eb 100644 --- a/src/Desc.cc +++ b/src/Desc.cc @@ -271,7 +271,7 @@ void ODesc::AddBytes(const void* bytes, unsigned int n) } else { - string esc_str = get_escaped_string(string(p.first, p.second)); + string esc_str = get_escaped_string(string(p.first, p.second), true); AddBytesRaw(esc_str.c_str(), esc_str.size()); } s = p.first + p.second; diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 9ccd0f27f8..d2c1d91370 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -82,19 +82,19 @@ bool LogWriterAscii::DoInit(string path, int num_fields, { string str = string(header_prefix, header_prefix_len) + "separator " // Always use space as separator here. - + get_escaped_string(string(separator, separator_len)) + + get_escaped_string(string(separator, separator_len), false) + "\n"; if( fwrite(str.c_str(), str.length(), 1, file) != 1 ) goto write_error; if ( ! (WriteHeaderField("set_separator", get_escaped_string( - string(set_separator, set_separator_len))) && + string(set_separator, set_separator_len), false)) && WriteHeaderField("empty_field", get_escaped_string( - string(empty_field, empty_field_len))) && + string(empty_field, empty_field_len), false)) && WriteHeaderField("unset_field", get_escaped_string( - string(unset_field, unset_field_len))) && - WriteHeaderField("path", path)) ) + string(unset_field, unset_field_len), false)) && + WriteHeaderField("path", get_escaped_string(path, false))) ) goto write_error; string names; diff --git a/src/util.cc b/src/util.cc index 01632a5a97..171756fc1c 100644 --- a/src/util.cc +++ b/src/util.cc @@ -42,22 +42,32 @@ #include "Reporter.h" /** - * Takes a string, escapes each character into its equivalent hex code (\x##), and + * Takes a string, escapes characters into equivalent hex codes (\x##), and * returns a string containing all escaped values. * * @param str string to escape - * @return A std::string containing a list of escaped hex values of the form \x## - */ -std::string get_escaped_string(const std::string& str) + * @param escape_all If true, all characters are escaped. If false, only + * characters are escaped that are either whitespace or not printable in + * ASCII. + * @return A std::string containing a list of escaped hex values of the form + * \x## */ +std::string get_escaped_string(const std::string& str, bool escape_all) { char tbuf[16]; string esc = ""; for ( size_t i = 0; i < str.length(); ++i ) { - snprintf(tbuf, sizeof(tbuf), "\\x%02x", str[i]); - esc += tbuf; - } + char c = str[i]; + + if ( escape_all || isspace(c) || ! isascii(c) || ! isprint(c) ) + { + snprintf(tbuf, sizeof(tbuf), "\\x%02x", str[i]); + esc += tbuf; + } + else + esc += c; + } return esc; } diff --git a/src/util.h b/src/util.h index 83986b59c3..498bdf00e4 100644 --- a/src/util.h +++ b/src/util.h @@ -89,7 +89,7 @@ void delete_each(T* t) delete *it; } -std::string get_escaped_string(const std::string& str); +std::string get_escaped_string(const std::string& str, bool escape_all); extern char* copy_string(const char* s); extern int streq(const char* s1, const char* s2); From c81477d9d3185930d9d2f42fb798d3c95d36ce73 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 19 Dec 2011 08:49:30 -0800 Subject: [PATCH 206/964] Executive decision: empty fields are now logged as "(empty)" by default. --- scripts/base/frameworks/logging/writers/ascii.bro | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/base/frameworks/logging/writers/ascii.bro b/scripts/base/frameworks/logging/writers/ascii.bro index 5c04fdd3d9..c285512dd5 100644 --- a/scripts/base/frameworks/logging/writers/ascii.bro +++ b/scripts/base/frameworks/logging/writers/ascii.bro @@ -19,8 +19,9 @@ export { ## Separator between set elements. const set_separator = "," &redef; - ## String to use for empty fields. - const empty_field = "-" &redef; + ## String to use for empty fields. This should be different from + ## *unset_field* to make the output non-ambigious. + const empty_field = "(empty)" &redef; ## String to use for an unset &optional field. const unset_field = "-" &redef; From 3ac4ff6b4245af888fefc144b313e63d66b47658 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 19 Dec 2011 09:09:32 -0800 Subject: [PATCH 207/964] Updates for log format changes. --- .../Baseline/core.expr-exception/reporter.log | 24 ++++---- .../core.print-bpf-filters-ipv4/conn.log | 6 +- .../core.print-bpf-filters-ipv4/output | 32 +++++----- .../core.reporter-error-in-handler/output | 2 +- .../Baseline/core.reporter-fmt-strings/output | 2 +- .../Baseline/core.reporter-parse-error/output | 2 +- .../core.reporter-runtime-error/output | 2 +- .../core.reporter-type-mismatch/output | 6 +- .../Baseline/core.reporter/logger-test.log | 12 ++-- testing/btest/Baseline/core.reporter/output | 6 +- .../btest/Baseline/core.vlan-mpls/conn.log | 6 +- .../canonified_loaded_scripts.log | 6 +- .../canonified_loaded_scripts.log | 6 +- .../btest/Baseline/istate.broccoli/bro.log | 6 +- .../istate.events-ssl/receiver.http.log | 8 +-- .../istate.events-ssl/sender.http.log | 8 +-- .../Baseline/istate.events/receiver.http.log | 8 +-- .../Baseline/istate.events/sender.http.log | 8 +-- .../language.wrong-delete-field/output | 2 +- .../send.log | 30 +++++----- .../ssh-new-default.log | 10 ++-- .../ssh.log | 8 +-- .../ssh.log | 18 +++--- .../http.log | 8 +-- .../test.log | 6 +- .../ssh.log | 18 +++--- .../ssh.log | 10 ++-- .../test.log | 6 +- .../ssh.log | 6 +- .../ssh.log | 6 +- .../ssh.log | 16 ++--- .../output | 4 +- .../ssh.log | 6 +- .../ssh.log | 8 +-- .../ssh.log | 16 ++--- .../local.log | 6 +- .../remote.log | 6 +- .../output | 56 ++++++++--------- .../test.failure.log | 8 +-- .../test.success.log | 8 +-- .../receiver.test.log | 8 +-- .../sender.test.failure.log | 12 ++-- .../sender.test.log | 16 ++--- .../sender.test.success.log | 10 ++-- .../ssh.failure.log | 10 ++-- .../ssh.log | 12 ++-- .../out | 6 +- .../out | 60 +++++++++---------- .../output | 16 ++--- .../ssh.log | 16 ++--- .../ssh.log | 8 +-- .../testing.log | 6 +- .../ssh.log | 6 +- .../manager-1.metrics.log | 12 ++-- .../metrics.log | 12 ++-- .../manager-1.notice.log | 8 +-- .../notice.log | 10 ++-- .../manager-1.notice.log | 8 +-- .../manager-1.notice.log | 8 +-- .../notice.log | 8 +-- .../http.log | 8 +-- .../http.log | 8 +-- .../http.log | 16 ++--- .../http.log | 16 ++--- .../scripts.base.protocols.irc.basic/irc.log | 14 ++--- .../irc.log | 14 ++--- .../smtp.log | 6 +- .../smtp_entities.log | 12 ++-- .../smtp_entities.log | 12 ++-- .../knownhosts-all.log | 6 +- .../knownhosts-local.log | 6 +- .../knownhosts-remote.log | 6 +- .../knownservices-all.log | 6 +- .../knownservices-local.log | 6 +- .../knownservices-remote.log | 6 +- .../dns.log | 6 +- 76 files changed, 406 insertions(+), 406 deletions(-) diff --git a/testing/btest/Baseline/core.expr-exception/reporter.log b/testing/btest/Baseline/core.expr-exception/reporter.log index b2a412d1d3..3767de37d8 100644 --- a/testing/btest/Baseline/core.expr-exception/reporter.log +++ b/testing/btest/Baseline/core.expr-exception/reporter.log @@ -1,16 +1,16 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path reporter #fields ts level message location #types time enum string string -1300475168.783842 Reporter::ERROR field value missing [c$ftp] /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.915940 Reporter::ERROR field value missing [c$ftp] /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.916118 Reporter::ERROR field value missing [c$ftp] /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.918295 Reporter::ERROR field value missing [c$ftp] /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.952193 Reporter::ERROR field value missing [c$ftp] /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.952228 Reporter::ERROR field value missing [c$ftp] /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.954761 Reporter::ERROR field value missing [c$ftp] /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475168.962628 Reporter::ERROR field value missing [c$ftp] /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 -1300475169.780331 Reporter::ERROR field value missing [c$ftp] /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.783842 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.915940 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.916118 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.918295 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.952193 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.952228 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.954761 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475168.962628 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 +1300475169.780331 Reporter::ERROR field value missing [c$ftp] /Users/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8 diff --git a/testing/btest/Baseline/core.print-bpf-filters-ipv4/conn.log b/testing/btest/Baseline/core.print-bpf-filters-ipv4/conn.log index 915c356a1f..5ce968d5e6 100644 --- a/testing/btest/Baseline/core.print-bpf-filters-ipv4/conn.log +++ b/testing/btest/Baseline/core.print-bpf-filters-ipv4/conn.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path conn #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes #types time string addr port addr port enum string interval count count string bool count string count count count count diff --git a/testing/btest/Baseline/core.print-bpf-filters-ipv4/output b/testing/btest/Baseline/core.print-bpf-filters-ipv4/output index bfcf8318c9..d7ff523927 100644 --- a/testing/btest/Baseline/core.print-bpf-filters-ipv4/output +++ b/testing/btest/Baseline/core.print-bpf-filters-ipv4/output @@ -1,32 +1,32 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path packet_filter #fields ts node filter init success #types time string string bool bool -1323275491.966719 - not ip6 T T +1324314285.981347 - not ip6 T T #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path packet_filter #fields ts node filter init success #types time string string bool bool -1323275492.165829 - (((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)) and (not ip6) T T +1324314286.168294 - (((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)) and (not ip6) T T #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path packet_filter #fields ts node filter init success #types time string string bool bool -1323275492.362403 - port 42 T T +1324314286.350780 - port 42 T T #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path packet_filter #fields ts node filter init success #types time string string bool bool -1323275492.563649 - port 56730 T T +1324314286.530768 - port 56730 T T diff --git a/testing/btest/Baseline/core.reporter-error-in-handler/output b/testing/btest/Baseline/core.reporter-error-in-handler/output index bfb2880ed4..3d8aa6ff54 100644 --- a/testing/btest/Baseline/core.reporter-error-in-handler/output +++ b/testing/btest/Baseline/core.reporter-error-in-handler/output @@ -1,2 +1,2 @@ -error in /da/home/robin/bro/seth/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2]) +error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2]) 1st error printed on script level diff --git a/testing/btest/Baseline/core.reporter-fmt-strings/output b/testing/btest/Baseline/core.reporter-fmt-strings/output index 10a883cb5d..4842dd9fc5 100644 --- a/testing/btest/Baseline/core.reporter-fmt-strings/output +++ b/testing/btest/Baseline/core.reporter-fmt-strings/output @@ -1 +1 @@ -error in /Users/jsiwek/tmp/bro/testing/btest/.tmp/core.reporter-fmt-strings/reporter-fmt-strings.bro, line 9: not an event (dont_interpret_this(%s)) +error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-fmt-strings/reporter-fmt-strings.bro, line 9: not an event (dont_interpret_this(%s)) diff --git a/testing/btest/Baseline/core.reporter-parse-error/output b/testing/btest/Baseline/core.reporter-parse-error/output index ca0bc9304b..7606fe5667 100644 --- a/testing/btest/Baseline/core.reporter-parse-error/output +++ b/testing/btest/Baseline/core.reporter-parse-error/output @@ -1 +1 @@ -error in /da/home/robin/bro/seth/testing/btest/.tmp/core.reporter-parse-error/reporter-parse-error.bro, line 7: unknown identifier TESTFAILURE, at or near "TESTFAILURE" +error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-parse-error/reporter-parse-error.bro, line 7: unknown identifier TESTFAILURE, at or near "TESTFAILURE" diff --git a/testing/btest/Baseline/core.reporter-runtime-error/output b/testing/btest/Baseline/core.reporter-runtime-error/output index 5c0feedf42..3a96954101 100644 --- a/testing/btest/Baseline/core.reporter-runtime-error/output +++ b/testing/btest/Baseline/core.reporter-runtime-error/output @@ -1 +1 @@ -error in /Users/seth/bro.git9/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1]) +error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1]) diff --git a/testing/btest/Baseline/core.reporter-type-mismatch/output b/testing/btest/Baseline/core.reporter-type-mismatch/output index 6211752225..4c038ea8c5 100644 --- a/testing/btest/Baseline/core.reporter-type-mismatch/output +++ b/testing/btest/Baseline/core.reporter-type-mismatch/output @@ -1,3 +1,3 @@ -error in string and /da/home/robin/bro/seth/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: arithmetic mixed with non-arithmetic (string and 42) -error in /da/home/robin/bro/seth/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11 and string: type mismatch (42 and string) -error in /da/home/robin/bro/seth/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: argument type mismatch in event invocation (foo(42)) +error in string and /Users/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: arithmetic mixed with non-arithmetic (string and 42) +error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11 and string: type mismatch (42 and string) +error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: argument type mismatch in event invocation (foo(42)) diff --git a/testing/btest/Baseline/core.reporter/logger-test.log b/testing/btest/Baseline/core.reporter/logger-test.log index 6f7ba1d8c7..bc2abd142a 100644 --- a/testing/btest/Baseline/core.reporter/logger-test.log +++ b/testing/btest/Baseline/core.reporter/logger-test.log @@ -1,6 +1,6 @@ -reporter_info|init test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000 -reporter_warning|init test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000 -reporter_error|init test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000 -reporter_info|done test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000 -reporter_warning|done test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000 -reporter_error|done test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000 +reporter_info|init test-info|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000 +reporter_warning|init test-warning|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000 +reporter_error|init test-error|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000 +reporter_info|done test-info|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000 +reporter_warning|done test-warning|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000 +reporter_error|done test-error|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000 diff --git a/testing/btest/Baseline/core.reporter/output b/testing/btest/Baseline/core.reporter/output index 2735adc931..185cabb1eb 100644 --- a/testing/btest/Baseline/core.reporter/output +++ b/testing/btest/Baseline/core.reporter/output @@ -1,3 +1,3 @@ -/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info -warning in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning -error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error +/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info +warning in /Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning +error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error diff --git a/testing/btest/Baseline/core.vlan-mpls/conn.log b/testing/btest/Baseline/core.vlan-mpls/conn.log index 48be03014c..f3c958ea99 100644 --- a/testing/btest/Baseline/core.vlan-mpls/conn.log +++ b/testing/btest/Baseline/core.vlan-mpls/conn.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path conn #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes #types time string addr port addr port enum string interval count count string bool count string count count count count diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 49f3d79365..8fab67304e 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path loaded_scripts #fields name #types string diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 6587b1ac97..3f77797df8 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path loaded_scripts #fields name #types string diff --git a/testing/btest/Baseline/istate.broccoli/bro.log b/testing/btest/Baseline/istate.broccoli/bro.log index eeebe944ef..4fbbfc81ae 100644 --- a/testing/btest/Baseline/istate.broccoli/bro.log +++ b/testing/btest/Baseline/istate.broccoli/bro.log @@ -1,3 +1,3 @@ -ping received, seq 0, 1303093042.542125 at src, 1303093042.583423 at dest, -ping received, seq 1, 1303093043.543167 at src, 1303093043.544026 at dest, -ping received, seq 2, 1303093044.544115 at src, 1303093044.545008 at dest, +ping received, seq 0, 1324314397.698781 at src, 1324314397.699240 at dest, +ping received, seq 1, 1324314398.698905 at src, 1324314398.699094 at dest, +ping received, seq 2, 1324314399.699012 at src, 1324314399.699231 at dest, diff --git a/testing/btest/Baseline/istate.events-ssl/receiver.http.log b/testing/btest/Baseline/istate.events-ssl/receiver.http.log index d4b71d5d53..1601f8ad3c 100644 --- a/testing/btest/Baseline/istate.events-ssl/receiver.http.log +++ b/testing/btest/Baseline/istate.events-ssl/receiver.http.log @@ -1,8 +1,8 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1324308802.436269 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - +1324314406.995958 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/istate.events-ssl/sender.http.log b/testing/btest/Baseline/istate.events-ssl/sender.http.log index d4b71d5d53..1601f8ad3c 100644 --- a/testing/btest/Baseline/istate.events-ssl/sender.http.log +++ b/testing/btest/Baseline/istate.events-ssl/sender.http.log @@ -1,8 +1,8 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1324308802.436269 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - +1324314406.995958 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/istate.events/receiver.http.log b/testing/btest/Baseline/istate.events/receiver.http.log index efb76a9af1..25a7f289c0 100644 --- a/testing/btest/Baseline/istate.events/receiver.http.log +++ b/testing/btest/Baseline/istate.events/receiver.http.log @@ -1,8 +1,8 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1324308826.107003 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - +1324314415.616486 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/istate.events/sender.http.log b/testing/btest/Baseline/istate.events/sender.http.log index efb76a9af1..25a7f289c0 100644 --- a/testing/btest/Baseline/istate.events/sender.http.log +++ b/testing/btest/Baseline/istate.events/sender.http.log @@ -1,8 +1,8 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1324308826.107003 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - - +1324314415.616486 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/language.wrong-delete-field/output b/testing/btest/Baseline/language.wrong-delete-field/output index f8271e43c2..c2aae8aae3 100644 --- a/testing/btest/Baseline/language.wrong-delete-field/output +++ b/testing/btest/Baseline/language.wrong-delete-field/output @@ -1 +1 @@ -error in /da/home/robin/bro/seth/testing/btest/.tmp/language.wrong-delete-field/wrong-delete-field.bro, line 10: illegal delete statement (delete x$a) +error in /Users/robin/bro/master/testing/btest/.tmp/language.wrong-delete-field/wrong-delete-field.bro, line 10: illegal delete statement (delete x$a) diff --git a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log index 665d21c153..e5dfb59592 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log +++ b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log @@ -1,19 +1,19 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path communication #fields ts peer src_name connected_peer_desc connected_peer_addr connected_peer_port level message #types time string string string addr port string string -1323275566.293849 bro parent - - - info [#1/127.0.0.1:47757] added peer -1323275566.300180 bro child - - - info [#1/127.0.0.1:47757] connected -1323275566.300467 bro parent - - - info [#1/127.0.0.1:47757] peer connected -1323275566.300467 bro parent - - - info [#1/127.0.0.1:47757] phase: version -1323275566.300936 bro script - - - info connection established -1323275566.300936 bro script - - - info requesting events matching /^?(NOTHING)$?/ -1323275566.300936 bro script - - - info accepting state -1323275566.302043 bro parent - - - info [#1/127.0.0.1:47757] phase: handshake -1323275566.302043 bro parent - - - info warning: no events to request -1323275566.302043 bro parent - - - info terminating... -1323275566.302043 bro parent - - - info [#1/127.0.0.1:47757] peer_description is bro -1323275566.302043 bro parent - - - info [#1/127.0.0.1:47757] closing connection +1324314302.411344 bro parent - - - info [#1/127.0.0.1:47757] added peer +1324314302.414978 bro child - - - info [#1/127.0.0.1:47757] connected +1324314302.415099 bro parent - - - info [#1/127.0.0.1:47757] peer connected +1324314302.415099 bro parent - - - info [#1/127.0.0.1:47757] phase: version +1324314302.417446 bro script - - - info connection established +1324314302.417446 bro script - - - info requesting events matching /^?(NOTHING)$?/ +1324314302.417446 bro script - - - info accepting state +1324314302.418003 bro parent - - - info [#1/127.0.0.1:47757] phase: handshake +1324314302.418003 bro parent - - - info warning: no events to request +1324314302.418003 bro parent - - - info terminating... +1324314302.418003 bro parent - - - info [#1/127.0.0.1:47757] peer_description is bro +1324314302.418003 bro parent - - - info [#1/127.0.0.1:47757] closing connection diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log b/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log index 76edd50f26..485bfe3eba 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log @@ -1,9 +1,9 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path ssh-new-default #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1323275589.577486 1.2.3.4 1234 2.3.4.5 80 success unknown -1323275589.577486 1.2.3.4 1234 2.3.4.5 80 failure US +1324314313.140603 1.2.3.4 1234 2.3.4.5 80 success unknown +1324314313.140603 1.2.3.4 1234 2.3.4.5 80 failure US diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log index f0006dbe37..144a7a6426 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log @@ -1,7 +1,7 @@ -#separator \x7c -#set_separator|\x2c -#empty_field|\x2d -#unset_field|\x2d +#separator | +#set_separator|, +#empty_field|(empty) +#unset_field|- #path|ssh #fields|data|data2 #types|string|string diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log index b11fd30678..10275205a5 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log @@ -1,12 +1,12 @@ -PREFIX<>separator \x7c -PREFIX<>set_separator|\x2c -PREFIX<>empty_field|\x45\x4d\x50\x54\x59 -PREFIX<>unset_field|\x4e\x4f\x54\x2d\x53\x45\x54 +PREFIX<>separator | +PREFIX<>set_separator|, +PREFIX<>empty_field|EMPTY +PREFIX<>unset_field|NOT-SET PREFIX<>path|ssh PREFIX<>fields|t|id.orig_h|id.orig_p|id.resp_h|id.resp_p|status|country|b PREFIX<>types|time|addr|port|addr|port|string|string|bool -1323275635.348361|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET -1323275635.348361|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET -1323275635.348361|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET -1323275635.348361|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET -1323275635.348361|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T +1324314313.345323|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET +1324314313.345323|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET +1324314313.345323|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET +1324314313.345323|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET +1324314313.345323|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log index 2b5f8d415d..97744b7df8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log @@ -1,8 +1,8 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1315799856.264750 UWkUyAuUGXf 10.0.1.104 64216 193.40.5.162 80 1 GET lepo.it.da.ut.ee /~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf - Wget/1.12 (darwin10.8.0) 0 346 404 Not Found - - - - - - - text/html - - +1315799856.264750 UWkUyAuUGXf 10.0.1.104 64216 193.40.5.162 80 1 GET lepo.it.da.ut.ee /~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf - Wget/1.12 (darwin10.8.0) 0 346 404 Not Found - - - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log index caf19240e0..b88627c806 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields ss #types table[string] diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log index 84e2991a47..0ef81128d3 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log @@ -1,12 +1,12 @@ -#separator \x7c\x7c -#set_separator||\x2c -#empty_field||\x2d -#unset_field||\x2d +#separator || +#set_separator||, +#empty_field||(empty) +#unset_field||- #path||ssh #fields||t||id.orig_h||id.orig_p||id.resp_h||id.resp_p||status||country #types||time||addr||port||addr||port||string||string -1323275761.036351||1.2.3.4||1234||2.3.4.5||80||success||unknown -1323275761.036351||1.2.3.4||1234||2.3.4.5||80||failure||US -1323275761.036351||1.2.3.4||1234||2.3.4.5||80||fa\x7c\x7cure||UK -1323275761.036351||1.2.3.4||1234||2.3.4.5||80||su\x7c\x7cess||BR -1323275761.036351||1.2.3.4||1234||2.3.4.5||80||failure||MX +1324314313.899736||1.2.3.4||1234||2.3.4.5||80||success||unknown +1324314313.899736||1.2.3.4||1234||2.3.4.5||80||failure||US +1324314313.899736||1.2.3.4||1234||2.3.4.5||80||fa\x7c\x7cure||UK +1324314313.899736||1.2.3.4||1234||2.3.4.5||80||su\x7c\x7cess||BR +1324314313.899736||1.2.3.4||1234||2.3.4.5||80||failure||MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-options/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-options/ssh.log index 33a922cc2b..f66dec7160 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-options/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-options/ssh.log @@ -1,5 +1,5 @@ -1299718506.38074|1.2.3.4|1234|2.3.4.5|80|success|unknown -1299718506.38074|1.2.3.4|1234|2.3.4.5|80|failure|US -1299718506.38074|1.2.3.4|1234|2.3.4.5|80|failure|UK -1299718506.38074|1.2.3.4|1234|2.3.4.5|80|success|BR -1299718506.38074|1.2.3.4|1234|2.3.4.5|80|failure|MX +1324314313.990741|1.2.3.4|1234|2.3.4.5|80|success|unknown +1324314313.990741|1.2.3.4|1234|2.3.4.5|80|failure|US +1324314313.990741|1.2.3.4|1234|2.3.4.5|80|failure|UK +1324314313.990741|1.2.3.4|1234|2.3.4.5|80|success|BR +1324314313.990741|1.2.3.4|1234|2.3.4.5|80|failure|MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log index 7a05376b43..00ab6c8ca0 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields data #types time diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log index e79320b415..5acaa7b2fc 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path ssh #fields status country a1 b1 b2 #types string string count count count diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log index 73266386c8..086a4836fe 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path ssh #fields status country #types string string diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log index 1e25a5b664..16ba17c62c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log @@ -1,12 +1,12 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path ssh #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1323275824.696040 1.2.3.4 1234 2.3.4.5 80 success unknown -1323275824.696040 1.2.3.4 1234 2.3.4.5 80 failure US -1323275824.696040 1.2.3.4 1234 2.3.4.5 80 failure UK -1323275824.696040 1.2.3.4 1234 2.3.4.5 80 success BR -1323275824.696040 1.2.3.4 1234 2.3.4.5 80 failure MX +1324314314.443785 1.2.3.4 1234 2.3.4.5 80 success unknown +1324314314.443785 1.2.3.4 1234 2.3.4.5 80 failure US +1324314314.443785 1.2.3.4 1234 2.3.4.5 80 failure UK +1324314314.443785 1.2.3.4 1234 2.3.4.5 80 success BR +1324314314.443785 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.events/output b/testing/btest/Baseline/scripts.base.frameworks.logging.events/output index 297b3dabd2..5da27764a5 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.events/output +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.events/output @@ -1,2 +1,2 @@ -[t=1323970492.986366, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=success, country=unknown] -[t=1323970492.986366, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=failure, country=US] +[t=1324314314.738385, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=success, country=unknown] +[t=1324314314.738385, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=failure, country=US] diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log index b70046bd15..4ccf4c836a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path ssh #fields id.orig_p id.resp_h id.resp_p status country #types port addr port string string diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log index 47d0f93f2e..4aa3d8f0a7 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log @@ -1,8 +1,8 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path ssh #fields t f #types time file -1323275842.508479 Foo.log +1324314314.940195 Foo.log diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log index e1b1d18ddc..00242d65c1 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log @@ -1,12 +1,12 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path ssh #fields t id.orig_h #types time addr -1323275846.507507 1.2.3.4 -1323275846.507507 1.2.3.4 -1323275846.507507 1.2.3.4 -1323275846.507507 1.2.3.4 -1323275846.507507 1.2.3.4 +1324314315.040480 1.2.3.4 +1324314315.040480 1.2.3.4 +1324314315.040480 1.2.3.4 +1324314315.040480 1.2.3.4 +1324314315.040480 1.2.3.4 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log index 1cc0c681fb..e2b3da6efd 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path local #fields ts id.orig_h #types time addr diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log index 20e096f399..1ac18ff5f7 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path remote #fields ts id.orig_h #types time addr diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output index d62ae42069..a6b8a4e090 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output @@ -6,58 +6,58 @@ static-prefix-1-US.log static-prefix-2-MX2.log static-prefix-2-UK.log #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path static-prefix-0-BR #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1323275860.153895 1.2.3.4 1234 2.3.4.5 80 success BR +1324314315.385189 1.2.3.4 1234 2.3.4.5 80 success BR #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path static-prefix-0-MX3 #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1323275860.153895 1.2.3.4 1234 2.3.4.5 80 failure MX3 +1324314315.385189 1.2.3.4 1234 2.3.4.5 80 failure MX3 #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path static-prefix-0-unknown #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1323275860.153895 1.2.3.4 1234 2.3.4.5 80 success unknown +1324314315.385189 1.2.3.4 1234 2.3.4.5 80 success unknown #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path static-prefix-1-MX #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1323275860.153895 1.2.3.4 1234 2.3.4.5 80 failure MX +1324314315.385189 1.2.3.4 1234 2.3.4.5 80 failure MX #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path static-prefix-1-US #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1323275860.153895 1.2.3.4 1234 2.3.4.5 80 failure US +1324314315.385189 1.2.3.4 1234 2.3.4.5 80 failure US #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path static-prefix-2-MX2 #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1323275860.153895 1.2.3.4 1234 2.3.4.5 80 failure MX2 +1324314315.385189 1.2.3.4 1234 2.3.4.5 80 failure MX2 #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path static-prefix-2-UK #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1323275860.153895 1.2.3.4 1234 2.3.4.5 80 failure UK +1324314315.385189 1.2.3.4 1234 2.3.4.5 80 failure UK diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log index f6a0779ff3..733bb02847 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log @@ -1,8 +1,8 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path test.failure #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1323275882.725518 1.2.3.4 1234 2.3.4.5 80 failure US +1324314315.498365 1.2.3.4 1234 2.3.4.5 80 failure US diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log index 18f73a3d6c..0261caeb06 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log @@ -1,8 +1,8 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path test.success #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1324308566.444800 1.2.3.4 1234 2.3.4.5 80 success unknown +1324314315.498365 1.2.3.4 1234 2.3.4.5 80 success unknown diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log index 938bf9ab1a..d9bd34309a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log @@ -1,8 +1,8 @@ #separator \x09 -#set_separator \x2c -#empty_field \x45\x4d\x50\x54\x59 -#unset_field \x2d +#set_separator , +#empty_field EMPTY +#unset_field - #path test #fields b i e c p sn a d t iv s sc ss se vc ve #types bool int enum count port subnet addr double time interval string table[count] table[string] table[string] vector[count] vector[string] -T -42 Test::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1324308572.066737 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY +T -42 Test::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1324314315.880694 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log index f6fdc8d4a7..6cb58bf4ac 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log @@ -1,10 +1,10 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path test.failure #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1323276013.684540 1.2.3.4 1234 2.3.4.5 80 failure US -1323276013.684540 1.2.3.4 1234 2.3.4.5 80 failure UK -1323276013.684540 1.2.3.4 1234 2.3.4.5 80 failure MX +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 failure US +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 failure UK +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log index 8c4d930c4e..f5b79ee2c4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log @@ -1,12 +1,12 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1324308589.020941 1.2.3.4 1234 2.3.4.5 80 success unknown -1324308589.020941 1.2.3.4 1234 2.3.4.5 80 failure US -1324308589.020941 1.2.3.4 1234 2.3.4.5 80 failure UK -1324308589.020941 1.2.3.4 1234 2.3.4.5 80 success BR -1324308589.020941 1.2.3.4 1234 2.3.4.5 80 failure MX +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 success unknown +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 failure US +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 failure UK +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 success BR +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log index ac0ce01ee0..c40e56af93 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log @@ -1,9 +1,9 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path test.success #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1324308589.020941 1.2.3.4 1234 2.3.4.5 80 success unknown -1324308589.020941 1.2.3.4 1234 2.3.4.5 80 success BR +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 success unknown +1324314321.061516 1.2.3.4 1234 2.3.4.5 80 success BR diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log index 9256f309ce..cb3d4aafb8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log @@ -1,9 +1,9 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path ssh.failure #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1323276050.103643 1.2.3.4 1234 2.3.4.5 80 failure US -1323276050.103643 1.2.3.4 1234 2.3.4.5 80 failure UK +1324314328.196443 1.2.3.4 1234 2.3.4.5 80 failure US +1324314328.196443 1.2.3.4 1234 2.3.4.5 80 failure UK diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log index 2e70ef44f3..38a5bb660c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log @@ -1,10 +1,10 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path ssh #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1323276050.103643 1.2.3.4 1234 2.3.4.5 80 failure US -1323276050.103643 1.2.3.4 1234 2.3.4.5 80 failure UK -1323276050.103643 1.2.3.4 1234 2.3.4.5 80 failure BR +1324314328.196443 1.2.3.4 1234 2.3.4.5 80 failure US +1324314328.196443 1.2.3.4 1234 2.3.4.5 80 failure UK +1324314328.196443 1.2.3.4 1234 2.3.4.5 80 failure BR diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out index 71d0413464..915915f43e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out @@ -18,14 +18,14 @@ custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_11.00.05.log, pat custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_11.59.55.log, path=test2, open=1299499195.0, close=1299499205.0, terminating=F] custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_12.00.05.log, path=test2, open=1299499205.0, close=1299502795.0, terminating=F] custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_12.59.55.log, path=test2, open=1299502795.0, close=1299502795.0, terminating=T] -#empty_field \x2d +#empty_field (empty) #fields t id.orig_h id.orig_p id.resp_h id.resp_p #path test #path test2 #separator \x09 -#set_separator \x2c +#set_separator , #types time addr port addr port -#unset_field \x2d +#unset_field - 1299466805.000000 10.0.0.1 20 10.0.0.2 1024 1299470395.000000 10.0.0.2 20 10.0.0.3 0 1299470405.000000 10.0.0.1 20 10.0.0.2 1025 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out index a5f9ae758b..d31783edc4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out @@ -10,9 +10,9 @@ test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0 test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 > test.2011-03-07-03-00-05.log #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -20,9 +20,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299470395.000000 10.0.0.2 20 10.0.0.3 0 > test.2011-03-07-04-00-05.log #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -30,9 +30,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299473995.000000 10.0.0.2 20 10.0.0.3 1 > test.2011-03-07-05-00-05.log #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -40,9 +40,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299477595.000000 10.0.0.2 20 10.0.0.3 2 > test.2011-03-07-06-00-05.log #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -50,9 +50,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299481195.000000 10.0.0.2 20 10.0.0.3 3 > test.2011-03-07-07-00-05.log #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -60,9 +60,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299484795.000000 10.0.0.2 20 10.0.0.3 4 > test.2011-03-07-08-00-05.log #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -70,9 +70,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299488395.000000 10.0.0.2 20 10.0.0.3 5 > test.2011-03-07-09-00-05.log #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -80,9 +80,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299491995.000000 10.0.0.2 20 10.0.0.3 6 > test.2011-03-07-10-00-05.log #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -90,9 +90,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299495595.000000 10.0.0.2 20 10.0.0.3 7 > test.2011-03-07-11-00-05.log #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port @@ -100,9 +100,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 1299499195.000000 10.0.0.2 20 10.0.0.3 8 > test.2011-03-07-12-00-05.log #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields t id.orig_h id.orig_p id.resp_h id.resp_p #types time addr port addr port diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output b/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output index 65c24b8752..09afe2031c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output @@ -1,12 +1,12 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path /dev/stdout #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1323276116.980214 1.2.3.4 1234 2.3.4.5 80 success unknown -1323276116.980214 1.2.3.4 1234 2.3.4.5 80 failure US -1323276116.980214 1.2.3.4 1234 2.3.4.5 80 failure UK -1323276116.980214 1.2.3.4 1234 2.3.4.5 80 success BR -1323276116.980214 1.2.3.4 1234 2.3.4.5 80 failure MX +1324314328.844271 1.2.3.4 1234 2.3.4.5 80 success unknown +1324314328.844271 1.2.3.4 1234 2.3.4.5 80 failure US +1324314328.844271 1.2.3.4 1234 2.3.4.5 80 failure UK +1324314328.844271 1.2.3.4 1234 2.3.4.5 80 success BR +1324314328.844271 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log index b283ad8856..53292324af 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log @@ -1,12 +1,12 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path ssh #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1323276164.251500 1.2.3.4 1234 2.3.4.5 80 success unknown -1323276164.251500 1.2.3.4 1234 2.3.4.5 80 failure US -1323276164.251500 1.2.3.4 1234 2.3.4.5 80 failure UK -1323276164.251500 1.2.3.4 1234 2.3.4.5 80 success BR -1323276164.251500 1.2.3.4 1234 2.3.4.5 80 failure MX +1324314328.950525 1.2.3.4 1234 2.3.4.5 80 success unknown +1324314328.950525 1.2.3.4 1234 2.3.4.5 80 failure US +1324314328.950525 1.2.3.4 1234 2.3.4.5 80 failure UK +1324314328.950525 1.2.3.4 1234 2.3.4.5 80 success BR +1324314328.950525 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log index 7d80a93fd5..74aa0312a1 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log @@ -1,8 +1,8 @@ #separator \x09 -#set_separator \x2c -#empty_field \x45\x4d\x50\x54\x59 -#unset_field \x2d +#set_separator , +#empty_field EMPTY +#unset_field - #path ssh #fields b i e c p sn a d t iv s sc ss se vc ve f #types bool int enum count port subnet addr double time interval string table[count] table[string] table[string] vector[count] vector[string] func -T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1324308607.500960 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1324314329.051618 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log b/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log index a9acb91cd3..7956ad11a0 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path testing #fields a.val1 a.val2 b #types count count count diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log index ffbae34d4c..65ab5592bf 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path ssh #fields vec #types vector[string] diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log index f3485833d2..a278bdc56a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log @@ -1,10 +1,10 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path metrics #fields ts metric_id filter_name index.host index.str index.network value #types time enum string addr string subnet count -1323276206.622034 TEST_METRIC foo-bar 6.5.4.3 - - 4 -1323276206.622034 TEST_METRIC foo-bar 1.2.3.4 - - 6 -1323276206.622034 TEST_METRIC foo-bar 7.2.1.5 - - 2 +1324314335.570789 TEST_METRIC foo-bar 6.5.4.3 - - 4 +1324314335.570789 TEST_METRIC foo-bar 1.2.3.4 - - 6 +1324314335.570789 TEST_METRIC foo-bar 7.2.1.5 - - 2 diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log index ef8fe73611..8ee19c255b 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log @@ -1,10 +1,10 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path metrics #fields ts metric_id filter_name index.host index.str index.network value #types time enum string addr string subnet count -1323276222.644659 TEST_METRIC foo-bar 6.5.4.3 - - 2 -1323276222.644659 TEST_METRIC foo-bar 1.2.3.4 - - 3 -1323276222.644659 TEST_METRIC foo-bar 7.2.1.5 - - 1 +1324314344.807073 TEST_METRIC foo-bar 6.5.4.3 - - 2 +1324314344.807073 TEST_METRIC foo-bar 1.2.3.4 - - 3 +1324314344.807073 TEST_METRIC foo-bar 7.2.1.5 - - 1 diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log index cc95398121..33f55ce608 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log @@ -1,8 +1,8 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network #types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1324308631.319990 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - - +1324314350.184962 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log index c96078f654..437b1465a1 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log @@ -1,9 +1,9 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network #types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1324308665.314874 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 3/2 - 1.2.3.4 - - 3 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - - -1324308665.314874 - - - - - Test_Notice Threshold crossed by metric_index(host=6.5.4.3) 2/2 - 6.5.4.3 - - 2 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - 6.5.4.3 - - +1324314359.357148 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 3/2 - 1.2.3.4 - - 3 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - - +1324314359.357148 - - - - - Test_Notice Threshold crossed by metric_index(host=6.5.4.3) 2/2 - 6.5.4.3 - - 2 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - 6.5.4.3 - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log index 91ee4eb823..fb1e1b3d47 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log @@ -1,8 +1,8 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network #types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1324308679.119923 - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - +1324314363.721823 - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log index e99bb9ee65..9e6e1b1916 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log @@ -1,8 +1,8 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network #types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1324308705.683375 - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - +1324314378.560010 - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log index ce93bb617d..d134c97049 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log @@ -1,8 +1,8 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path notice #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude #types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double -1324308722.344582 - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 6 3600.000000 F - - - - - +1324314387.663586 - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 6 3600.000000 F - - - - - diff --git a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log index 3d711d430c..ddcea2e9c7 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log @@ -1,8 +1,8 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 1 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 60731 200 OK 100 Continue - - - - - text/html - - +1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 1 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 60731 200 OK 100 Continue - (empty) - - - text/html - - diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log index e063bcbaf5..cec098a50b 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log @@ -1,8 +1,8 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1128727435.634189 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - - - - - text/html - http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat +1128727435.634189 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log index c2e74fcae1..d4e5679da1 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log @@ -1,12 +1,12 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - - - - - FAKE_MIME - - -1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - - - - - FAKE_MIME - - -1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - - - - - FAKE_MIME - - -1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - - - - - image/png e0029eea80812e9a8e57b8d05d52938a - -1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - - - - - image/png 30aa926344f58019d047e85ba049ca1e - +1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - FAKE_MIME - - +1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - FAKE_MIME - - +1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - FAKE_MIME - - +1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - image/png e0029eea80812e9a8e57b8d05d52938a - +1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - image/png 30aa926344f58019d047e85ba049ca1e - diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log index 0246cf450a..dfaf34acbf 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log @@ -1,12 +1,12 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path http #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied md5 extraction_file #types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string file -1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - - - - - - - -1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - - - - - - - -1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - - - - - - - -1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - - - - - - - -1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - - - - - - - +1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - - - +1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - - - +1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - - - +1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - - - +1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - - - diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log index 6a200a9e41..39ff897fae 100644 --- a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log +++ b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log @@ -1,11 +1,11 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path irc #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user channels command value addl tags dcc_file_name dcc_file_size extraction_file #types time string addr port addr port string string table[string] string string string table[enum] string count file -1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - - NICK bloed - - - - - -1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - - -1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - JOIN #easymovies - - - - - -1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - DCC #easymovies - - ladyvampress-default(2011-07-07)-OS.zip 42208 - +1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - - NICK bloed - (empty) - - - +1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq (empty) - - - +1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - JOIN #easymovies (empty) (empty) - - - +1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - DCC #easymovies (empty) (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 - diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log index f3d0b2ff32..342923ba7b 100644 --- a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log +++ b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log @@ -1,11 +1,11 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path irc #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user channels command value addl tags dcc_file_name dcc_file_size dcc_mime_type extraction_file #types time string addr port addr port string string table[string] string string string table[enum] string count string file -1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - - NICK bloed - - - - - - -1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - - - -1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - JOIN #easymovies - - - - - - -1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - DCC #easymovies - IRC::EXTRACTED_FILE ladyvampress-default(2011-07-07)-OS.zip 42208 FAKE_MIME irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat +1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - - NICK bloed - (empty) - - - - +1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq (empty) - - - - +1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - JOIN #easymovies (empty) (empty) - - - - +1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - DCC #easymovies (empty) IRC::EXTRACTED_FILE ladyvampress-default(2011-07-07)-OS.zip 42208 FAKE_MIME irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log b/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log index 4d1b8cca84..2c1380cb44 100644 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path smtp #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth helo mailfrom rcptto date from to reply_to msg_id in_reply_to subject x_originating_ip first_received second_received last_reply path user_agent #types time string addr port addr port count string string table[string] string string table[string] string string string string addr string string string vector[addr] string diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log index 13ec1b02d5..453b55932e 100644 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log @@ -1,10 +1,10 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path smtp_entities #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth filename content_len mime_type md5 extraction_file excerpt #types time string addr port addr port count string count string string file string -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 FAKE_MIME - smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat - -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 FAKE_MIME - - - -1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 FAKE_MIME - smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat - +1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 FAKE_MIME - smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat (empty) +1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 FAKE_MIME - - (empty) +1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 FAKE_MIME - smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat (empty) diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log index 26e0c2dc01..2b471782d5 100644 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log @@ -1,10 +1,10 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path smtp_entities #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth filename content_len mime_type md5 extraction_file excerpt #types time string addr port addr port count string count string string file string -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 FAKE_MIME 92bca2e6cdcde73647125da7dccbdd07 - - -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 FAKE_MIME - - - -1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 FAKE_MIME a968bb0f9f9d95835b2e74c845877e87 - - +1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 FAKE_MIME 92bca2e6cdcde73647125da7dccbdd07 - (empty) +1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 FAKE_MIME - - (empty) +1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 FAKE_MIME a968bb0f9f9d95835b2e74c845877e87 - (empty) diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log index 13646617a9..0799292857 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path known_hosts #fields ts host #types time addr diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log index 3ab2a480b0..6fdba24d39 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path known_hosts #fields ts host #types time addr diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log index cc06e288aa..9ef6ee47b7 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path known_hosts #fields ts host #types time addr diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log index 9e73167aba..d53da6f693 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path known_services #fields ts host port_num port_proto service #types time addr port enum table[string] diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log index 04928321c5..ef1722d6a1 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path known_services #fields ts host port_num port_proto service #types time addr port enum table[string] diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log index edfe160a7d..3fc68cdb91 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path known_services #fields ts host port_num port_proto service #types time addr port enum table[string] diff --git a/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log b/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log index c50278a533..9d80898e0f 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log +++ b/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log @@ -1,7 +1,7 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path dns #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id query qclass qclass_name qtype qtype_name rcode rcode_name QR AA TC RD RA Z answers TTLs auth addl #types time string addr port addr port enum count string count string count string count string bool bool bool bool bool count vector[string] vector[interval] table[string] table[string] From 578cd0617648ebafdfb7402c7cfba388690847d6 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 19 Dec 2011 13:12:02 -0600 Subject: [PATCH 208/964] Increase timeout interval of communication-related btests. This may help clear up some transient test failures on the NMI testbed. --- testing/btest/scripts/base/frameworks/cluster/start-it-up.bro | 4 ++-- .../frameworks/communication/communication_log_baseline.bro | 2 +- .../scripts/base/frameworks/control/configuration_update.bro | 4 ++-- testing/btest/scripts/base/frameworks/control/id_value.bro | 2 +- testing/btest/scripts/base/frameworks/control/shutdown.bro | 2 +- .../btest/scripts/base/frameworks/logging/remote-types.bro | 2 +- testing/btest/scripts/base/frameworks/logging/remote.bro | 2 +- .../btest/scripts/base/frameworks/metrics/basic-cluster.bro | 2 +- .../base/frameworks/metrics/cluster-intermediate-update.bro | 2 +- testing/btest/scripts/base/frameworks/notice/cluster.bro | 2 +- .../scripts/base/frameworks/notice/suppression-cluster.bro | 2 +- 11 files changed, 13 insertions(+), 13 deletions(-) diff --git a/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro b/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro index d1eb94d5e1..b8ee4c33e8 100644 --- a/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro +++ b/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro @@ -3,7 +3,7 @@ # @TEST-EXEC: btest-bg-run proxy-2 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT -# @TEST-EXEC: btest-bg-wait -k 2 +# @TEST-EXEC: btest-bg-wait -k 10 # @TEST-EXEC: btest-diff manager-1/.stdout # @TEST-EXEC: btest-diff proxy-1/.stdout # @TEST-EXEC: btest-diff proxy-2/.stdout @@ -23,4 +23,4 @@ redef Cluster::nodes = { event remote_connection_handshake_done(p: event_peer) { print "Connected to a peer"; - } \ No newline at end of file + } diff --git a/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro b/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro index c3078684af..3a4c1253eb 100644 --- a/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro +++ b/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro @@ -1,7 +1,7 @@ # # @TEST-EXEC: btest-bg-run receiver bro -b ../receiver.bro # @TEST-EXEC: btest-bg-run sender bro -b ../sender.bro -# @TEST-EXEC: btest-bg-wait -k 2 +# @TEST-EXEC: btest-bg-wait -k 10 # # Don't diff the receiver log just because port is always going to change # @TEST-EXEC: egrep -v 'pid|socket buffer size' sender/communication.log >send.log diff --git a/testing/btest/scripts/base/frameworks/control/configuration_update.bro b/testing/btest/scripts/base/frameworks/control/configuration_update.bro index eb86ec58e8..9b16faee69 100644 --- a/testing/btest/scripts/base/frameworks/control/configuration_update.bro +++ b/testing/btest/scripts/base/frameworks/control/configuration_update.bro @@ -1,7 +1,7 @@ # @TEST-EXEC: btest-bg-run controllee BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controllee Communication::listen_port=65531/tcp # @TEST-EXEC: btest-bg-run controller BROPATH=$BROPATH:.. bro %INPUT test-redef frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=configuration_update # @TEST-EXEC: btest-bg-run controller2 BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=shutdown -# @TEST-EXEC: btest-bg-wait 1 +# @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: btest-diff controllee/.stdout redef Communication::nodes = { @@ -23,4 +23,4 @@ event bro_init() event bro_done() { print test_var; - } \ No newline at end of file + } diff --git a/testing/btest/scripts/base/frameworks/control/id_value.bro b/testing/btest/scripts/base/frameworks/control/id_value.bro index 90a5367f76..e06fa46e74 100644 --- a/testing/btest/scripts/base/frameworks/control/id_value.bro +++ b/testing/btest/scripts/base/frameworks/control/id_value.bro @@ -1,6 +1,6 @@ # @TEST-EXEC: btest-bg-run controllee BROPATH=$BROPATH:.. bro %INPUT only-for-controllee frameworks/control/controllee Communication::listen_port=65532/tcp # @TEST-EXEC: btest-bg-run controller BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65532/tcp Control::cmd=id_value Control::arg=test_var -# @TEST-EXEC: btest-bg-wait -k 1 +# @TEST-EXEC: btest-bg-wait -k 10 # @TEST-EXEC: btest-diff controller/.stdout redef Communication::nodes = { diff --git a/testing/btest/scripts/base/frameworks/control/shutdown.bro b/testing/btest/scripts/base/frameworks/control/shutdown.bro index 73319a7c4a..9953a8382a 100644 --- a/testing/btest/scripts/base/frameworks/control/shutdown.bro +++ b/testing/btest/scripts/base/frameworks/control/shutdown.bro @@ -1,6 +1,6 @@ # @TEST-EXEC: btest-bg-run controllee BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controllee Communication::listen_port=65530/tcp # @TEST-EXEC: btest-bg-run controller BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65530/tcp Control::cmd=shutdown -# @TEST-EXEC: btest-bg-wait 1 +# @TEST-EXEC: btest-bg-wait 10 redef Communication::nodes = { # We're waiting for connections from this host for control. diff --git a/testing/btest/scripts/base/frameworks/logging/remote-types.bro b/testing/btest/scripts/base/frameworks/logging/remote-types.bro index 9af45cf991..4e866cc985 100644 --- a/testing/btest/scripts/base/frameworks/logging/remote-types.bro +++ b/testing/btest/scripts/base/frameworks/logging/remote-types.bro @@ -1,7 +1,7 @@ # # @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro # @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro -# @TEST-EXEC: btest-bg-wait -k 1 +# @TEST-EXEC: btest-bg-wait -k 10 # @TEST-EXEC: btest-diff receiver/test.log # @TEST-EXEC: cmp receiver/test.log sender/test.log diff --git a/testing/btest/scripts/base/frameworks/logging/remote.bro b/testing/btest/scripts/base/frameworks/logging/remote.bro index b244c72cdf..8ed3405aed 100644 --- a/testing/btest/scripts/base/frameworks/logging/remote.bro +++ b/testing/btest/scripts/base/frameworks/logging/remote.bro @@ -3,7 +3,7 @@ # @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro # @TEST-EXEC: sleep 1 -# @TEST-EXEC: btest-bg-wait -k 1 +# @TEST-EXEC: btest-bg-wait -k 10 # @TEST-EXEC: btest-diff sender/test.log # @TEST-EXEC: btest-diff sender/test.failure.log # @TEST-EXEC: btest-diff sender/test.success.log diff --git a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro index 4b7f177f15..23b87053ab 100644 --- a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro +++ b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro @@ -3,7 +3,7 @@ # @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT -# @TEST-EXEC: btest-bg-wait -k 6 +# @TEST-EXEC: btest-bg-wait -k 10 # @TEST-EXEC: btest-diff manager-1/metrics.log @TEST-START-FILE cluster-layout.bro diff --git a/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro b/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro index 89d771e05e..45d44898aa 100644 --- a/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro +++ b/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro @@ -3,7 +3,7 @@ # @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT -# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-bg-wait -k 10 # @TEST-EXEC: btest-diff manager-1/notice.log @TEST-START-FILE cluster-layout.bro diff --git a/testing/btest/scripts/base/frameworks/notice/cluster.bro b/testing/btest/scripts/base/frameworks/notice/cluster.bro index f44ba72f3a..125d021d82 100644 --- a/testing/btest/scripts/base/frameworks/notice/cluster.bro +++ b/testing/btest/scripts/base/frameworks/notice/cluster.bro @@ -2,7 +2,7 @@ # @TEST-EXEC: btest-bg-run proxy-1 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT # @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT -# @TEST-EXEC: btest-bg-wait -k 6 +# @TEST-EXEC: btest-bg-wait -k 10 # @TEST-EXEC: btest-diff manager-1/notice.log @TEST-START-FILE cluster-layout.bro diff --git a/testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro b/testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro index a7e720d5f5..e084fb74e0 100644 --- a/testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro +++ b/testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro @@ -3,7 +3,7 @@ # @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT -# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-bg-wait -k 10 # @TEST-EXEC: btest-diff manager-1/notice.log @TEST-START-FILE cluster-layout.bro From 8199d85d6a70fc51c56ae14ebc2b64ba5149361e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 19 Dec 2011 11:27:29 -0800 Subject: [PATCH 209/964] Updating submodule(s). --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- aux/btest | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aux/binpac b/aux/binpac index a29ccf131c..e94d92b01f 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit a29ccf131cc44da73b8e8c8d78f7939b34d154be +Subproject commit e94d92b01f327655fd2061157942b95ae75b5f0f diff --git a/aux/bro-aux b/aux/bro-aux index 3ee84004b3..f6b92bf573 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 3ee84004b351755d8156378093abdf01a6541c68 +Subproject commit f6b92bf5732c26e54eb4387efadc612663980389 diff --git a/aux/broccoli b/aux/broccoli index a287654121..d7b8a43759 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit a287654121c83cf0a0b2845fb558be07afe1391a +Subproject commit d7b8a43759bfcbe1381d132d8ab388937e52a6d4 diff --git a/aux/broctl b/aux/broctl index c4405333fd..a42e4d133b 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit c4405333fdb684925168ecd32659395c9a6f47e9 +Subproject commit a42e4d133b94622c612055047f8534d5122e6e88 diff --git a/aux/btest b/aux/btest index 7230a09a8c..38890e8514 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 7230a09a8c220d2117e491fdf293bf5c19819b65 +Subproject commit 38890e851416fa9fc827a1d36f06c4cb9f7d4e69 From 43124d4b1c7f1963dd18eca12b1cab34db7a3489 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 19 Dec 2011 11:37:15 -0800 Subject: [PATCH 210/964] Update one more baseline. --- CHANGES | 8 ++++++++ VERSION | 2 +- .../test.log | 8 ++++---- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CHANGES b/CHANGES index ce7d654166..0979d54939 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,12 @@ +2.0-beta-145 | 2011-12-19 11:37:15 -0800 + + * Empty fields are now logged as "(empty)" by default. (Robin + Sommer) + + * In log headers, only escape information when necessary. (Robin + Sommer) + 2.0-beta-139 | 2011-12-19 07:06:29 -0800 * The hostname notice email extension works now, plus a general diff --git a/VERSION b/VERSION index 03329dc416..391d4e570f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0-beta-139 +2.0-beta-145 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log index 8c6fbee126..c9e69994fc 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log @@ -1,8 +1,8 @@ #separator \x09 -#set_separator \x2c -#empty_field \x2d -#unset_field \x2d +#set_separator , +#empty_field (empty) +#unset_field - #path test #fields x y z #types string string string -\x2d - - +\x2d - (empty) From a14ec02d3b75788966db7dccb1e8a533e627300a Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 19 Dec 2011 12:43:25 -0800 Subject: [PATCH 211/964] change empty field defenition like in logging framework --- scripts/base/frameworks/input/readers/ascii.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base/frameworks/input/readers/ascii.bro b/scripts/base/frameworks/input/readers/ascii.bro index 9f630975a2..14c04757f7 100644 --- a/scripts/base/frameworks/input/readers/ascii.bro +++ b/scripts/base/frameworks/input/readers/ascii.bro @@ -12,7 +12,7 @@ export { const set_separator = "," &redef; ## String to use for empty fields. - const empty_field = "-" &redef; + const empty_field = "(empty)" &redef; ## String to use for an unset &optional field. const unset_field = "-" &redef; From 70a2cf67324c607014b067af13fc9c6887e571e3 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 19 Dec 2011 12:43:51 -0800 Subject: [PATCH 212/964] update baseline to include input framework --- .../coverage.bare-load-baseline/canonified_loaded_scripts.log | 4 ++++ .../canonified_loaded_scripts.log | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 8fab67304e..5a0de92a6e 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -18,4 +18,8 @@ scripts/base/init-bare.bro scripts/base/frameworks/logging/./postprocessors/__load__.bro scripts/base/frameworks/logging/./postprocessors/./scp.bro scripts/base/frameworks/logging/./writers/ascii.bro + scripts/base/frameworks/input/__load__.bro + scripts/base/frameworks/input/./main.bro + build/src/base/input.bif.bro + scripts/base/frameworks/input/./readers/ascii.bro scripts/policy/misc/loaded-scripts.bro diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 3f77797df8..e727928579 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -18,6 +18,10 @@ scripts/base/init-bare.bro scripts/base/frameworks/logging/./postprocessors/__load__.bro scripts/base/frameworks/logging/./postprocessors/./scp.bro scripts/base/frameworks/logging/./writers/ascii.bro + scripts/base/frameworks/input/__load__.bro + scripts/base/frameworks/input/./main.bro + build/src/base/input.bif.bro + scripts/base/frameworks/input/./readers/ascii.bro scripts/base/init-default.bro scripts/base/utils/site.bro scripts/base/utils/./patterns.bro From 436be4e07b1ce826146d902c19262cfb3cbd69aa Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 19 Dec 2011 15:06:52 -0600 Subject: [PATCH 213/964] Remove dead code related to record type inheritance. --- src/Serializer.h | 2 +- src/Type.cc | 154 +++-------------------------------------------- src/Type.h | 21 +------ src/parse.y | 15 ++--- 4 files changed, 14 insertions(+), 178 deletions(-) diff --git a/src/Serializer.h b/src/Serializer.h index db09cc837f..93581d83ce 100644 --- a/src/Serializer.h +++ b/src/Serializer.h @@ -121,7 +121,7 @@ protected: // This will be increased whenever there is an incompatible change // in the data format. - static const uint32 DATA_FORMAT_VERSION = 20; + static const uint32 DATA_FORMAT_VERSION = 21; ChunkedIO* io; diff --git a/src/Type.cc b/src/Type.cc index cd40583aae..4d80eda6f7 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -876,74 +876,12 @@ void CommentedTypeDecl::DescribeReST(ODesc* d) const } } -RecordField::RecordField(int arg_base, int arg_offset, int arg_total_offset) - { - base = arg_base; - offset = arg_offset; - total_offset = arg_total_offset; - } - RecordType::RecordType(type_decl_list* arg_types) : BroType(TYPE_RECORD) { types = arg_types; - base = 0; - fields = 0; num_fields = types ? types->length() : 0; } -RecordType::RecordType(TypeList* arg_base, type_decl_list* refinements) - : BroType(TYPE_RECORD) - { - if ( refinements ) - arg_base->Append(new RecordType(refinements)); - - Init(arg_base); - } - -void RecordType::Init(TypeList* arg_base) - { - assert(false); // Is this ever used? - - base = arg_base; - - if ( ! base ) - Internal("empty RecordType"); - - fields = new PDict(RecordField)(ORDERED); - types = 0; - - type_list* t = base->Types(); - - loop_over_list(*t, i) - { - BroType* ti = (*t)[i]; - - if ( ti->Tag() != TYPE_RECORD ) - (*t)[i]->Error("non-record in base type list"); - - RecordType* rti = ti->AsRecordType(); - int n = rti->NumFields(); - - for ( int j = 0; j < n; ++j ) - { - const TypeDecl* tdij = rti->FieldDecl(j); - - if ( fields->Lookup(tdij->id) ) - { - reporter->Error("duplicate field %s", tdij->id); - continue; - } - - RecordField* rf = new RecordField(i, j, fields->Length()); - - if ( fields->Insert(tdij->id, rf) ) - Internal("duplicate field when constructing record"); - } - } - - num_fields = fields->Length(); - } - RecordType::~RecordType() { if ( types ) @@ -953,9 +891,6 @@ RecordType::~RecordType() delete types; } - - delete fields; - Unref(base); } int RecordType::HasField(const char* field) const @@ -971,17 +906,7 @@ BroType* RecordType::FieldType(const char* field) const BroType* RecordType::FieldType(int field) const { - if ( types ) - return (*types)[field]->type; - else - { - RecordField* rf = fields->NthEntry(field); - if ( ! rf ) - Internal("missing field in RecordType::FieldType"); - BroType* bt = (*base->Types())[rf->base]; - RecordType* rbt = bt->AsRecordType(); - return rbt->FieldType(rf->offset); - } + return (*types)[field]->type; } Val* RecordType::FieldDefault(int field) const @@ -998,26 +923,14 @@ Val* RecordType::FieldDefault(int field) const int RecordType::FieldOffset(const char* field) const { - if ( types ) + loop_over_list(*types, i) { - loop_over_list(*types, i) - { - TypeDecl* td = (*types)[i]; - if ( streq(td->id, field) ) - return i; - } - - return -1; + TypeDecl* td = (*types)[i]; + if ( streq(td->id, field) ) + return i; } - else - { - RecordField* rf = fields->Lookup(field); - if ( ! rf ) - return -1; - else - return rf->total_offset; - } + return -1; } const char* RecordType::FieldName(int field) const @@ -1027,33 +940,12 @@ const char* RecordType::FieldName(int field) const const TypeDecl* RecordType::FieldDecl(int field) const { - if ( types ) - return (*types)[field]; - else - { - RecordField* rf = fields->NthEntry(field); - if ( ! rf ) - reporter->InternalError("missing field in RecordType::FieldDecl"); - - BroType* bt = (*base->Types())[rf->base]; - RecordType* rbt = bt->AsRecordType(); - return rbt->FieldDecl(rf->offset); - } + return (*types)[field]; } TypeDecl* RecordType::FieldDecl(int field) { - if ( types ) - return (*types)[field]; - else - { - RecordField* rf = fields->NthEntry(field); - if ( ! rf ) - Internal("missing field in RecordType::FieldDecl"); - BroType* bt = (*base->Types())[rf->base]; - RecordType* rbt = bt->AsRecordType(); - return rbt->FieldDecl(rf->offset); - } + return (*types)[field]; } void RecordType::Describe(ODesc* d) const @@ -1151,11 +1043,6 @@ void RecordType::DescribeFields(ODesc* d) const d->SP(); } } - else - { - d->AddCount(1); - base->Describe(d); - } } } @@ -1208,9 +1095,6 @@ bool RecordType::DoSerialize(SerialInfo* info) const else if ( ! SERIALIZE(false) ) return false; - SERIALIZE_OPTIONAL(base); - - // We don't serialize the fields as we can reconstruct them. return true; } @@ -1245,13 +1129,6 @@ bool RecordType::DoUnserialize(UnserialInfo* info) else types = 0; - BroType* type; - UNSERIALIZE_OPTIONAL(type, BroType::Unserialize(info, TYPE_LIST)); - base = (TypeList*) type; - - if ( base ) - Init(base); - return true; } @@ -1594,21 +1471,6 @@ bool VectorType::DoUnserialize(UnserialInfo* info) return yield_type != 0; } -BroType* refine_type(TypeList* base, type_decl_list* refinements) - { - type_list* t = base->Types(); - - if ( t->length() == 1 && ! refinements ) - { // Just a direct reference to a single type. - BroType* rt = (*t)[0]->Ref(); - Unref(base); - return rt; - } - - return new RecordType(base, refinements); - } - - BroType* base_type(TypeTag tag) { static BroType* base_types[NUM_TYPES]; diff --git a/src/Type.h b/src/Type.h index 5ebc5761a3..e935ba2267 100644 --- a/src/Type.h +++ b/src/Type.h @@ -426,20 +426,9 @@ public: std::list* comments; }; -class RecordField { -public: - RecordField(int arg_base, int arg_offset, int arg_total_offset); - - int base; // which base element it belongs to - int offset; // where it is in that base - int total_offset; // where it is in the aggregate record -}; -declare(PDict,RecordField); - class RecordType : public BroType { public: RecordType(type_decl_list* types); - RecordType(TypeList* base, type_decl_list* refinements); ~RecordType(); @@ -473,15 +462,11 @@ public: void DescribeFieldsReST(ODesc* d, bool func_args) const; protected: - RecordType() { fields = 0; base = 0; types = 0; } - - void Init(TypeList* arg_base); + RecordType() { types = 0; } DECLARE_SERIAL(RecordType) int num_fields; - PDict(RecordField)* fields; - TypeList* base; type_decl_list* types; }; @@ -587,10 +572,6 @@ protected: BroType* yield_type; }; - -// Returns the given type refinement, or error_type() if it's illegal. -extern BroType* refine_type(TypeList* base, type_decl_list* refinements); - // Returns the BRO basic (non-parameterized) type with the given type. extern BroType* base_type(TypeTag tag); diff --git a/src/parse.y b/src/parse.y index 495931aae0..7ab6c6bd32 100644 --- a/src/parse.y +++ b/src/parse.y @@ -2,7 +2,7 @@ // See the file "COPYING" in the main distribution directory for copyright. %} -%expect 88 +%expect 87 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF @@ -53,7 +53,7 @@ %type expr init anonymous_function %type event %type stmt stmt_list func_body for_head -%type type opt_type refined_type enum_body +%type type opt_type enum_body %type func_hdr func_params %type type_list %type type_decl formal_args_decl @@ -1104,7 +1104,7 @@ decl: } } - | TOK_TYPE global_id ':' refined_type opt_attr ';' + | TOK_TYPE global_id ':' type opt_attr ';' { add_type($2, $4, $5, 0); @@ -1134,7 +1134,7 @@ decl: } } - | TOK_EVENT event_id ':' refined_type opt_attr ';' + | TOK_EVENT event_id ':' type_list opt_attr ';' { add_type($2, $4, $5, 1); @@ -1220,13 +1220,6 @@ func_params: { $$ = new FuncType($2, base_type(TYPE_VOID), 0); } ; -refined_type: - type_list '{' type_decl_list '}' - { $$ = refine_type($1, $3); } - | type_list - { $$ = refine_type($1, 0); } - ; - opt_type: ':' type { $$ = $2; } From 275420dd299e21aca1c7b4cb8d2ad45b9a748e59 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 19 Dec 2011 16:28:30 -0600 Subject: [PATCH 214/964] Minor notice documentation tweaks. --- scripts/base/frameworks/notice/extend-email/hostnames.bro | 5 +++++ scripts/base/frameworks/notice/main.bro | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/base/frameworks/notice/extend-email/hostnames.bro b/scripts/base/frameworks/notice/extend-email/hostnames.bro index 17a597678d..2ec6dbb23f 100644 --- a/scripts/base/frameworks/notice/extend-email/hostnames.bro +++ b/scripts/base/frameworks/notice/extend-email/hostnames.bro @@ -1,3 +1,8 @@ +##! Loading this script extends the :bro:enum:`Notice::ACTION_EMAIL` action +##! by appending to the email the hostnames associated with +##! :bro:type:`Notice::Info`'s *src* and *dst* fields as determined by a +##! DNS lookup. + @load ../main module Notice; diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index 709b5eeae9..9982041978 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -48,7 +48,7 @@ export { }; ## The notice framework is able to do automatic notice supression by - ## utilizing the $identifier field in :bro:type:`Info` records. + ## utilizing the $identifier field in :bro:type:`Notice::Info` records. ## Set this to "0secs" to completely disable automated notice suppression. const default_suppression_interval = 1hrs &redef; @@ -111,7 +111,7 @@ export { ## Adding a string "token" to this set will cause the notice framework's ## built-in emailing functionality to delay sending the email until ## either the token has been removed or the email has been delayed - ## for :bro:id:`max_email_delay`. + ## for :bro:id:`Notice::max_email_delay`. email_delay_tokens: set[string] &optional; ## This field is to be provided when a notice is generated for the From ff1768f857abcd01599854b65677a87e293b1cf6 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 20 Dec 2011 11:47:39 -0600 Subject: [PATCH 215/964] Minor fixes to external test process. (addresses #298) - Skip diffing of debug.log always. - Skip diffing of reporter.log if it only contains an error about missing GeoIP support. - Canonicalize X.509 Distinguished Name subjects since that can vary depending on installed OpenSSL version. --- testing/external/scripts/diff-all | 11 +++++++- testing/scripts/diff-canonifier-external | 1 + testing/scripts/diff-remove-x509-names | 32 ++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100755 testing/scripts/diff-remove-x509-names diff --git a/testing/external/scripts/diff-all b/testing/external/scripts/diff-all index 329bbb7f00..1fd633b1e9 100755 --- a/testing/external/scripts/diff-all +++ b/testing/external/scripts/diff-all @@ -22,7 +22,16 @@ files_cwd=`ls $@` files_baseline=`cd $TEST_BASELINE && ls $@` for i in `echo $files_cwd $files_baseline | sort | uniq`; do - if [[ "$i" != "loaded_scripts.log" && "$i" != "prof.log" ]]; then + if [[ "$i" != "loaded_scripts.log" && "$i" != "prof.log" && "$i" != "debug.log" ]]; then + + if [[ "$i" == "reporter.log" ]]; then + # Do not diff the reporter.log if it only complains about missing + # GeoIP support. + if ! egrep -v "^#|Bro was not configured for GeoIP support" $i; then + continue + fi + fi + if ! btest-diff $i; then echo "" >>$diag echo "#### btest-diff $i" >>$diag diff --git a/testing/scripts/diff-canonifier-external b/testing/scripts/diff-canonifier-external index 76210cc494..1f953183d3 100755 --- a/testing/scripts/diff-canonifier-external +++ b/testing/scripts/diff-canonifier-external @@ -5,3 +5,4 @@ `dirname $0`/diff-remove-timestamps \ | `dirname $0`/diff-remove-uids \ | `dirname $0`/diff-remove-mime-types \ + | `dirname $0`/diff-remove-x509-names \ diff --git a/testing/scripts/diff-remove-x509-names b/testing/scripts/diff-remove-x509-names new file mode 100755 index 0000000000..6209edfc65 --- /dev/null +++ b/testing/scripts/diff-remove-x509-names @@ -0,0 +1,32 @@ +#! /usr/bin/awk -f +# +# A diff canonifier that removes all X.509 Distinguished Name subject fields +# because that output can differ depending on installed OpenSSL version. + +BEGIN { FS="\t"; OFS="\t"; s_col = -1; i_col = -1 } + +/^#fields/ { + for ( i = 2; i < NF; ++i ) + { + if ( $i == "subject" ) + s_col = i-1; + if ( $i == "issuer_subject" ) + i_col = i-1; + } +} + +s_col >= 0 { + if ( $s_col != "-" ) + # Mark that it's set, but ignore content. + $s_col = "+"; +} + +i_col >= 0 { + if ( $i_col != "-" ) + # Mark that it's set, but ignore content. + $i_col = "+"; +} + +{ + print; +} From 0cd03986a195c9321aa79805254d93e8139bde38 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 3 Jan 2012 10:56:33 -0600 Subject: [PATCH 216/964] CMake 2.6 top-level 'install' target compat. (fixes #729) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 83a149ca58..277be4d017 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ HAVE_MODULES=git submodule | grep -v cmake >/dev/null all: configured $(MAKE) -C $(BUILD) $@ -install: configured +install: configured all $(MAKE) -C $(BUILD) $@ install-aux: configured From 8200544293a7cbb3747985ec3066a765a26fae6b Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 3 Jan 2012 15:29:25 -0800 Subject: [PATCH 217/964] Updating test baselines for log change --- .../manager-1.notice.log | 6 +++--- .../scripts.base.frameworks.metrics.notice/notice.log | 8 ++++---- .../manager-1.notice.log | 6 +++--- .../manager-1.notice.log | 6 +++--- .../scripts.base.frameworks.notice.suppression/notice.log | 6 +++--- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log index 33f55ce608..59d70896fb 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log @@ -3,6 +3,6 @@ #empty_field (empty) #unset_field - #path notice -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1324314350.184962 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network +#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet +1325633225.777902 - - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log index 437b1465a1..58346b79e6 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path notice -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1324314359.357148 - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 3/2 - 1.2.3.4 - - 3 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - - -1324314359.357148 - - - - - Test_Notice Threshold crossed by metric_index(host=6.5.4.3) 2/2 - 6.5.4.3 - - 2 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - 6.5.4.3 - - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network +#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet +1325633274.875473 - - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 3/2 - 1.2.3.4 - - 3 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - - +1325633274.875473 - - - - - - Test_Notice Threshold crossed by metric_index(host=6.5.4.3) 2/2 - 6.5.4.3 - - 2 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - 6.5.4.3 - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log index fb1e1b3d47..10888b21ec 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log @@ -3,6 +3,6 @@ #empty_field (empty) #unset_field - #path notice -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1324314363.721823 - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network +#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet +1325633122.490990 - - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log index 9e6e1b1916..5deac88071 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log @@ -3,6 +3,6 @@ #empty_field (empty) #unset_field - #path notice -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1324314378.560010 - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network +#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet +1325633150.723248 - - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log index d134c97049..1d168d7613 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log @@ -3,6 +3,6 @@ #empty_field (empty) #unset_field - #path notice -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude -#types time string addr port addr port enum string string addr addr port count string table[enum] table[count] interval bool string string string double double -1324314387.663586 - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 6 3600.000000 F - - - - - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude +#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double +1325633207.922993 - - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 6 3600.000000 F - - - - - From 167b645ed079b11ad787ccce868861106166292f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 3 Jan 2012 15:58:15 -0800 Subject: [PATCH 218/964] Updating submodule(s). [nomail] --- aux/broccoli | 2 +- aux/broctl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aux/broccoli b/aux/broccoli index d7b8a43759..c5cee3d574 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit d7b8a43759bfcbe1381d132d8ab388937e52a6d4 +Subproject commit c5cee3d5746ed3d5c14348c1f264d19404caa761 diff --git a/aux/broctl b/aux/broctl index a42e4d133b..d3d5934310 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit a42e4d133b94622c612055047f8534d5122e6e88 +Subproject commit d3d5934310a94452b1dddabb2e75f6c5c86b4860 From adfbed8e56f95ca8bd2d4959c72214f54cb8b2f9 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 4 Jan 2012 13:37:07 -0500 Subject: [PATCH 219/964] The silliest, tiniest little whitespace fixes. --- .../frameworks/notice/actions/pp-alarms.bro | 60 +++++++++---------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/scripts/base/frameworks/notice/actions/pp-alarms.bro b/scripts/base/frameworks/notice/actions/pp-alarms.bro index 89cf310ac5..609f6a6bf1 100644 --- a/scripts/base/frameworks/notice/actions/pp-alarms.bro +++ b/scripts/base/frameworks/notice/actions/pp-alarms.bro @@ -10,19 +10,19 @@ module Notice; export { ## Activate pretty-printed alarm summaries. const pretty_print_alarms = T &redef; - + ## Address to send the pretty-printed reports to. Default if not set is ## :bro:id:`Notice::mail_dest`. const mail_dest_pretty_printed = "" &redef; - - ## If an address from one of these networks is reported, we mark + + ## If an address from one of these networks is reported, we mark ## the entry with an addition quote symbol (i.e., ">"). Many MUAs ## then highlight such lines differently. global flag_nets: set[subnet] &redef; - + ## Function that renders a single alarm. Can be overidden. global pretty_print_alarm: function(out: file, n: Info) &redef; - + ## Force generating mail file, even if reading from traces or no mail ## destination is defined. This is mainly for testing. global force_email_summaries = F &redef; @@ -38,7 +38,7 @@ function want_pp() : bool { if ( force_email_summaries ) return T; - + return (pretty_print_alarms && ! reading_traces() && (mail_dest != "" || mail_dest_pretty_printed != "")); } @@ -48,7 +48,7 @@ function pp_open() { if ( pp_alarms_open ) return; - + pp_alarms_open = T; pp_alarms = open(pp_alarms_name); } @@ -58,17 +58,17 @@ function pp_send(rinfo: Log::RotationInfo) { if ( ! pp_alarms_open ) return; - + write_file(pp_alarms, "\n\n--\n[Automatically generated]\n\n"); close(pp_alarms); pp_alarms_open = F; - + local from = strftime("%H:%M:%S", rinfo$open); local to = strftime("%H:%M:%S", rinfo$close); local subject = fmt("Alarm summary from %s-%s", from, to); local dest = mail_dest_pretty_printed != "" ? mail_dest_pretty_printed : mail_dest; - + if ( dest == "" ) # No mail destination configured, just leave the file alone. This is mainly for # testing. @@ -90,7 +90,7 @@ function pp_postprocessor(info: Log::RotationInfo): bool { if ( want_pp() ) pp_send(info); - + return T; } @@ -98,7 +98,7 @@ event bro_init() { if ( ! want_pp() ) return; - + # This replaces the standard non-pretty-printing filter. Log::add_filter(Notice::ALARM_LOG, [$name="alarm-mail", $writer=Log::WRITER_NONE, @@ -110,13 +110,13 @@ event notice(n: Notice::Info) &priority=-5 { if ( ! want_pp() ) return; - + if ( ACTION_ALARM !in n$actions ) return; - + if ( ! pp_alarms_open ) pp_open(); - + pretty_print_alarm(pp_alarms, n); } @@ -126,12 +126,12 @@ function do_msg(out: file, n: Info, line1: string, line2: string, line3: string, @ifdef ( Notice::ACTION_ADD_GEODATA ) # Make tests happy, cyclic dependency. if ( n?$remote_location && n$remote_location?$country_code ) country = fmt(" (remote location %s)", n$remote_location$country_code); -@endif - +@endif + line1 = cat(line1, country); - + local resolved = ""; - + if ( host1 != 0.0.0.0 ) resolved = fmt("%s # %s = %s", resolved, host1, name1); @@ -151,60 +151,58 @@ function do_msg(out: file, n: Info, line1: string, line2: string, line3: string, function pretty_print_alarm(out: file, n: Info) { local pdescr = ""; - + @if ( Cluster::is_enabled() ) pdescr = "local"; - + if ( n?$src_peer ) pdescr = n$src_peer?$descr ? n$src_peer$descr : fmt("%s", n$src_peer$host); pdescr = fmt("<%s> ", pdescr); @endif - + local msg = fmt( "%s%s", pdescr, n$msg); - + local who = ""; local h1 = 0.0.0.0; local h2 = 0.0.0.0; - + local orig_p = ""; local resp_p = ""; - + if ( n?$id ) { h1 = n$id$orig_h; h2 = n$id$resp_h; who = fmt("%s:%s -> %s:%s", h1, n$id$orig_p, h2, n$id$resp_p); } - else if ( n?$src && n?$dst ) { h1 = n$src; h2 = n$dst; who = fmt("%s -> %s", h1, h2); } - else if ( n?$src ) { h1 = n$src; who = fmt("%s%s", h1, (n?$p ? fmt(":%s", n$p) : "")); } - + if ( n?$uid ) who = fmt("%s (uid %s)", who, n$uid ); local flag = (h1 in flag_nets || h2 in flag_nets); - + local line1 = fmt(">%s %D %s %s", (flag ? ">" : " "), network_time(), n$note, who); local line2 = fmt(" %s", msg); local line3 = n?$sub ? fmt(" %s", n$sub) : ""; - + if ( h1 == 0.0.0.0 ) { do_msg(out, n, line1, line2, line3, h1, "", h2, ""); return; } - + if ( reading_traces() ) { do_msg(out, n, line1, line2, line3, h1, "", h2, ""); From 0b93b071ea2940ff8b644b34804183bcbd3e5e42 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 4 Jan 2012 13:53:26 -0500 Subject: [PATCH 220/964] Added the ssl_session_ticket_handshake event back. --- src/event.bif | 1 + 1 file changed, 1 insertion(+) diff --git a/src/event.bif b/src/event.bif index 3f0e2992ed..beceaaa6e3 100644 --- a/src/event.bif +++ b/src/event.bif @@ -279,6 +279,7 @@ event ssh_server_version%(c: connection, version: string%); event ssl_client_hello%(c: connection, version: count, possible_ts: time, session_id: string, ciphers: count_set%); event ssl_server_hello%(c: connection, version: count, possible_ts: time, session_id: string, cipher: count, comp_method: count%); +event ssl_session_ticket_handshake%(c: connection, ticket_lifetime_hint: count, ticket: string%); event ssl_extension%(c: connection, is_orig: bool, code: count, val: string%); event ssl_alert%(c: connection, is_orig: bool, level: count, desc: count%); event ssl_established%(c: connection%); From e5bb76e684257036ace1286ac8522a10d3eea0bb Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 4 Jan 2012 14:51:19 -0500 Subject: [PATCH 221/964] Removed the SSLv2.cc file since it's not used. --- src/SSLv2.cc | 944 --------------------------------------------------- 1 file changed, 944 deletions(-) delete mode 100644 src/SSLv2.cc diff --git a/src/SSLv2.cc b/src/SSLv2.cc deleted file mode 100644 index 9fa654048d..0000000000 --- a/src/SSLv2.cc +++ /dev/null @@ -1,944 +0,0 @@ -#include "SSLv2.h" -#include "SSLv3.h" - -// --- Initalization of static variables -------------------------------------- - -uint SSLv2_Interpreter::totalConnections = 0; -uint SSLv2_Interpreter::analyzedConnections = 0; -uint SSLv2_Interpreter::openedConnections = 0; -uint SSLv2_Interpreter::failedConnections = 0; -uint SSLv2_Interpreter::weirdConnections = 0; -uint SSLv2_Interpreter::totalRecords = 0; -uint SSLv2_Interpreter::clientHelloRecords = 0; -uint SSLv2_Interpreter::serverHelloRecords = 0; -uint SSLv2_Interpreter::clientMasterKeyRecords = 0; -uint SSLv2_Interpreter::errorRecords = 0; - - -// --- SSLv2_Interpreter ------------------------------------------------------- - -/*! - * The Constructor. - * - * \param proxy Pointer to the SSLProxy_Analyzer who created this instance. - */ -SSLv2_Interpreter::SSLv2_Interpreter(SSLProxy_Analyzer* proxy) -: SSL_Interpreter(proxy) - { - ++totalConnections; - records = 0; - bAnalyzedCounted = false; - connState = START; - - pServerCipherSpecs = 0; - pClientCipherSpecs = 0; - bClientWantsCachedSession = false; - usedCipherSpec = (SSLv2_CipherSpec) 0; - - pConnectionId = 0; - pChallenge = 0; - pSessionId = 0; - pMasterClearKey = 0; - pMasterEncryptedKey = 0; - pClientReadKey = 0; - pServerReadKey = 0; - } - -/*! - * The Destructor. - */ -SSLv2_Interpreter::~SSLv2_Interpreter() - { - if ( connState != CLIENT_MASTERKEY_SEEN && - connState != CACHED_SESSION && - connState != START && // we only complain if we saw some data - connState != ERROR_SEEN ) - ++failedConnections; - - if ( connState != CLIENT_MASTERKEY_SEEN && connState != CACHED_SESSION ) - ++weirdConnections; - - delete pServerCipherSpecs; - delete pClientCipherSpecs; - delete pConnectionId; - delete pChallenge; - delete pSessionId; - delete pMasterClearKey; - delete pMasterEncryptedKey; - delete pClientReadKey; - delete pServerReadKey; - } - -/*! - * This method implements SSL_Interpreter::BuildInterpreterEndpoints() - */ -void SSLv2_Interpreter::BuildInterpreterEndpoints() - { - orig = new SSLv2_Endpoint(this, 1); - resp = new SSLv2_Endpoint(this, 0); - } - -/*! - * This method prints some counters. - */ -void SSLv2_Interpreter::printStats() - { - printf("SSLv2:\n"); - printf("totalConnections = %u\n", totalConnections); - printf("analyzedConnections = %u\n", analyzedConnections); - printf("openedConnections = %u\n", openedConnections); - printf("failedConnections = %u\n", failedConnections); - printf("weirdConnections = %u\n", weirdConnections); - - printf("totalRecords = %u\n", totalRecords); - printf("clientHelloRecords = %u\n", clientHelloRecords); - printf("serverHelloRecords = %u\n", serverHelloRecords); - printf("clientMasterKeyRecords = %u\n", clientMasterKeyRecords); - printf("errorRecords = %u\n", errorRecords); - - printf("SSL_RecordBuilder::maxAllocCount = %u\n", SSL_RecordBuilder::maxAllocCount); - printf("SSL_RecordBuilder::maxFragmentCount = %u\n", SSL_RecordBuilder::maxFragmentCount); - printf("SSL_RecordBuilder::fragmentedHeaders = %u\n", SSL_RecordBuilder::fragmentedHeaders); - } - -/*! - * \return the current state of the ssl connection - */ -SSLv2_States SSLv2_Interpreter::ConnState() - { - return connState; - } - -/*! - * This method is called by SSLv2_Endpoint::Deliver(). It is the main entry - * point of this class. The header of the given SSLV2 record is analyzed and - * its contents are then passed to the corresponding analyzer method. After - * the record has been analyzed, the ssl connection state is updated. - * - * \param s Pointer to the endpoint which sent the record - * \param length length of SSLv2 record - * \param data pointer to SSLv2 record to analyze - */ -void SSLv2_Interpreter::NewSSLRecord(SSL_InterpreterEndpoint* s, - int length, const u_char* data) - { - ++records; - ++totalRecords; - - if ( ! bAnalyzedCounted ) - { - ++analyzedConnections; - bAnalyzedCounted = true; - } - - // We should see a maximum of 4 cleartext records. - if ( records == 5 ) - { // so this should never happen - Weird("SSLv2: Saw more than 4 records, skipping connection..."); - proxy->SetSkip(1); - return; - } - - // SSLv2 record header analysis - uint32 recordLength = 0; // data length of SSLv2 record - bool isEscape = false; - uint8 padding = 0; - const u_char* contents; - - if ( (data[0] & 0x80) > 0 ) - { // we have a two-byte record header - recordLength = ((data[0] & 0x7f) << 8) | data[1]; - contents = data + 2; - if ( recordLength + 2 != uint32(length) ) - { - // This should never happen, otherwise - // we have a bug in the SSL_RecordBuilder. - Weird("SSLv2: FATAL: recordLength doesn't match data block length!"); - connState = ERROR_REQUIRED; - proxy->SetSkip(1); - return; - } - } - else - { // We have a three-byte record header. - recordLength = ((data[0] & 0x3f) << 8) | data[1]; - isEscape = (data[0] & 0x40) != 0; - padding = data[2]; - contents = data + 3; - if ( recordLength + 3 != uint32(length) ) - { - // This should never happen, otherwise - // we have a bug in the SSL_RecordBuilder. - Weird("SSLv2: FATAL: recordLength doesn't match data block length!"); - connState = ERROR_REQUIRED; - proxy->SetSkip(1); - return; - } - - if ( padding == 0 && ! isEscape ) - Weird("SSLv2: 3 Byte record header, but no escape, no padding!"); - } - - if ( recordLength == 0 ) - { - Weird("SSLv2: Record length is zero (no record data)!"); - return; - } - - if ( isEscape ) - Weird("SSLv2: Record has escape bit set (security escape)!"); - - if ( padding > 0 && connState != CACHED_SESSION && - connState != CLIENT_MASTERKEY_SEEN ) - Weird("SSLv2 record with padding > 0 in cleartext!"); - - // MISSING: - // A final consistency check is done when a block cipher is used - // and the protocol is using encryption. The amount of data present - // in a record (RECORD-LENGTH))must be a multiple of the cipher's - // block size. If the received record is not a multiple of the - // cipher's block size then the record is considered damaged, and it - // is to be treated as if an "I/O Error" had occurred (i.e. an - // unrecoverable error is asserted and the connection is closed). - - switch ( connState ) { - case START: - // Only CLIENT-HELLLOs allowed here. - if ( contents[0] != SSLv2_MT_CLIENT_HELLO ) - { - Weird("SSLv2: First packet is not a CLIENT-HELLO!"); - analyzeRecord(s, recordLength, contents); - connState = ERROR_REQUIRED; - } - else - connState = ClientHelloRecord(s, recordLength, contents); - break; - - case CLIENT_HELLO_SEEN: - // Only SERVER-HELLOs or ERRORs allowed here. - if ( contents[0] == SSLv2_MT_SERVER_HELLO ) - connState = ServerHelloRecord(s, recordLength, contents); - else if ( contents[0] == SSLv2_MT_ERROR ) - connState = ErrorRecord(s, recordLength, contents); - else - { - Weird("SSLv2: State violation in CLIENT_HELLO_SEEN!"); - analyzeRecord(s, recordLength, contents); - connState = ERROR_REQUIRED; - } - break; - - case NEW_SESSION: - // We expect a client master key. - if ( contents[0] == SSLv2_MT_CLIENT_MASTER_KEY ) - connState = ClientMasterKeyRecord(s, recordLength, contents); - else if ( contents[0] == SSLv2_MT_ERROR ) - connState = ErrorRecord(s, recordLength, contents); - else - { - Weird("SSLv2: State violation in NEW_SESSION or encrypted record!"); - analyzeRecord(s, recordLength, contents); - connState = ERROR_REQUIRED; - } - - delete pServerCipherSpecs; - pServerCipherSpecs = 0; - break; - - case CACHED_SESSION: - delete pServerCipherSpecs; - pServerCipherSpecs = 0; - // No break here. - - case CLIENT_MASTERKEY_SEEN: - // If no error record, no further analysis. - if ( contents[0] == SSLv2_MT_ERROR && - recordLength == SSLv2_ERROR_RECORD_SIZE ) - connState = ErrorRecord(s, recordLength, contents); - else - { - // So we finished the cleartext handshake. - // Skip all further data. - - proxy->SetSkip(1); - ++openedConnections; - } - break; - - case ERROR_REQUIRED: - if ( contents[0] == SSLv2_MT_ERROR ) - connState = ErrorRecord(s, recordLength, contents); - else - { - // We lost tracking: this should not happen. - Weird("SSLv2: State inconsistency in ERROR_REQUIRED (lost tracking!)!"); - analyzeRecord(s, recordLength, contents); - connState = ERROR_REQUIRED; - } - break; - - case ERROR_SEEN: - // We don't have recoverable errors in cleartext phase, - // so we shouldn't see anymore packets. - Weird("SSLv2: Traffic after error record!"); - analyzeRecord(s, recordLength, contents); - break; - - default: - reporter->InternalError("SSLv2: unknown state"); - break; - } - } - -/*! - * This method is called whenever the connection tracking failed. It calls - * the corresponding analyzer method for the given SSLv2 record, but does not - * update the ssl connection state. - * - * \param s Pointer to the endpoint which sent the record - * \param length length of SSLv2 record - * \param data pointer to SSLv2 record to analyze - */ -void SSLv2_Interpreter::analyzeRecord(SSL_InterpreterEndpoint* s, - int length, const u_char* data) - { - switch ( data[0] ) { - case SSLv2_MT_ERROR: - ErrorRecord(s, length, data); - break; - - case SSLv2_MT_CLIENT_HELLO: - ClientHelloRecord(s, length, data); - break; - - case SSLv2_MT_CLIENT_MASTER_KEY: - ClientMasterKeyRecord(s, length, data); - break; - - case SSLv2_MT_SERVER_HELLO: - ServerHelloRecord(s, length, data); - break; - - case SSLv2_MT_CLIENT_FINISHED: - case SSLv2_MT_SERVER_VERIFY: - case SSLv2_MT_SERVER_FINISHED: - case SSLv2_MT_REQUEST_CERTIFICATE: - case SSLv2_MT_CLIENT_CERTIFICATE: - Weird("SSLv2: Encrypted record type seems to be in cleartext"); - break; - - default: - // Unknown record type. - Weird("SSLv2: Unknown record type or encrypted record"); - break; - } - } - -/*! - * This method analyses a SSLv2 CLIENT-HELLO record. - * - * \param s Pointer to the endpoint which sent the record - * \param length length of SSLv2 CLIENT-HELLO record - * \param data pointer to SSLv2 CLIENT-HELLO record to analyze - * - * \return the updated state of the current ssl connection - */ -SSLv2_States SSLv2_Interpreter::ClientHelloRecord(SSL_InterpreterEndpoint* s, - int recordLength, const u_char* recordData) - { - // This method gets the record's data (without the header). - ++clientHelloRecords; - - if ( s != orig ) - Weird("SSLv2: CLIENT-HELLO record from server!"); - - // There should not be any pending data in the SSLv2 reassembler, - // because the client should wait for a server response. - if ( ((SSLv2_Endpoint*) s)->isDataPending() ) - Weird("SSLv2: Pending data in SSL_RecordBuilder after CLIENT-HELLO!"); - - // Client hello minimum header size check. - if ( recordLength < SSLv2_CLIENT_HELLO_HEADER_SIZE ) - { - Weird("SSLv2: CLIENT-HELLO is too small!"); - return ERROR_REQUIRED; - } - - // Extract the data of the client hello header. - SSLv2_ClientHelloHeader ch; - ch.clientVersion = uint16(recordData[1] << 8) | recordData[2]; - ch.cipherSpecLength = uint16(recordData[3] << 8) | recordData[4]; - ch.sessionIdLength = uint16(recordData[5] << 8) | recordData[6]; - ch.challengeLength = uint16(recordData[7] << 8) | recordData[8]; - - if ( ch.clientVersion != SSLProxy_Analyzer::SSLv20 && - ch.clientVersion != SSLProxy_Analyzer::SSLv30 && - ch.clientVersion != SSLProxy_Analyzer::SSLv31 ) - { - Weird("SSLv2: Unsupported SSL-Version in CLIENT-HELLO"); - return ERROR_REQUIRED; - } - - if ( ch.challengeLength + ch.cipherSpecLength + ch.sessionIdLength + - SSLv2_CLIENT_HELLO_HEADER_SIZE != recordLength ) - { - Weird("SSLv2: Size inconsistency in CLIENT-HELLO"); - return ERROR_REQUIRED; - } - - // The CIPHER-SPECS-LENGTH must be > 0 and a multiple of 3. - if ( ch.cipherSpecLength == 0 || ch.cipherSpecLength % 3 != 0 ) - { - Weird("SSLv2: Nonconform CIPHER-SPECS-LENGTH in CLIENT-HELLO."); - return ERROR_REQUIRED; - } - - // The SESSION-ID-LENGTH must either be zero or 16. - if ( ch.sessionIdLength != 0 && ch.sessionIdLength != 16 ) - Weird("SSLv2: Nonconform SESSION-ID-LENGTH in CLIENT-HELLO."); - - if ( (ch.challengeLength < 16) || (ch.challengeLength > 32)) - Weird("SSLv2: Nonconform CHALLENGE-LENGTH in CLIENT-HELLO."); - - const u_char* ptr = recordData; - ptr += SSLv2_CLIENT_HELLO_HEADER_SIZE + ch.cipherSpecLength; - - pSessionId = new SSL_DataBlock(ptr, ch.sessionIdLength); - - // If decrypting, store the challenge. - if ( ssl_store_key_material && ch.challengeLength <= 32 ) - pChallenge = new SSL_DataBlock(ptr, ch.challengeLength); - - bClientWantsCachedSession = ch.sessionIdLength != 0; - - TableVal* currentCipherSuites = - analyzeCiphers(s, ch.cipherSpecLength, - recordData + SSLv2_CLIENT_HELLO_HEADER_SIZE); - - fire_ssl_conn_attempt(ch.clientVersion, currentCipherSuites); - - return CLIENT_HELLO_SEEN; - } - -/*! - * This method analyses a SSLv2 SERVER-HELLO record. - * - * \param s Pointer to the endpoint which sent the record - * \param length length of SSLv2 SERVER-HELLO record - * \param data pointer to SSLv2 SERVER-HELLO record to analyze - * - * \return the updated state of the current ssl connection - */ -SSLv2_States SSLv2_Interpreter::ServerHelloRecord(SSL_InterpreterEndpoint* s, - int recordLength, const u_char* recordData) - { - ++serverHelloRecords; - TableVal* currentCipherSuites = NULL; - - if ( s != resp ) - Weird("SSLv2: SERVER-HELLO from client!"); - - if ( recordLength < SSLv2_SERVER_HELLO_HEADER_SIZE ) - { - Weird("SSLv2: SERVER-HELLO is too small!"); - return ERROR_REQUIRED; - } - - // Extract the data of the client hello header. - SSLv2_ServerHelloHeader sh; - sh.sessionIdHit = recordData[1]; - sh.certificateType = recordData[2]; - sh.serverVersion = uint16(recordData[3] << 8) | recordData[4]; - sh.certificateLength = uint16(recordData[5] << 8) | recordData[6]; - sh.cipherSpecLength = uint16(recordData[7] << 8) | recordData[8]; - sh.connectionIdLength = uint16(recordData[9] << 8) | recordData[10]; - - if ( sh.serverVersion != SSLProxy_Analyzer::SSLv20 ) - { - Weird("SSLv2: Unsupported SSL-Version in SERVER-HELLO"); - return ERROR_REQUIRED; - } - - if ( sh.certificateLength + sh.cipherSpecLength + - sh.connectionIdLength + - SSLv2_SERVER_HELLO_HEADER_SIZE != recordLength ) - { - Weird("SSLv2: Size inconsistency in SERVER-HELLO"); - return ERROR_REQUIRED; - } - - // The length of the CONNECTION-ID must be between 16 and 32 bytes. - if ( sh.connectionIdLength < 16 || sh.connectionIdLength > 32 ) - Weird("SSLv2: Nonconform CONNECTION-ID-LENGTH in SERVER-HELLO"); - - // If decrypting, store the connection ID. - if ( ssl_store_key_material && sh.connectionIdLength <= 32 ) - { - const u_char* ptr = recordData; - - ptr += SSLv2_SERVER_HELLO_HEADER_SIZE + sh.cipherSpecLength + - sh.certificateLength; - - pConnectionId = new SSL_DataBlock(ptr, sh.connectionIdLength); - } - - if ( sh.sessionIdHit == 0 ) - { - // Generating reusing-connection event. - EventHandlerPtr event = ssl_session_insertion; - - if ( event ) - { - TableVal* sessionIDTable = - MakeSessionID( - recordData + - SSLv2_SERVER_HELLO_HEADER_SIZE + - sh.certificateLength + - sh.cipherSpecLength, - sh.connectionIdLength); - - val_list* vl = new val_list; - vl->append(proxy->BuildConnVal()); - vl->append(sessionIDTable); - - proxy->ConnectionEvent(ssl_session_insertion, vl); - } - } - - SSLv2_States nextState; - - if ( sh.sessionIdHit != 0 ) - { // we're using a cached session - - // There should not be any pending data in the SSLv2 - // reassembler, because the server should wait for a - // client response. - if ( ((SSLv2_Endpoint*) s)->isDataPending() ) - { - // But turns out some SSL Implementations do this - // when using a cached session. - } - - // Consistency check for SESSION-ID-HIT. - if ( ! bClientWantsCachedSession ) - Weird("SSLv2: SESSION-ID hit in SERVER-HELLO, but no SESSION-ID in CLIENT-HELLO!"); - - // If the SESSION-ID-HIT flag is non-zero then the - // CERTIFICATE-TYPE, CERTIFICATE-LENGTH and - // CIPHER-SPECS-LENGTH fields will be zero. - if ( sh.certificateType != 0 || sh.certificateLength != 0 || - sh.cipherSpecLength != 0 ) - Weird("SSLv2: SESSION-ID-HIT, but session data in SERVER-HELLO"); - - // Generate reusing-connection event. - if ( pSessionId ) - { - fire_ssl_conn_reused(pSessionId); - delete pSessionId; - pSessionId = 0; - } - - nextState = CACHED_SESSION; - } - else - { // we're starting a new session - - // There should not be any pending data in the SSLv2 - // reassembler, because the server should wait for - // a client response. - if ( ((SSLv2_Endpoint*) s)->isDataPending() ) - Weird("SSLv2: Pending data in SSL_RecordBuilder after SERVER-HELLO (new session)!"); - - // TODO: check certificate length ??? - if ( sh.certificateLength == 0 ) - Weird("SSLv2: No certificate in SERVER-HELLO!"); - - // The CIPHER-SPECS-LENGTH must be > zero and a multiple of 3. - if ( sh.cipherSpecLength == 0 ) - Weird("SSLv2: No CIPHER-SPECS in SERVER-HELLO!"); - - if ( sh.cipherSpecLength % 3 != 0 ) - { - Weird("SSLv2: Nonconform CIPHER-SPECS-LENGTH in SERVER-HELLO"); - return ERROR_REQUIRED; - } - - const u_char* ptr = recordData; - ptr += sh.certificateLength + SSLv2_SERVER_HELLO_HEADER_SIZE; - currentCipherSuites = analyzeCiphers(s, sh.cipherSpecLength, ptr); - - nextState = NEW_SESSION; - } - - // Check if at least one cipher is supported by the client. - if ( pClientCipherSpecs && pServerCipherSpecs ) - { - bool bFound = false; - for ( int i = 0; i < pClientCipherSpecs->len; i += 3 ) - { - for ( int j = 0; j < pServerCipherSpecs->len; j += 3 ) - { - if ( memcmp(pClientCipherSpecs + i, - pServerCipherSpecs + j, 3) == 0 ) - { - bFound = true; - i = pClientCipherSpecs->len; - break; - } - } - } - - if ( ! bFound ) - { - Weird("SSLv2: Client's and server's CIPHER-SPECS don't match!"); - nextState = ERROR_REQUIRED; - } - - delete pClientCipherSpecs; - pClientCipherSpecs = 0; - } - - // Certificate analysis. - if ( sh.certificateLength > 0 && ssl_analyze_certificates != 0 ) - { - analyzeCertificate(s, recordData + SSLv2_SERVER_HELLO_HEADER_SIZE, - sh.certificateLength, sh.certificateType, false); - } - - if ( nextState == NEW_SESSION ) - // generate server-reply event - fire_ssl_conn_server_reply(sh.serverVersion, currentCipherSuites); - - else if ( nextState == CACHED_SESSION ) - { // generate server-reply event - fire_ssl_conn_server_reply(sh.serverVersion, currentCipherSuites); - // Generate a connection-established event with a dummy - // cipher suite, since we can't remember session information - // (yet). - // Note: A new session identifier is sent encrypted in SSLv2! - fire_ssl_conn_established(sh.serverVersion, 0xABCD); - } - else - // Unref, since the table is not delivered to any event. - Unref(currentCipherSuites); - - return nextState; - } - -/*! - * This method analyses a SSLv2 CLIENT-MASTER-KEY record. - * - * \param s Pointer to the endpoint which sent the record - * \param length length of SSLv2 CLIENT-MASTER-KEY record - * \param data pointer to SSLv2 CLIENT-MASTER-KEY record to analyze - * - * \return the updated state of the current ssl connection - */ -SSLv2_States SSLv2_Interpreter:: - ClientMasterKeyRecord(SSL_InterpreterEndpoint* s, int recordLength, - const u_char* recordData) - { - ++clientMasterKeyRecords; - SSLv2_States nextState = CLIENT_MASTERKEY_SEEN; - - if ( s != orig ) - Weird("SSLv2: CLIENT-MASTER-KEY from server!"); - - if ( recordLength < SSLv2_CLIENT_MASTER_KEY_HEADER_SIZE ) - { - Weird("SSLv2: CLIENT-MASTER-KEY is too small!"); - return ERROR_REQUIRED; - } - - // Extract the data of the client master key header. - SSLv2_ClientMasterKeyHeader cmk; - cmk.cipherKind = - ((recordData[1] << 16) | recordData[2] << 8) | recordData[3]; - cmk.clearKeyLength = uint16(recordData[4] << 8) | recordData[5]; - cmk.encryptedKeyLength = uint16(recordData[6] << 8) | recordData[7]; - cmk.keyArgLength = uint16(recordData[8] << 8) | recordData[9]; - - if ( cmk.clearKeyLength + cmk.encryptedKeyLength + cmk.keyArgLength + - SSLv2_CLIENT_MASTER_KEY_HEADER_SIZE != recordLength ) - { - Weird("SSLv2: Size inconsistency in CLIENT-MASTER-KEY"); - return ERROR_REQUIRED; - } - - // Check if cipher is supported by the server. - if ( pServerCipherSpecs ) - { - bool bFound = false; - for ( int i = 0; i < pServerCipherSpecs->len; i += 3 ) - { - uint32 cipherSpec = - ((pServerCipherSpecs->data[i] << 16) | - pServerCipherSpecs->data[i+1] << 8) | - pServerCipherSpecs->data[i+2]; - - if ( cmk.cipherKind == cipherSpec ) - { - bFound = true; - break; - } - } - - if ( ! bFound ) - { - Weird("SSLv2: Client chooses unadvertised cipher in CLIENT-MASTER-KEY!"); - nextState = ERROR_REQUIRED; - } - else - nextState = CLIENT_MASTERKEY_SEEN; - - delete pServerCipherSpecs; - pServerCipherSpecs = 0; - } - - // TODO: check if cipher has been advertised before. - - SSL_CipherSpec* pCipherSpecTemp = 0; - - HashKey h(static_cast(cmk.cipherKind)); - pCipherSpecTemp = (SSL_CipherSpec*) SSL_CipherSpecDict.Lookup(&h); - if ( ! pCipherSpecTemp || ! (pCipherSpecTemp->flags & SSL_FLAG_SSLv20) ) - Weird("SSLv2: Unknown CIPHER-SPEC in CLIENT-MASTER-KEY!"); - else - { // check for conistency of clearKeyLength - if ( cmk.clearKeyLength * 8 != pCipherSpecTemp->clearKeySize ) - { - Weird("SSLv2: Inconsistency of clearKeyLength in CLIENT-MASTER-KEY!"); - // nextState = ERROR_REQUIRED; - } - - // TODO: check for consistency of encryptedKeyLength. - // TODO: check for consistency of keyArgLength. -// switch ( cmk.cipherKind ) -// { -// case SSL_CK_RC4_128_WITH_MD5: -// case SSL_CK_RC4_128_EXPORT40_WITH_MD5: -// if ( cmk.keyArgLength != 0 ) -// { -// Weird("SSLv2: Inconsistency of keyArgLength in CLIENT-MASTER-KEY!"); -// //nextState = ERROR_REQUIRED; -// } -// break; -// case SSL_CK_DES_64_CBC_WITH_MD5: -// case SSL_CK_RC2_128_CBC_EXPORT40_WITH_MD5: -// case SSL_CK_RC2_128_CBC_WITH_MD5: -// case SSL_CK_IDEA_128_CBC_WITH_MD5: -// case SSL_CK_DES_192_EDE3_CBC_WITH_MD5: -// if ( cmk.keyArgLength != 8 ) -// { -// Weird("SSLv2: Inconsistency of keyArgLength in CLIENT-MASTER-KEY!"); -// } -// break; -// } - } - - // Remember the used cipher spec. - usedCipherSpec = SSLv2_CipherSpec(cmk.cipherKind); - - // If decrypting, store the clear key part of the master key. - if ( ssl_store_key_material /* && cmk.clearKeyLength == 11 */ ) - { - pMasterClearKey = - new SSL_DataBlock((recordData + SSLv2_CLIENT_MASTER_KEY_HEADER_SIZE), cmk.clearKeyLength); - - pMasterEncryptedKey = - new SSL_DataBlock((recordData + SSLv2_CLIENT_MASTER_KEY_HEADER_SIZE + cmk.clearKeyLength ), cmk.encryptedKeyLength); - } - - if ( nextState == CLIENT_MASTERKEY_SEEN ) - fire_ssl_conn_established(SSLProxy_Analyzer::SSLv20, - cmk.cipherKind); - - return nextState; - } - - -/*! - * This method analyses a SSLv2 ERROR record. - * - * \param s Pointer to the endpoint which sent the record - * \param length length of SSLv2 ERROR record - * \param data pointer to SSLv2 ERROR record to analyze - * - * \return the updated state of the current ssl connection - */ -SSLv2_States SSLv2_Interpreter::ErrorRecord(SSL_InterpreterEndpoint* s, - int recordLength, const u_char* recordData) - { - ++errorRecords; - - if ( unsigned(recordLength) != SSLv2_ERROR_RECORD_SIZE ) - { - Weird("SSLv2: Size mismatch in Error Record!"); - return ERROR_REQUIRED; - } - - SSLv2_ErrorRecord er; - er.errorCode = (recordData[1] << 8) | recordData[2]; - SSL3x_AlertLevel al = SSL3x_AlertLevel(255); - - switch ( er.errorCode ) { - case SSLv2_PE_NO_CIPHER: - // The client doesn't support a cipher which the server - // supports. Only from client to server and not recoverable! - al = SSL3x_ALERT_LEVEL_FATAL; - break; - - case SSLv2_PE_NO_CERTIFICATE: - if ( s == orig ) - // from client to server: not recoverable - al = SSL3x_ALERT_LEVEL_FATAL; - else - // from server to client: recoverable - al = SSL3x_ALERT_LEVEL_WARNING; - break; - - case SSLv2_PE_BAD_CERTIFICATE: - if ( s == orig ) - // from client to server: not recoverable - al = SSL3x_ALERT_LEVEL_FATAL; - else - // from server to client: recoverable - al = SSL3x_ALERT_LEVEL_WARNING; - break; - - case SSLv2_PE_UNSUPPORTED_CERTIFICATE_TYPE: - if ( s == orig ) - // from client to server: not recoverable - al = SSL3x_ALERT_LEVEL_FATAL; - else - // from server to client: recoverable - al = SSL3x_ALERT_LEVEL_WARNING; - break; - - default: - al = SSL3x_ALERT_LEVEL_FATAL; - break; - } - - fire_ssl_conn_alert(SSLProxy_Analyzer::SSLv20, al, er.errorCode); - - return ERROR_SEEN; - } - -/*! - * This method analyses a set of SSLv2 cipher suites. - * - * \param s Pointer to the endpoint which sent the cipher suites - * \param length length of cipher suites - * \param data pointer to cipher suites to analyze - * - * \return a pointer to a Bro TableVal (of type cipher_suites_list) which contains - * the cipher suites list of the current analyzed record - */ -TableVal* SSLv2_Interpreter::analyzeCiphers(SSL_InterpreterEndpoint* s, - int length, const u_char* data) - { - if ( length > MAX_CIPHERSPEC_SIZE ) - { - if ( s == orig ) - Weird("SSLv2: Client has CipherSpecs > MAX_CIPHERSPEC_SIZE"); - else - Weird("SSLv2: Server has CipherSpecs > MAX_CIPHERSPEC_SIZE"); - } - else - { // cipher specs are not too big - if ( ssl_compare_cipherspecs ) - { // store cipher specs for state analysis - if ( s == resp ) - pServerCipherSpecs = - new SSL_DataBlock(data, length); - else - pClientCipherSpecs = - new SSL_DataBlock(data, length); - } - } - - const u_char* pCipher = data; - bool bExtractCipherSuite = false; - TableVal* pCipherTable = 0; - - // We only extract the cipher suite when the corresponding - // ssl events are defined (otherwise we do work for nothing - // and suffer a memory leak). - // FIXME: This check needs to be done only once! - if ( (s == orig && ssl_conn_attempt) || - (s == resp && ssl_conn_server_reply) ) - { - pCipherTable = new TableVal(cipher_suites_list); - bExtractCipherSuite = true; - } - - for ( int i = 0; i < length; i += 3 ) - { - SSL_CipherSpec* pCurrentCipherSpec; - uint32 cipherSpecID = - ((pCipher[0] << 16) | pCipher[1] << 8) | pCipher[2]; - - // Check for unknown cipher specs. - HashKey h(static_cast(cipherSpecID)); - pCurrentCipherSpec = - (SSL_CipherSpec*) SSL_CipherSpecDict.Lookup(&h); - - if ( ! pCurrentCipherSpec ) - { - if ( s == orig ) - Weird("SSLv2: Unknown CIPHER-SPEC in CLIENT-HELLO!"); - else - Weird("SSLv2: Unknown CIPHER-SPEC in SERVER-HELLO!"); - } - - if ( bExtractCipherSuite ) - { - Val* index = new Val(cipherSpecID, TYPE_COUNT); - pCipherTable->Assign(index, 0); - Unref(index); - } - - pCipher += 3; - } - - return pCipherTable; - } - -// --- SSLv2_EndPoint --------------------------------------------------------- - -/*! - * The constructor. - * - * \param interpreter Pointer to the SSLv2 interpreter to whom this endpoint belongs to - * \param is_orig true if this is the originating endpoint of the ssl connection, - * false otherwise - */ -SSLv2_Endpoint::SSLv2_Endpoint(SSLv2_Interpreter* interpreter, int is_orig) -: SSL_InterpreterEndpoint(interpreter, is_orig) - { - sentRecords = 0; - } - -/*! - * The destructor. - */ -SSLv2_Endpoint::~SSLv2_Endpoint() - { - } - -/*! - * This method is called by the SSLProxy_Analyzer with a complete reassembled - * SSLv2 record. It passes the record to SSLv2_Interpreter::NewSSLRecord(). - * - * \param t reserved (always zero) - * \param seq reserved (always zero) - * \param len length of the data block containing the ssl record - * \param data pointer to the data block containing the ssl record - */ -void SSLv2_Endpoint::Deliver(int len, const u_char* data) - { - ++((SSLv2_Endpoint*)peer)->sentRecords; - - ((SSLv2_Interpreter*)interpreter)->NewSSLRecord(this, len, data); - } From ba30713ec460f26dd8461686c80f7ac3f8e8e02f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 4 Jan 2012 12:30:54 -0800 Subject: [PATCH 222/964] Adding SWIG as dependency to INSTALL. And a bit of polishing. --- INSTALL | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/INSTALL b/INSTALL index 97b2f4b0b9..deb71c7b84 100644 --- a/INSTALL +++ b/INSTALL @@ -14,10 +14,11 @@ before you begin: * OpenSSL (headers and libraries) http://www.openssl.org - * Libmagic For identifying file types (e.g., in FTP transfers). + * SWIG http://www.swig.org - * Libz For decompressing HTTP bodies by the HTTP analyzer, and for - compressed Bro-to-Bro communication. + * Libmagic + + * Libz Bro can make uses of some optional libraries if they are found at installation time: @@ -32,6 +33,7 @@ already come preinstalled: * Flex (Fast Lexical Analyzer) * Perl (Used only during the Bro build process) + Installation ============ @@ -69,8 +71,10 @@ Running Bro =========== Bro is a complex program and it takes a bit of time to get familiar -with it. A good place for newcomers to start is the -:doc:`quick start guide `. +with it. A good place for newcomers to start is the Quickstart Guide at + + http://www.bro-ids.org/documentation/quickstart.bro.html + For developers that wish to run Bro directly from the ``build/`` directory (i.e., without performing ``make install``), they will have From aa4cc3c4a1d53d342affb625f457dbdb1f6cd025 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 4 Jan 2012 12:34:36 -0800 Subject: [PATCH 223/964] Adding top-level "test" target to Makefile. --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 277be4d017..4ec6d04a95 100644 --- a/Makefile +++ b/Makefile @@ -60,6 +60,9 @@ bindist: distclean: rm -rf $(BUILD) +test: + @(cd testing && make ) + configured: @test -d $(BUILD) || ( echo "Error: No build/ directory found. Did you run configure?" && exit 1 ) @test -e $(BUILD)/Makefile || ( echo "Error: No build/Makefile found. Did you run configure?" && exit 1 ) From 645c80f9742049e45cab9f7349ed34a1ddb722d8 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 4 Jan 2012 16:30:15 -0600 Subject: [PATCH 224/964] Reduce snaplen default from 65535 to old default of 8192. (fixes #720) Also replaced the --snaplen/-l command line option with a scripting-layer option called "snaplen" (which can also be redefined on the command line, e.g. `bro -i eth0 snaplen=65535`). --- scripts/base/init-bare.bro | 3 +++ src/main.cc | 10 +++------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 859a69f2dc..5e334496c6 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -1505,6 +1505,9 @@ const skip_http_data = F &redef; ## UDP tunnels. See also: udp_tunnel_port, policy/udp-tunnel.bro. const parse_udp_tunnels = F &redef; +## Number of bytes per packet to capture from live interfaces. +const snaplen = 8192 &redef; + # Load the logging framework here because it uses fairly deep integration with # BiFs and script-land defined types. @load base/frameworks/logging diff --git a/src/main.cc b/src/main.cc index b4a27862c9..3d096c7d51 100644 --- a/src/main.cc +++ b/src/main.cc @@ -97,7 +97,7 @@ extern char version[]; char* command_line_policy = 0; vector params; char* proc_status_file = 0; -int snaplen = 65535; // really want "capture entire packet" +int snaplen = 0; // this gets set from the scripting-layer's value int FLAGS_use_binpac = false; @@ -145,7 +145,6 @@ void usage() fprintf(stderr, " -g|--dump-config | dump current config into .state dir\n"); fprintf(stderr, " -h|--help|-? | command line help\n"); fprintf(stderr, " -i|--iface | read from given interface\n"); - fprintf(stderr, " -l|--snaplen | number of bytes per packet to capture from interfaces (default 65535)\n"); fprintf(stderr, " -p|--prefix | add given prefix to policy file resolution\n"); fprintf(stderr, " -r|--readfile | read from given tcpdump file\n"); fprintf(stderr, " -y|--flowfile [=] | read from given flow file\n"); @@ -372,7 +371,6 @@ int main(int argc, char** argv) {"filter", required_argument, 0, 'f'}, {"help", no_argument, 0, 'h'}, {"iface", required_argument, 0, 'i'}, - {"snaplen", required_argument, 0, 'l'}, {"doc-scripts", no_argument, 0, 'Z'}, {"prefix", required_argument, 0, 'p'}, {"readfile", required_argument, 0, 'r'}, @@ -481,10 +479,6 @@ int main(int argc, char** argv) interfaces.append(optarg); break; - case 'l': - snaplen = atoi(optarg); - break; - case 'p': prefixes.append(optarg); break; @@ -833,6 +827,8 @@ int main(int argc, char** argv) } } + snaplen = internal_val("snaplen")->AsCount(); + // Initialize the secondary path, if it's needed. secondary_path = new SecondaryPath(); From 7d85308b761db2fe61289532ef2fd60d33d91a1b Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 4 Jan 2012 16:41:39 -0600 Subject: [PATCH 225/964] Remove upgrade documentation regarding change in default snaplen. --- doc/upgrade.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/upgrade.rst b/doc/upgrade.rst index 71cc5e401d..885f8f8b8c 100644 --- a/doc/upgrade.rst +++ b/doc/upgrade.rst @@ -168,10 +168,6 @@ New Default Settings are loaded. See ``PacketFilter::all_packets`` for how to revert to old behavior. -- By default, Bro now sets a libpcap snaplen of 65535. Depending on - the OS, this may have performance implications and you can use the - ``--snaplen`` option to change the value. - API Changes ----------- From aae60a6d765da39a9659a9b9174d2cb3ebaba7fe Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 4 Jan 2012 16:44:25 -0600 Subject: [PATCH 226/964] Allow local table variables to be initialized with {} list expressions. --- src/Expr.cc | 19 +++++++++++++++--- src/Expr.h | 4 ++-- src/Var.cc | 3 ++- .../btest/Baseline/language.table-init/output | 10 ++++++++++ testing/btest/language/table-init.bro | 20 +++++++++++++++++++ 5 files changed, 50 insertions(+), 6 deletions(-) create mode 100644 testing/btest/Baseline/language.table-init/output create mode 100644 testing/btest/language/table-init.bro diff --git a/src/Expr.cc b/src/Expr.cc index c34c44a7d1..ee2d90aeed 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -2435,7 +2435,7 @@ bool RefExpr::DoUnserialize(UnserialInfo* info) } AssignExpr::AssignExpr(Expr* arg_op1, Expr* arg_op2, int arg_is_init, - Val* arg_val) + Val* arg_val, attr_list* arg_attrs) : BinaryExpr(EXPR_ASSIGN, arg_is_init ? arg_op1 : arg_op1->MakeLvalue(), arg_op2) { @@ -2455,14 +2455,14 @@ AssignExpr::AssignExpr(Expr* arg_op1, Expr* arg_op2, int arg_is_init, // We discard the status from TypeCheck since it has already // generated error messages. - (void) TypeCheck(); + (void) TypeCheck(arg_attrs); val = arg_val ? arg_val->Ref() : 0; SetLocationInfo(arg_op1->GetLocationInfo(), arg_op2->GetLocationInfo()); } -bool AssignExpr::TypeCheck() +bool AssignExpr::TypeCheck(attr_list* attrs) { TypeTag bt1 = op1->Type()->Tag(); TypeTag bt2 = op2->Type()->Tag(); @@ -2494,6 +2494,19 @@ bool AssignExpr::TypeCheck() return true; } + if ( bt1 == TYPE_TABLE && op2->Tag() == EXPR_LIST ) + { + attr_list* attr_copy = 0; + if ( attrs ) + { + attr_copy = new attr_list; + loop_over_list(*attrs, i) + attr_copy->append((*attrs)[i]); + } + op2 = new TableConstructorExpr(op2->AsListExpr(), attr_copy); + return true; + } + if ( bt1 == TYPE_VECTOR && bt2 == bt1 && op2->Type()->AsVectorType()->IsUnspecifiedVector() ) { diff --git a/src/Expr.h b/src/Expr.h index 95016a8d13..8676a1ad7e 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -623,7 +623,7 @@ class AssignExpr : public BinaryExpr { public: // If val is given, evaluating this expression will always yield the val // yet still perform the assignment. Used for triggers. - AssignExpr(Expr* op1, Expr* op2, int is_init, Val* val = 0); + AssignExpr(Expr* op1, Expr* op2, int is_init, Val* val = 0, attr_list* attrs = 0); virtual ~AssignExpr() { Unref(val); } Expr* Simplify(SimplifyType simp_type); @@ -638,7 +638,7 @@ protected: friend class Expr; AssignExpr() { } - bool TypeCheck(); + bool TypeCheck(attr_list* attrs = 0); bool TypeCheckArithmetics(TypeTag bt1, TypeTag bt2); DECLARE_SERIAL(AssignExpr); diff --git a/src/Var.cc b/src/Var.cc index 897a454670..d54d94a078 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -202,7 +202,8 @@ Stmt* add_local(ID* id, BroType* t, init_class c, Expr* init, Ref(id); Stmt* stmt = - new ExprStmt(new AssignExpr(new NameExpr(id), init, 0)); + new ExprStmt(new AssignExpr(new NameExpr(id), init, 0, 0, + id->Attrs() ? id->Attrs()->Attrs() : 0 )); stmt->SetLocationInfo(init->GetLocationInfo()); return stmt; diff --git a/testing/btest/Baseline/language.table-init/output b/testing/btest/Baseline/language.table-init/output new file mode 100644 index 0000000000..0272e12319 --- /dev/null +++ b/testing/btest/Baseline/language.table-init/output @@ -0,0 +1,10 @@ +{ +[2] = two, +[1] = one +} +global table default +{ +[4] = four, +[3] = three +} +local table default diff --git a/testing/btest/language/table-init.bro b/testing/btest/language/table-init.bro new file mode 100644 index 0000000000..5df682c5d2 --- /dev/null +++ b/testing/btest/language/table-init.bro @@ -0,0 +1,20 @@ +# @TEST-EXEC: bro %INPUT >output +# @TEST-EXEC: btest-diff output + +global global_table: table[count] of string = { + [1] = "one", + [2] = "two" +} &default = "global table default"; + +event bro_init() + { + local local_table: table[count] of string = { + [3] = "three", + [4] = "four" + } &default = "local table default"; + + print global_table; + print global_table[0]; + print local_table; + print local_table[0]; + } From e48f62622ce8a4399f8569fa89b2e8897b98d35a Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 5 Jan 2012 15:22:25 -0800 Subject: [PATCH 227/964] Addign a file NEWS with release notes. Using that to capture the most important changes between 2.0 Beta and 2.0 Final. --- NEWS | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ README | 10 ++++++---- 2 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 NEWS diff --git a/NEWS b/NEWS new file mode 100644 index 0000000000..aca9a820b6 --- /dev/null +++ b/NEWS @@ -0,0 +1,51 @@ + +Release Notes +============= + +This document summarizes the most important changes in the current Bro +release. For a complete list of changes, see the ``CHANGES`` file. + +Bro 2.0 +------- + +As the version number jump suggests, Bro 2.0 is a major upgrade and +lots of things have changed. We have assembled a separate upprade +guide with the most important changes compared to Bro 1.5 at +http://www.bro-ids.org/documentation/upgrade.bro.html. You can find +the offline version of that document in ``doc/upgrade.rst.``. + +Compared to the earlier 2.0 Beta version, the major changes in the +final release are: + + * The default scripts now come with complete reference + documentation. See + http://www.bro-ids.org/documentation/index.html. + + * libz and libmagic are now required dependencies. + + * Reduced snaplen default from 65535 to old default of 8192. The + large value was introducing performance problems on many + systems. + + * Replaced the --snaplen/-l command line option with a + scripting-layer option called "snaplen". The new option can also + be redefined on the command line, e.g. ``bro -i eth0 + snaplen=65535``. + + * Reintroduced the BRO_LOG_SUFFIX environment that the ASCII + logger now respects to add a suffix to the log files it creates. + + * The ASCII logs now include further header information, and + fields set to an empty value are now logged as ``(empty)`` by + default (instead of ``-``, which is already used for fields that + are not set at all). + + * Some NOTICES were renamed, and the signatures of some SSL events + have changed. + + * Many smaller bug fixes, portability improvements, and general + polishing. + + + + diff --git a/README b/README index 435e60225a..37cee2f333 100644 --- a/README +++ b/README @@ -4,13 +4,15 @@ Bro Network Security Monitor Bro is a powerful framework for network analysis and security monitoring. Please see the INSTALL file for installation instructions -and pointers for getting started. For more documentation, research -publications, and community contact information, see Bro's home page: +and pointers for getting started. NEWS contains releases notes for the +current version, and CHANGES has the complete history of changes. +Please see COPYING for licensing information. + +For more documentation, research publications, and community contact +information, please see Bro's home page: http://www.bro-ids.org -Please see COPYING for licensing information. - On behalf of the Bro Development Team, Vern Paxson & Robin Sommer, From d661ccfc8e88dc544c2038800934f510c5d471dc Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 6 Jan 2012 11:39:38 -0600 Subject: [PATCH 228/964] Tweaks for OpenBSD support. - Add note about top-level Makefile possibly lacking OpenBSD support. - Extend config.h with missing/different preprocessor definitions --- INSTALL | 6 ++++++ config.h.in | 11 +++++++++-- src/ARP.h | 2 ++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/INSTALL b/INSTALL index deb71c7b84..0275837caa 100644 --- a/INSTALL +++ b/INSTALL @@ -48,6 +48,12 @@ called ``build/``, using default build options. It then installs all required files into ``/usr/local/bro``, including the Bro binary in ``/usr/local/bro/bin/bro``. +Note that the top-level Makefile may not work with OpenBSD's default +make program, in which case you can either install the ``gmake`` +package and use it instead or first change into the ``build/`` +directory before doing either ``make`` or ``make install`` such that +the CMake-generated Makefile's are used directly. + You can specify a different installation directory with:: ./configure --prefix= diff --git a/config.h.in b/config.h.in index e3c9984373..261cd0ccb9 100644 --- a/config.h.in +++ b/config.h.in @@ -32,8 +32,8 @@ /* Define if you have the header file. */ #cmakedefine HAVE_NET_ETHERNET_H -/* We are on a OpenBSD system */ -#cmakedefine HAVE_OPENBSD +/* Define if you have the header file. */ +#cmakedefine HAVE_NET_ETHERTYPES_H /* have os-proto.h */ #cmakedefine HAVE_OS_PROTO_H @@ -145,3 +145,10 @@ /* Define u_int8_t */ #cmakedefine u_int8_t @u_int8_t@ + +/* OpenBSD's bpf.h may not declare this data link type, but it's supposed to be + used consistently for the same purpose on all platforms. */ +#cmakedefine HAVE_DLT_PPP_SERIAL +#ifndef HAVE_DLT_PPP_SERIAL +#define DLT_PPP_SERIAL @DLT_PPP_SERIAL@ +#endif diff --git a/src/ARP.h b/src/ARP.h index 6b84dbd587..37f20ced3c 100644 --- a/src/ARP.h +++ b/src/ARP.h @@ -15,6 +15,8 @@ #include #elif defined(HAVE_NETINET_IF_ETHER_H) #include +#elif defined(HAVE_NET_ETHERTYPES_H) +#include #endif #ifndef arp_pkthdr From e83df9487a828b4bd5f1567e308ecab89c242bf6 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 6 Jan 2012 13:10:07 -0600 Subject: [PATCH 229/964] Add FAQ entry about disabling NIC offloading features. --- doc/faq.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/doc/faq.rst b/doc/faq.rst index bdb1f50292..510d03c5af 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -28,6 +28,23 @@ Here are some pointers to more information: Lothar Braun et. al evaluates packet capture performance on commodity hardware +Are there any gotchas regarding interface configuration for live capture? Or why might I be seeing abnormally large packets much greater than interface MTU? +------------------------------------------------------------------------------------------------------------------------------------------------------------- + +Some NICs offload the reassembly of traffic into "superpackets" so that +fewer packets are then passed up the stack (e.g. "TCP segmentation +offload", or "generic segmentation offload"). The result is that the +capturing application will observe packets much larger than the MTU size +of the interface they were captured from and may also interfere with the +maximum packet capture length, ``snaplen``, so it's a good idea to disable +an interface's offloading features. + +You can use the ``ethtool`` program on Linux to view and disable +offloading features of an interface. See this page for more explicit +directions: + +http://securityonion.blogspot.com/2011/10/when-is-full-packet-capture-not-full.html + What does an error message like ``internal error: NB-DNS error`` mean? --------------------------------------------------------------------------------------------------------------------------------- From e7cf347288e4318c4629baf9f2f9ff44b8eef333 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 6 Jan 2012 14:56:11 -0600 Subject: [PATCH 230/964] Add SFTP log postprocessor that transfers logs to remote hosts. Addresses #737 --- .../logging/postprocessors/__load__.bro | 1 + .../logging/postprocessors/sftp.bro | 65 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 scripts/base/frameworks/logging/postprocessors/sftp.bro diff --git a/scripts/base/frameworks/logging/postprocessors/__load__.bro b/scripts/base/frameworks/logging/postprocessors/__load__.bro index c5d92cfb4b..830a69aa75 100644 --- a/scripts/base/frameworks/logging/postprocessors/__load__.bro +++ b/scripts/base/frameworks/logging/postprocessors/__load__.bro @@ -1 +1,2 @@ @load ./scp +@load ./sftp diff --git a/scripts/base/frameworks/logging/postprocessors/sftp.bro b/scripts/base/frameworks/logging/postprocessors/sftp.bro new file mode 100644 index 0000000000..c0423bb1c4 --- /dev/null +++ b/scripts/base/frameworks/logging/postprocessors/sftp.bro @@ -0,0 +1,65 @@ +##! This script defines a postprocessing function that can be applied +##! to a logging filter in order to automatically SFTP +##! a log stream (or a subset of it) to a remote host at configurable +##! rotation time intervals. Generally, to use this functionality +##! you must handle the :bro:id:`bro_init` event and do the following +##! in your handler: +##! +##! 1) Create a new :bro:type:`Log::Filter` record that defines a name/path, +##! rotation interval, and set the ``postprocessor`` to +##! :bro:id:`Log::sftp_postprocessor`. +##! 2) Add the filter to a logging stream using :bro:id:`Log::add_filter`. +##! 3) Add a table entry to :bro:id:`Log::sftp_destinations` for the filter's +##! writer/path pair which defines a set of :bro:type:`Log::SFTPDestination` +##! records. + +module Log; + +export { + ## Securely transfers the rotated-log to all the remote hosts + ## defined in :bro:id:`Log::sftp_destinations` and then deletes + ## the local copy of the rotated-log. It's not active when + ## reading from trace files. + ## + ## info: A record holding meta-information about the log file to be + ## postprocessed. + ## + ## Returns: True if sftp system command was initiated or + ## if no destination was configured for the log as described + ## by *info*. + global sftp_postprocessor: function(info: Log::RotationInfo): bool; + + ## A container that describes the remote destination for the SFTP command, + ## comprised of the username, host, and path at which to upload the file. + type SFTPDestination: record { + ## The remote user to log in as. A trust mechanism should be + ## pre-established. + user: string; + ## The remote host to which to transfer logs. + host: string; + ## The path/directory on the remote host to send logs. + path: string; + }; + + ## A table indexed by a particular log writer and filter path, that yields + ## a set remote destinations. The :bro:id:`Log::sftp_postprocessor` + ## function queries this table upon log rotation and performs a secure + ## transfer of the rotated-log to each destination in the set. This + ## table can be modified at run-time. + global sftp_destinations: table[Writer, string] of set[SFTPDestination]; +} + +function sftp_postprocessor(info: Log::RotationInfo): bool + { + if ( reading_traces() || [info$writer, info$path] !in sftp_destinations ) + return T; + + local command = ""; + for ( d in sftp_destinations[info$writer, info$path] ) + command += fmt("echo put %s %s | sftp -b - %s@%s;", info$fname, d$path, + d$user, d$host); + + command += fmt("/bin/rm %s", info$fname); + system(command); + return T; + } From f603d0121b7052ac20810866761852bf360cac58 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Fri, 6 Jan 2012 16:36:22 -0500 Subject: [PATCH 231/964] This completes framework documentation package 4. - Closes ticket #709 --- scripts/base/frameworks/control/main.bro | 40 +++------ scripts/base/frameworks/intel/main.bro | 90 ++++++++++++++----- scripts/base/frameworks/metrics/cluster.bro | 10 +-- scripts/base/frameworks/metrics/main.bro | 64 ++++++++++--- scripts/base/frameworks/software/main.bro | 76 +++++++++------- .../policy/frameworks/control/controllee.bro | 9 ++ .../policy/frameworks/control/controller.bro | 7 ++ .../frameworks/metrics/conn-example.bro | 3 + .../frameworks/metrics/http-example.bro | 20 +++-- .../policy/frameworks/metrics/ssl-example.bro | 5 ++ .../frameworks/software/version-changes.bro | 17 ++-- .../policy/frameworks/software/vulnerable.bro | 8 +- 12 files changed, 238 insertions(+), 111 deletions(-) diff --git a/scripts/base/frameworks/control/main.bro b/scripts/base/frameworks/control/main.bro index 497936c8f1..4fe8872801 100644 --- a/scripts/base/frameworks/control/main.bro +++ b/scripts/base/frameworks/control/main.bro @@ -1,43 +1,30 @@ -##! This is a utility script that sends the current values of all &redef'able -##! consts to a remote Bro then sends the -##! :bro:id:`Control::configuration_update` event and terminates processing. -##! -##! Intended to be used from the command line like this when starting a controller:: -##! -##! bro frameworks/control/controller Control::host= Control::port= Control::cmd= [Control::arg=] -##! -##! A controllee only needs to load the controllee script in addition -##! to the specific analysis scripts desired. It may also need a node -##! configured as a controller node in the communications nodes configuration:: -##! -##! bro frameworks/control/controllee -##! -##! To use the framework as a controllee, it only needs to be loaded and -##! the controlled node need to accept all events in the "Control::" namespace -##! from the host where the control actions will be performed from along with -##! using the "control" class. +##! The control framework provides the foundation for providing "commands" +##! that can be taken remotely at runtime to modify a running Bro instance +##! or collect information from the running instance. module Control; export { - ## This is the address of the host that will be controlled. + ## The address of the host that will be controlled. const host = 0.0.0.0 &redef; - ## This is the port of the host that will be controlled. + ## The port of the host that will be controlled. const host_port = 0/tcp &redef; - ## This is the command that is being done. It's typically set on the - ## command line and influences whether this instance starts up as a - ## controller or controllee. + ## The command that is being done. It's typically set on the + ## command line. const cmd = "" &redef; ## This can be used by commands that take an argument. const arg = "" &redef; + ## Events that need to be handled by controllers. const controller_events = /Control::.*_request/ &redef; + + ## Events that need to be handled by controllees. const controllee_events = /Control::.*_response/ &redef; - ## These are the commands that can be given on the command line for + ## The commands that can currently be given on the command line for ## remote control. const commands: set[string] = { "id_value", @@ -45,11 +32,10 @@ export { "net_stats", "configuration_update", "shutdown", - }; + } &redef; ## Variable IDs that are to be ignored by the update process. - const ignore_ids: set[string] = { - }; + const ignore_ids: set[string] = { }; ## Event for requesting the value of an ID (a variable). global id_value_request: event(id: string); diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro index 6ee4435831..9ee1c75100 100644 --- a/scripts/base/frameworks/intel/main.bro +++ b/scripts/base/frameworks/intel/main.bro @@ -11,7 +11,7 @@ # user_name # file_name # file_md5 -# x509_cert - DER encoded, not PEM (ascii armored) +# x509_md5 # Example tags: # infrastructure @@ -25,6 +25,7 @@ module Intel; export { + ## The intel logging stream identifier. redef enum Log::ID += { LOG }; redef enum Notice::Type += { @@ -33,72 +34,117 @@ export { Detection, }; + ## Record type used for logging information from the intelligence framework. + ## Primarily for problems or oddities with inserting and querying data. + ## This is important since the content of the intelligence framework can + ## change quite dramatically during runtime and problems may be introduced + ## into the data. type Info: record { + ## The current network time. ts: time &log; + ## Represents the severity of the message. ## This value should be one of: "info", "warn", "error" level: string &log; + ## The message. message: string &log; }; + ## Record to represent metadata associated with a single piece of + ## intelligence. type MetaData: record { + ## A description for the data. desc: string &optional; + ## A URL where more information may be found about the intelligence. url: string &optional; + ## The time at which the data was first declared to be intelligence. first_seen: time &optional; + ## When this data was most recent inserted into the framework. latest_seen: time &optional; + ## Arbitrary text tags for the data. tags: set[string]; }; + ## Record to represent a singular piece of intelligence. type Item: record { + ## If the data is an IP address, this hold the address. ip: addr &optional; + ## If the data is textual, this holds the text. str: string &optional; + ## If the data is numeric, this holds the number. num: int &optional; + ## The subtype of the data for when either the $str or $num fields are + ## given. If one of those fields are given, this field must be present. subtype: string &optional; + ## The next five fields are temporary until a better model for + ## attaching metadata to an intelligence item is created. desc: string &optional; url: string &optional; first_seen: time &optional; latest_seen: time &optional; tags: set[string]; - ## These single string tags are throw away until pybroccoli supports sets + ## These single string tags are throw away until pybroccoli supports sets. tag1: string &optional; tag2: string &optional; tag3: string &optional; }; + ## Record model used for constructing queries against the intelligence + ## framework. type QueryItem: record { - ip: addr &optional; - str: string &optional; - num: int &optional; - subtype: string &optional; + ## If an IP address is being queried for, this field should be given. + ip: addr &optional; + ## If a string is being queried for, this field should be given. + str: string &optional; + ## If numeric data is being queried for, this field should be given. + num: int &optional; + ## If either a string or number is being queried for, this field should + ## indicate the subtype of the data. + subtype: string &optional; - or_tags: set[string] &optional; - and_tags: set[string] &optional; + ## A set of tags where if a single metadata record attached to an item + ## has any one of the tags defined in this field, it will match. + or_tags: set[string] &optional; + ## A set of tags where a single metadata record attached to an item + ## must have all of the tags defined in this field. + and_tags: set[string] &optional; ## The predicate can be given when searching for a match. It will ## be tested against every :bro:type:`Intel::MetaData` item associated ## with the data being matched on. If it returns T a single time, the - ## matcher will consider that the item has matched. - pred: function(meta: Intel::MetaData): bool &optional; + ## matcher will consider that the item has matched. This field can + ## be used for constructing arbitrarily complex queries that may not + ## be possible with the $or_tags or $and_tags fields. + pred: function(meta: Intel::MetaData): bool &optional; }; - + ## Function to insert data into the intelligence framework. + ## + ## item: The data item. + ## + ## Returns: T if the data was successfully inserted into the framework, + ## otherwise it returns F. global insert: function(item: Item): bool; + + ## A wrapper for the :bro:id:`Intel::insert` function. This is primarily + ## used as the external API for inserting data into the intelligence + ## using Broccoli. global insert_event: event(item: Item); + + ## Function for matching data within the intelligence framework. global matcher: function(item: QueryItem): bool; - - type MetaDataStore: table[count] of MetaData; - type DataStore: record { - ip_data: table[addr] of MetaDataStore; - ## The first string is the actual value and the second string is the subtype. - string_data: table[string, string] of MetaDataStore; - int_data: table[int, string] of MetaDataStore; - }; - global data_store: DataStore; - - } +type MetaDataStore: table[count] of MetaData; +type DataStore: record { + ip_data: table[addr] of MetaDataStore; + # The first string is the actual value and the second string is the subtype. + string_data: table[string, string] of MetaDataStore; + int_data: table[int, string] of MetaDataStore; +}; +global data_store: DataStore; + event bro_init() { Log::create_stream(Intel::LOG, [$columns=Info]); diff --git a/scripts/base/frameworks/metrics/cluster.bro b/scripts/base/frameworks/metrics/cluster.bro index a3a2b8b53c..4804bc5005 100644 --- a/scripts/base/frameworks/metrics/cluster.bro +++ b/scripts/base/frameworks/metrics/cluster.bro @@ -13,11 +13,11 @@ module Metrics; export { - ## This value allows a user to decide how large of result groups the - ## workers should transmit values. + ## Allows a user to decide how large of result groups the + ## workers should transmit values for cluster metric aggregation. const cluster_send_in_groups_of = 50 &redef; - ## This is the percent of the full threshold value that needs to be met + ## The percent of the full threshold value that needs to be met ## on a single worker for that worker to send the value to its manager in ## order for it to request a global view for that value. There is no ## requirement that the manager requests a global view for the index @@ -25,11 +25,11 @@ export { ## recently. const cluster_request_global_view_percent = 0.1 &redef; - ## This event is sent by the manager in a cluster to initiate the + ## Event sent by the manager in a cluster to initiate the ## collection of metrics values for a filter. global cluster_filter_request: event(uid: string, id: ID, filter_name: string); - ## This event is sent by nodes that are collecting metrics after receiving + ## Event sent by nodes that are collecting metrics after receiving ## a request for the metric filter from the manager. global cluster_filter_response: event(uid: string, id: ID, filter_name: string, data: MetricTable, done: bool); diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro index c30807e179..5710de60a1 100644 --- a/scripts/base/frameworks/metrics/main.bro +++ b/scripts/base/frameworks/metrics/main.bro @@ -1,13 +1,16 @@ -##! This is the implementation of the metrics framework. +##! The metrics framework provides a way to count and measure data. @load base/frameworks/notice module Metrics; export { + ## The metrics logging stream identifier. redef enum Log::ID += { LOG }; + ## Identifiers for metrics to collect. type ID: enum { + ## Blank placeholder value. NOTHING, }; @@ -15,10 +18,13 @@ export { ## current value to the logging stream. const default_break_interval = 15mins &redef; - ## This is the interval for how often notices will happen after they have - ## already fired. + ## This is the interval for how often threshold based notices will happen + ## after they have already fired. const renotice_interval = 1hr &redef; + ## Represents a thing which is having metrics collected for it. An instance + ## of this record type and a :bro:type:`Metrics::ID` together represent a + ## single measurement. type Index: record { ## Host is the value to which this metric applies. host: addr &optional; @@ -37,17 +43,30 @@ export { network: subnet &optional; } &log; + ## The record type that is used for logging metrics. type Info: record { + ## Timestamp at which the metric was "broken". ts: time &log; + ## What measurement the metric represents. metric_id: ID &log; + ## The name of the filter being logged. :bro:type:`Metrics::ID` values + ## can have multiple filters which represent different perspectives on + ## the data so this is necessary to understand the value. filter_name: string &log; + ## What the metric value applies to. index: Index &log; + ## The simple numeric value of the metric. value: count &log; }; - # TODO: configure a metrics filter logging stream to log the current + # TODO: configure a metrics filter logging stream to log the current # metrics configuration in case someone is looking through # old logs and the configuration has changed since then. + + ## Filters define how the data from a metric is aggregated and handled. + ## Filters can be used to set how often the measurements are cut or "broken" + ## and logged or how the data within them is aggregated. It's also + ## possible to disable logging and use filters for thresholding. type Filter: record { ## The :bro:type:`Metrics::ID` that this filter applies to. id: ID &optional; @@ -62,7 +81,7 @@ export { aggregation_mask: count &optional; ## This is essentially a mapping table between addresses and subnets. aggregation_table: table[subnet] of subnet &optional; - ## The interval at which the metric should be "broken" and written + ## The interval at which this filter should be "broken" and written ## to the logging stream. The counters are also reset to zero at ## this time so any threshold based detection needs to be set to a ## number that should be expected to happen within this period. @@ -79,24 +98,47 @@ export { notice_threshold: count &optional; ## A series of thresholds at which to generate notices. notice_thresholds: vector of count &optional; - ## How often this notice should be raised for this metric index. It + ## How often this notice should be raised for this filter. It ## will be generated everytime it crosses a threshold, but if the ## $break_interval is set to 5mins and this is set to 1hr the notice ## only be generated once per hour even if something crosses the ## threshold in every break interval. notice_freq: interval &optional; }; - - type MetricTable: table[Index] of count &default=0; + ## Function to associate a metric filter with a metric ID. + ## + ## id: The metric ID that the filter should be associated with. + ## + ## filter: The record representing the filter configuration. global add_filter: function(id: ID, filter: Filter); + + ## Add data into a :bro:type:`Metrics::ID`. This should be called when + ## a script has measured some point value and is ready to increment the + ## counters. + ## + ## id: The metric ID that the data represents. + ## + ## index: The metric index that the value is to be added to. + ## + ## increment: How much to increment the counter by. global add_data: function(id: ID, index: Index, increment: count); + + ## Helper function to represent a :bro:type:`Metrics::Index` value as + ## a simple string + ## + ## index: The metric index that is to be converted into a string. + ## + ## Returns: A string reprentation of the metric index. global index2str: function(index: Index): string; - # This is the event that is used to "finish" metrics and adapt the metrics - # framework for clustered or non-clustered usage. + ## Event that is used to "finish" metrics and adapt the metrics + ## framework for clustered or non-clustered usage. + ## + ## ..note: This is primarily intended for internal use. global log_it: event(filter: Filter); + ## Event to access metrics records as they are passed to the logging framework. global log_metrics: event(rec: Info); } @@ -107,6 +149,8 @@ redef record Notice::Info += { global metric_filters: table[ID] of vector of Filter = table(); global filter_store: table[ID, string] of Filter = table(); +type MetricTable: table[Index] of count &default=0; + # This is indexed by metric ID and stream filter name. global store: table[ID, string] of MetricTable = table() &default=table(); diff --git a/scripts/base/frameworks/software/main.bro b/scripts/base/frameworks/software/main.bro index e244915077..f5bbf00a2f 100644 --- a/scripts/base/frameworks/software/main.bro +++ b/scripts/base/frameworks/software/main.bro @@ -1,5 +1,5 @@ ##! This script provides the framework for software version detection and -##! parsing, but doesn't actually do any detection on it's own. It relys on +##! parsing but doesn't actually do any detection on it's own. It relys on ##! other protocol specific scripts to parse out software from the protocols ##! that they analyze. The entry point for providing new software detections ##! to this framework is through the :bro:id:`Software::found` function. @@ -10,39 +10,44 @@ module Software; export { - + ## The software logging stream identifier. redef enum Log::ID += { LOG }; - + + ## Scripts detecting new types of software need to redef this enum to add + ## their own specific software types which would then be used when they + ## create :bro:type:`Software::Info` records. type Type: enum { + ## A placeholder type for when the type of software is not known. UNKNOWN, - OPERATING_SYSTEM, - DATABASE_SERVER, - # There are a number of ways to detect printers on the - # network, we just need to codify them in a script and move - # this out of here. It isn't currently used for anything. - PRINTER, }; - + + ## A structure to represent the numeric version of software. type Version: record { - major: count &optional; ##< Major version number - minor: count &optional; ##< Minor version number - minor2: count &optional; ##< Minor subversion number - addl: string &optional; ##< Additional version string (e.g. "beta42") + ## Major version number + major: count &optional; + ## Minor version number + minor: count &optional; + ## Minor subversion number + minor2: count &optional; + ## Additional version string (e.g. "beta42") + addl: string &optional; } &log; - + + ## The record type that is used for representing and logging software. type Info: record { - ## The time at which the software was first detected. + ## The time at which the software was detected. ts: time &log; ## The IP address detected running the software. host: addr &log; - ## The type of software detected (e.g. WEB_SERVER) + ## The type of software detected (e.g. :bro:enum:`HTTP::SERVER`). software_type: Type &log &default=UNKNOWN; - ## Name of the software (e.g. Apache) + ## Name of the software (e.g. Apache). name: string &log; - ## Version of the software + ## Version of the software. version: Version &log; ## The full unparsed version string found because the version parsing - ## doesn't work 100% reliably and this acts as a fall back in the logs. + ## doesn't always work reliably in all cases and this acts as a + ## fallback in the logs. unparsed_version: string &log &optional; ## This can indicate that this software being detected should @@ -55,37 +60,48 @@ export { force_log: bool &default=F; }; - ## The hosts whose software should be detected and tracked. + ## Hosts whose software should be detected and tracked. ## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS const asset_tracking = LOCAL_HOSTS &redef; - ## Other scripts should call this function when they detect software. ## unparsed_version: This is the full string from which the ## :bro:type:`Software::Info` was extracted. + ## + ## id: The connection id where the software was discovered. + ## + ## info: A record representing the software discovered. + ## ## Returns: T if the software was logged, F otherwise. global found: function(id: conn_id, info: Software::Info): bool; - ## This function can take many software version strings and parse them + ## Take many common software version strings and parse them ## into a sensible :bro:type:`Software::Version` record. There are ## still many cases where scripts may have to have their own specific ## version parsing though. + ## + ## unparsed_version: The raw version string. + ## + ## host: The host where the software was discovered. + ## + ## software_type: The type of software. + ## + ## Returns: A complete record ready for the :bro:id:`Software::found` function. global parse: function(unparsed_version: string, host: addr, software_type: Type): Info; - ## Compare two versions. + ## Compare two version records. + ## ## Returns: -1 for v1 < v2, 0 for v1 == v2, 1 for v1 > v2. ## If the numerical version numbers match, the addl string ## is compared lexicographically. global cmp_versions: function(v1: Version, v2: Version): int; - ## This type represents a set of software. It's used by the - ## :bro:id:`Software::tracked` variable to store all known pieces of - ## software for a particular host. It's indexed with the name of a piece - ## of software such as "Firefox" and it yields a - ## :bro:type:`Software::Info` record with more information about the - ## software. + ## Type to represent a collection of :bro:type:`Software::Info` records. + ## It's indexed with the name of a piece of software such as "Firefox" + ## and it yields a :bro:type:`Software::Info` record with more information + ## about the software. type SoftwareSet: table[string] of Info; ## The set of software associated with an address. Data expires from diff --git a/scripts/policy/frameworks/control/controllee.bro b/scripts/policy/frameworks/control/controllee.bro index 798ab8814a..b4769764f4 100644 --- a/scripts/policy/frameworks/control/controllee.bro +++ b/scripts/policy/frameworks/control/controllee.bro @@ -1,3 +1,12 @@ +##! The controllee portion of the control framework. Load this script if remote +##! runtime control of the Bro process is desired. +##! +##! A controllee only needs to load the controllee script in addition +##! to the specific analysis scripts desired. It may also need a node +##! configured as a controller node in the communications nodes configuration:: +##! +##! bro frameworks/control/controllee + @load base/frameworks/control # If an instance is a controllee, it implicitly needs to listen for remote # connections. diff --git a/scripts/policy/frameworks/control/controller.bro b/scripts/policy/frameworks/control/controller.bro index cb76a8b322..39647095db 100644 --- a/scripts/policy/frameworks/control/controller.bro +++ b/scripts/policy/frameworks/control/controller.bro @@ -1,3 +1,10 @@ +##! This is a utility script that implements the controller interface for the +##! control framework. It's intended to be run to control a remote Bro +##! and then shutdown. +##! +##! It's intended to be used from the command line like this:: +##! bro frameworks/control/controller Control::host= Control::port= Control::cmd= [Control::arg=] + @load base/frameworks/control @load base/frameworks/communication diff --git a/scripts/policy/frameworks/metrics/conn-example.bro b/scripts/policy/frameworks/metrics/conn-example.bro index b3800c3ed3..974012963b 100644 --- a/scripts/policy/frameworks/metrics/conn-example.bro +++ b/scripts/policy/frameworks/metrics/conn-example.bro @@ -1,3 +1,6 @@ +##! An example of using the metrics framework to collect connection metrics +##! aggregated into /24 CIDR ranges. + @load base/frameworks/metrics @load base/utils/site diff --git a/scripts/policy/frameworks/metrics/http-example.bro b/scripts/policy/frameworks/metrics/http-example.bro index 50b18b2a27..58ca4e6614 100644 --- a/scripts/policy/frameworks/metrics/http-example.bro +++ b/scripts/policy/frameworks/metrics/http-example.bro @@ -1,9 +1,17 @@ +##! Provides an example of aggregating and limiting collection down to +##! only local networks. Additionally, the status code for the response from +##! the request is added into the metric. + @load base/frameworks/metrics @load base/protocols/http @load base/utils/site redef enum Metrics::ID += { + ## Measures HTTP requests indexed on both the request host and the response + ## code from the server. HTTP_REQUESTS_BY_STATUS_CODE, + + ## Currently unfinished and not working. HTTP_REQUESTS_BY_HOST_HEADER, }; @@ -11,13 +19,13 @@ event bro_init() { # TODO: these are waiting on a fix with table vals + records before they will work. #Metrics::add_filter(HTTP_REQUESTS_BY_HOST_HEADER, - # [$pred(index: Index) = { return Site:is_local_addr(index$host) }, + # [$pred(index: Metrics::Index) = { return Site::is_local_addr(index$host); }, # $aggregation_mask=24, - # $break_interval=5mins]); - # - ## Site::local_nets must be defined in order for this to actually do anything. - #Metrics::add_filter(HTTP_REQUESTS_BY_STATUS_CODE, [$aggregation_table=Site::local_nets_table, - # $break_interval=5mins]); + # $break_interval=1min]); + + # Site::local_nets must be defined in order for this to actually do anything. + Metrics::add_filter(HTTP_REQUESTS_BY_STATUS_CODE, [$aggregation_table=Site::local_nets_table, + $break_interval=1min]); } event HTTP::log_http(rec: HTTP::Info) diff --git a/scripts/policy/frameworks/metrics/ssl-example.bro b/scripts/policy/frameworks/metrics/ssl-example.bro index 46dd0e4741..5ec675779a 100644 --- a/scripts/policy/frameworks/metrics/ssl-example.bro +++ b/scripts/policy/frameworks/metrics/ssl-example.bro @@ -1,3 +1,8 @@ +##! Provides an example of using the metrics framework to collect the number +##! of times a specific server name indicator value is seen in SSL session +##! establishments. Names ending in google.com are being filtered out as an +##! example of the predicate based filtering in metrics filters. + @load base/frameworks/metrics @load base/protocols/ssl diff --git a/scripts/policy/frameworks/software/version-changes.bro b/scripts/policy/frameworks/software/version-changes.bro index 6d46151f0f..974a23dc76 100644 --- a/scripts/policy/frameworks/software/version-changes.bro +++ b/scripts/policy/frameworks/software/version-changes.bro @@ -1,3 +1,7 @@ +##! Provides the possibly to define software names that are interesting to +##! watch for changes. A notice is generated if software versions change on a +##! host. + @load base/frameworks/notice @load base/frameworks/software @@ -5,24 +9,17 @@ module Software; export { redef enum Notice::Type += { - ## For certain softwares, a version changing may matter. In that case, + ## For certain software, a version changing may matter. In that case, ## this notice will be generated. Software that matters if the version ## changes can be configured with the ## :bro:id:`Software::interesting_version_changes` variable. Software_Version_Change, }; - ## Some software is more interesting when the version changes and this + ## Some software is more interesting when the version changes and this is ## a set of all software that should raise a notice when a different ## version is seen on a host. - const interesting_version_changes: set[string] = { - "SSH" - } &redef; - - ## Some software is more interesting when the version changes and this - ## a set of all software that should raise a notice when a different - ## version is seen on a host. - const interesting_type_changes: set[string] = {}; + const interesting_version_changes: set[string] = { } &redef; } event log_software(rec: Info) diff --git a/scripts/policy/frameworks/software/vulnerable.bro b/scripts/policy/frameworks/software/vulnerable.bro index 0ce949b83d..c2c2ba5b32 100644 --- a/scripts/policy/frameworks/software/vulnerable.bro +++ b/scripts/policy/frameworks/software/vulnerable.bro @@ -1,3 +1,7 @@ +##! Provides a variable to define vulnerable versions of software and if a +##! a version of that software as old or older than the defined version a +##! notice will be generated. + @load base/frameworks/notice @load base/frameworks/software @@ -5,6 +9,7 @@ module Software; export { redef enum Notice::Type += { + ## Indicates that a vulnerable version of software was detected. Vulnerable_Version, }; @@ -18,6 +23,7 @@ event log_software(rec: Info) if ( rec$name in vulnerable_versions && cmp_versions(rec$version, vulnerable_versions[rec$name]) <= 0 ) { - NOTICE([$note=Vulnerable_Version, $src=rec$host, $msg=software_fmt(rec)]); + NOTICE([$note=Vulnerable_Version, $src=rec$host, + $msg=fmt("A vulnerable version of software was detected: %s", software_fmt(rec))]); } } From 1afe8b011c2d6191aa87bb4eb5a9c16306fedb50 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Fri, 6 Jan 2012 16:50:20 -0500 Subject: [PATCH 232/964] Fixing a warning from the documentation generation. --- scripts/base/frameworks/metrics/main.bro | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro index 5710de60a1..d322d128fe 100644 --- a/scripts/base/frameworks/metrics/main.bro +++ b/scripts/base/frameworks/metrics/main.bro @@ -140,6 +140,9 @@ export { ## Event to access metrics records as they are passed to the logging framework. global log_metrics: event(rec: Info); + + ## Type to store a table of metrics values. Interal use only! + type MetricTable: table[Index] of count &default=0; } redef record Notice::Info += { @@ -149,8 +152,6 @@ redef record Notice::Info += { global metric_filters: table[ID] of vector of Filter = table(); global filter_store: table[ID, string] of Filter = table(); -type MetricTable: table[Index] of count &default=0; - # This is indexed by metric ID and stream filter name. global store: table[ID, string] of MetricTable = table() &default=table(); From 48ed922e0606ff9a926204d461deb455117ede6f Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Sun, 8 Jan 2012 01:16:40 -0500 Subject: [PATCH 233/964] SSH&FTP Documentation updates. Closes #732 --- scripts/base/protocols/ftp/file-extract.bro | 12 ++-- scripts/base/protocols/ftp/main.bro | 60 +++++++++++-------- scripts/base/protocols/ftp/utils-commands.bro | 12 +++- scripts/base/protocols/ssh/main.bro | 20 ++++--- scripts/policy/protocols/ftp/detect.bro | 2 +- scripts/policy/protocols/ftp/software.bro | 8 ++- .../protocols/ssh/detect-bruteforcing.bro | 10 ++-- scripts/policy/protocols/ssh/geo-data.bro | 7 +-- .../protocols/ssh/interesting-hostnames.bro | 2 +- scripts/policy/protocols/ssh/software.bro | 4 +- 10 files changed, 85 insertions(+), 52 deletions(-) diff --git a/scripts/base/protocols/ftp/file-extract.bro b/scripts/base/protocols/ftp/file-extract.bro index db5c8a0afa..7cee4995ba 100644 --- a/scripts/base/protocols/ftp/file-extract.bro +++ b/scripts/base/protocols/ftp/file-extract.bro @@ -1,4 +1,4 @@ -##! File extraction for FTP. +##! File extraction support for FTP. @load ./main @load base/utils/files @@ -6,7 +6,7 @@ module FTP; export { - ## Pattern of file mime types to extract from FTP entity bodies. + ## Pattern of file mime types to extract from FTP transfers. const extract_file_types = /NO_DEFAULT/ &redef; ## The on-disk prefix for files to be extracted from FTP-data transfers. @@ -14,10 +14,15 @@ export { } redef record Info += { - ## The file handle for the file to be extracted + ## On disk file where it was extracted to. extraction_file: file &log &optional; + ## Indicates if the current command/response pair should attempt to + ## extract the file if a file was transferred. extract_file: bool &default=F; + + ## Internal tracking of the total number of files extracted during this + ## session. num_extracted_files: count &default=0; }; @@ -33,7 +38,6 @@ event file_transferred(c: connection, prefix: string, descr: string, if ( extract_file_types in s$mime_type ) { s$extract_file = T; - add s$tags["extracted_file"]; ++s$num_extracted_files; } } diff --git a/scripts/base/protocols/ftp/main.bro b/scripts/base/protocols/ftp/main.bro index e8eb96d3ee..9e16804a32 100644 --- a/scripts/base/protocols/ftp/main.bro +++ b/scripts/base/protocols/ftp/main.bro @@ -1,11 +1,7 @@ ##! The logging this script does is primarily focused on logging FTP commands ##! along with metadata. For example, if files are transferred, the argument ##! will take on the full path that the client is at along with the requested -##! file name. -##! -##! TODO: -##! -##! * Handle encrypted sessions correctly (get an example?) +##! file name. @load ./utils-commands @load base/utils/paths @@ -14,38 +10,64 @@ module FTP; export { + ## The FTP protocol logging stream identifier. redef enum Log::ID += { LOG }; - + + ## List of commands that should have their command/response pairs logged. + const logged_commands = { + "APPE", "DELE", "RETR", "STOR", "STOU", "ACCT" + } &redef; + ## This setting changes if passwords used in FTP sessions are captured or not. const default_capture_password = F &redef; + ## User IDs that can be considered "anonymous". + const guest_ids = { "anonymous", "ftp", "guest" } &redef; + type Info: record { + ## Time when the command was sent. ts: time &log; uid: string &log; id: conn_id &log; + ## User name for the current FTP session. user: string &log &default=""; + ## Password for the current FTP session if captured. password: string &log &optional; + ## Command given by the client. command: string &log &optional; + ## Argument for the command if one is given. arg: string &log &optional; - + + ## Libmagic "sniffed" file type if the command indicates a file transfer. mime_type: string &log &optional; + ## Libmagic "sniffed" file description if the command indicates a file transfer. mime_desc: string &log &optional; + ## Size of the file if the command indicates a file transfer. file_size: count &log &optional; + + ## Reply code from the server in response to the command. reply_code: count &log &optional; + ## Reply message from the server in response to the command. reply_msg: string &log &optional; + ## Arbitrary tags that may indicate a particular attribute of this command. tags: set[string] &log &default=set(); - ## By setting the CWD to '/.', we can indicate that unless something + ## Current working directory that this session is in. By making + ## the default value '/.', we can indicate that unless something ## more concrete is discovered that the existing but unknown ## directory is ok to use. cwd: string &default="/."; + + ## Command that is currently waiting for a response. cmdarg: CmdArg &optional; + ## Queue for commands that have been sent but not yet responded to + ## are tracked here. pending_commands: PendingCmds; - ## This indicates if the session is in active or passive mode. + ## Indicates if the session is in active or passive mode. passive: bool &default=F; - ## This determines if the password will be captured for this request. + ## Determines if the password will be captured for this request. capture_password: bool &default=default_capture_password; }; @@ -56,22 +78,12 @@ export { y: count; z: count; }; - - # TODO: add this back in some form. raise a notice again? - #const excessive_filename_len = 250 &redef; - #const excessive_filename_trunc_len = 32 &redef; - - ## These are user IDs that can be considered "anonymous". - const guest_ids = { "anonymous", "ftp", "guest" } &redef; - ## The list of commands that should have their command/response pairs logged. - const logged_commands = { - "APPE", "DELE", "RETR", "STOR", "STOU", "ACCT" - } &redef; - - ## This function splits FTP reply codes into the three constituent + ## Parse FTP reply codes into the three constituent single digit values. global parse_ftp_reply_code: function(code: count): ReplyCode; - + + ## Event that can be handled to access the :bro:type:`FTP::Info` + ## record as it is sent on to the logging framework. global log_ftp: event(rec: Info); } diff --git a/scripts/base/protocols/ftp/utils-commands.bro b/scripts/base/protocols/ftp/utils-commands.bro index 40dacf9b66..ddfad3e08d 100644 --- a/scripts/base/protocols/ftp/utils-commands.bro +++ b/scripts/base/protocols/ftp/utils-commands.bro @@ -2,14 +2,22 @@ module FTP; export { type CmdArg: record { + ## Time when the command was sent. ts: time; + ## Command. cmd: string &default=""; + ## Argument for the command if one was given. arg: string &default=""; + ## Counter to track how many commands have been executed. seq: count &default=0; }; - + + ## Structure for tracking pending commands in the event that the client + ## sends a large number of commands before the server has a chance to + ## reply. type PendingCmds: table[count] of CmdArg; - + + ## Possible response codes for a wide variety of FTP commands. const cmd_reply_code: set[string, count] = { # According to RFC 959 ["", [120, 220, 421]], diff --git a/scripts/base/protocols/ssh/main.bro b/scripts/base/protocols/ssh/main.bro index 3a60244184..0d3439bb1f 100644 --- a/scripts/base/protocols/ssh/main.bro +++ b/scripts/base/protocols/ssh/main.bro @@ -14,15 +14,17 @@ module SSH; export { + ## The SSH protocol logging stream identifier. redef enum Log::ID += { LOG }; redef enum Notice::Type += { - ## This indicates that a heuristically detected "successful" SSH + ## Indicates that a heuristically detected "successful" SSH ## authentication occurred. Login }; type Info: record { + ## Time when the SSH connection began. ts: time &log; uid: string &log; id: conn_id &log; @@ -34,11 +36,11 @@ export { ## would be set for the opposite situation. # TODO: handle local-local and remote-remote better. direction: Direction &log &optional; - ## The software string given by the client. + ## Software string given by the client. client: string &log &optional; - ## The software string given by the server. + ## Software string given by the server. server: string &log &optional; - ## The amount of data returned from the server. This is currently + ## Amount of data returned from the server. This is currently ## the only measure of the success heuristic and it is logged to ## assist analysts looking at the logs to make their own determination ## about the success on a case-by-case basis. @@ -48,8 +50,8 @@ export { done: bool &default=F; }; - ## The size in bytes at which the SSH connection is presumed to be - ## successful. + ## The size in bytes of data sent by the server at which the SSH + ## connection is presumed to be successful. const authentication_data_size = 5500 &redef; ## If true, we tell the event engine to not look at further data @@ -58,14 +60,16 @@ export { ## kinds of analyses (e.g., tracking connection size). const skip_processing_after_detection = F &redef; - ## This event is generated when the heuristic thinks that a login + ## Event that is generated when the heuristic thinks that a login ## was successful. global heuristic_successful_login: event(c: connection); - ## This event is generated when the heuristic thinks that a login + ## Event that is generated when the heuristic thinks that a login ## failed. global heuristic_failed_login: event(c: connection); + ## Event that can be handled to access the :bro:type:`SSH::Info` + ## record as it is sent on to the logging framework. global log_ssh: event(rec: Info); } diff --git a/scripts/policy/protocols/ftp/detect.bro b/scripts/policy/protocols/ftp/detect.bro index abb62e08fc..e1bd627921 100644 --- a/scripts/policy/protocols/ftp/detect.bro +++ b/scripts/policy/protocols/ftp/detect.bro @@ -7,7 +7,7 @@ module FTP; export { redef enum Notice::Type += { - ## This indicates that a successful response to a "SITE EXEC" + ## Indicates that a successful response to a "SITE EXEC" ## command/arg pair was seen. Site_Exec_Success, }; diff --git a/scripts/policy/protocols/ftp/software.bro b/scripts/policy/protocols/ftp/software.bro index 622357a608..8834e5cd2e 100644 --- a/scripts/policy/protocols/ftp/software.bro +++ b/scripts/policy/protocols/ftp/software.bro @@ -12,8 +12,10 @@ module FTP; export { redef enum Software::Type += { - FTP_CLIENT, - FTP_SERVER, + ## Identifier for FTP clients in the software framework. + CLIENT, + ## Not currently implemented. + SERVER, }; } @@ -21,7 +23,7 @@ event ftp_request(c: connection, command: string, arg: string) &priority=4 { if ( command == "CLNT" ) { - local si = Software::parse(arg, c$id$orig_h, FTP_CLIENT); + local si = Software::parse(arg, c$id$orig_h, CLIENT); Software::found(c$id, si); } } diff --git a/scripts/policy/protocols/ssh/detect-bruteforcing.bro b/scripts/policy/protocols/ssh/detect-bruteforcing.bro index d6c848f42e..aa6e920c12 100644 --- a/scripts/policy/protocols/ssh/detect-bruteforcing.bro +++ b/scripts/policy/protocols/ssh/detect-bruteforcing.bro @@ -1,3 +1,5 @@ +##! Detect hosts which are doing password guessing attacks and/or password +##! bruteforcing over SSH. @load base/protocols/ssh @load base/frameworks/metrics @@ -13,13 +15,13 @@ export { ## determined failed logins. Password_Guessing, ## Indicates that a host previously identified as a "password guesser" - ## has now had a heuristically successful login attempt. + ## has now had a heuristically successful login attempt. This is not + ## currently implemented. Login_By_Password_Guesser, }; redef enum Metrics::ID += { - ## This metric is to measure failed logins with the hope of detecting - ## bruteforcing hosts. + ## Metric is to measure failed logins. FAILED_LOGIN, }; @@ -37,7 +39,7 @@ export { ## client subnets and the yield value represents server subnets. const ignore_guessers: table[subnet] of subnet &redef; - ## Keeps track of hosts identified as guessing passwords. + ## Tracks hosts identified as guessing passwords. global password_guessers: set[addr] &read_expire=guessing_timeout+1hr &synchronized &redef; } diff --git a/scripts/policy/protocols/ssh/geo-data.bro b/scripts/policy/protocols/ssh/geo-data.bro index daa05f4ebc..0f8bb932fe 100644 --- a/scripts/policy/protocols/ssh/geo-data.bro +++ b/scripts/policy/protocols/ssh/geo-data.bro @@ -1,5 +1,4 @@ -##! This implements all of the additional information and geodata detections -##! for SSH analysis. +##! Geodata based detections for SSH analysis. @load base/frameworks/notice @load base/protocols/ssh @@ -19,8 +18,8 @@ export { remote_location: geo_location &log &optional; }; - ## The set of countries for which you'd like to throw notices upon - ## successful login + ## The set of countries for which you'd like to generate notices upon + ## successful login. const watched_countries: set[string] = {"RO"} &redef; } diff --git a/scripts/policy/protocols/ssh/interesting-hostnames.bro b/scripts/policy/protocols/ssh/interesting-hostnames.bro index 4e870b854e..f79c67ede9 100644 --- a/scripts/policy/protocols/ssh/interesting-hostnames.bro +++ b/scripts/policy/protocols/ssh/interesting-hostnames.bro @@ -10,7 +10,7 @@ module SSH; export { redef enum Notice::Type += { - ## Generated if a login originates or responds with a host and the + ## Generated if a login originates or responds with a host where the ## reverse hostname lookup resolves to a name matched by the ## :bro:id:`SSH::interesting_hostnames` regular expression. Interesting_Hostname_Login, diff --git a/scripts/policy/protocols/ssh/software.bro b/scripts/policy/protocols/ssh/software.bro index a239655270..44704ec845 100644 --- a/scripts/policy/protocols/ssh/software.bro +++ b/scripts/policy/protocols/ssh/software.bro @@ -1,4 +1,4 @@ -##! This script extracts SSH client and server information from SSH +##! Extracts SSH client and server information from SSH ##! connections and forwards it to the software framework. @load base/frameworks/software @@ -7,7 +7,9 @@ module SSH; export { redef enum Software::Type += { + ## Identifier for SSH clients in the software framework. SERVER, + ## Identifier for SSH servers in the software framework. CLIENT, }; } From 3be1222532eaf8a19030570a001c57ffe28e27ca Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Sun, 8 Jan 2012 02:22:52 -0500 Subject: [PATCH 234/964] Documentation updates for HTTP & IRC scripts. Closes #733 --- scripts/base/protocols/http/file-extract.bro | 33 +++++---- scripts/base/protocols/http/file-hash.bro | 7 +- scripts/base/protocols/http/file-ident.bro | 21 +++--- scripts/base/protocols/http/main.bro | 72 +++++++++++-------- scripts/base/protocols/http/utils.bro | 23 ++++++ scripts/base/protocols/irc/dcc-send.bro | 30 ++++---- scripts/base/protocols/irc/main.bro | 26 +++---- scripts/policy/protocols/http/detect-MHR.bro | 8 +-- .../policy/protocols/http/detect-intel.bro | 2 +- scripts/policy/protocols/http/detect-sqli.bro | 6 +- .../policy/protocols/http/detect-webapps.bro | 4 ++ .../http/software-browser-plugins.bro | 5 +- scripts/policy/protocols/http/software.bro | 3 + .../protocols/http/var-extraction-cookies.bro | 3 +- .../protocols/http/var-extraction-uri.bro | 4 +- .../scripts.base.protocols.irc.basic/irc.log | 12 ++-- .../irc.log | 12 ++-- 17 files changed, 160 insertions(+), 111 deletions(-) diff --git a/scripts/base/protocols/http/file-extract.bro b/scripts/base/protocols/http/file-extract.bro index ffb925ae28..466d18c3b4 100644 --- a/scripts/base/protocols/http/file-extract.bro +++ b/scripts/base/protocols/http/file-extract.bro @@ -8,29 +8,24 @@ module HTTP; export { - ## Pattern of file mime types to extract from HTTP entity bodies. + ## Pattern of file mime types to extract from HTTP response entity bodies. const extract_file_types = /NO_DEFAULT/ &redef; ## The on-disk prefix for files to be extracted from HTTP entity bodies. const extraction_prefix = "http-item" &redef; redef record Info += { - ## This field can be set per-connection to determine if the entity body - ## will be extracted. It must be set to T on or before the first - ## entity_body_data event. - extracting_file: bool &default=F; - - ## This is the holder for the file handle as the file is being written - ## to disk. + ## On-disk file where the response body was extracted to. extraction_file: file &log &optional; - }; - - redef record State += { - entity_bodies: count &default=0; + + ## Indicates if the response body is to be extracted or not. Must be + ## set before or by the first :bro:id:`http_entity_data` event for the + ## content. + extract_file: bool &default=F; }; } -event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=5 +event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=-5 { # Client body extraction is not currently supported in this script. if ( is_orig ) @@ -41,8 +36,12 @@ event http_entity_data(c: connection, is_orig: bool, length: count, data: string if ( c$http?$mime_type && extract_file_types in c$http$mime_type ) { - c$http$extracting_file = T; - local suffix = fmt("%s_%d.dat", is_orig ? "orig" : "resp", ++c$http_state$entity_bodies); + c$http$extract_file = T; + } + + if ( c$http$extract_file ) + { + local suffix = fmt("%s_%d.dat", is_orig ? "orig" : "resp", c$http_state$current_response); local fname = generate_extraction_filename(extraction_prefix, c, suffix); c$http$extraction_file = open(fname); @@ -50,12 +49,12 @@ event http_entity_data(c: connection, is_orig: bool, length: count, data: string } } - if ( c$http$extracting_file ) + if ( c$http?$extraction_file ) print c$http$extraction_file, data; } event http_end_entity(c: connection, is_orig: bool) { - if ( c$http$extracting_file ) + if ( c$http?$extraction_file ) close(c$http$extraction_file); } diff --git a/scripts/base/protocols/http/file-hash.bro b/scripts/base/protocols/http/file-hash.bro index 094a905eeb..7e8e5cceaf 100644 --- a/scripts/base/protocols/http/file-hash.bro +++ b/scripts/base/protocols/http/file-hash.bro @@ -11,7 +11,8 @@ export { }; redef record Info += { - ## The MD5 sum for a file transferred over HTTP will be stored here. + ## MD5 sum for a file transferred over HTTP calculated from the + ## response body. md5: string &log &optional; ## This value can be set per-transfer to determine per request @@ -19,8 +20,8 @@ export { ## set to T at the time of or before the first chunk of body data. calc_md5: bool &default=F; - ## This boolean value indicates if an MD5 sum is currently being - ## calculated for the current file transfer. + ## Indicates if an MD5 sum is being calculated for the current + ## request/response pair. calculating_md5: bool &default=F; }; diff --git a/scripts/base/protocols/http/file-ident.bro b/scripts/base/protocols/http/file-ident.bro index c2d858852b..61f6a1199f 100644 --- a/scripts/base/protocols/http/file-ident.bro +++ b/scripts/base/protocols/http/file-ident.bro @@ -1,5 +1,4 @@ -##! This script is involved in the identification of file types in HTTP -##! response bodies. +##! Identification of file types in HTTP response bodies with file content sniffing. @load base/frameworks/signatures @load base/frameworks/notice @@ -15,27 +14,23 @@ module HTTP; export { redef enum Notice::Type += { - # This notice is thrown when the file extension doesn't - # seem to match the file contents. + ## Indicates when the file extension doesn't seem to match the file contents. Incorrect_File_Type, }; redef record Info += { - ## This will record the mime_type identified. + ## Mime type of response body identified by content sniffing. mime_type: string &log &optional; - ## This indicates that no data of the current file transfer has been + ## Indicates that no data of the current file transfer has been ## seen yet. After the first :bro:id:`http_entity_data` event, it - ## will be set to T. + ## will be set to F. first_chunk: bool &default=T; }; - - redef enum Tags += { - IDENTIFIED_FILE - }; - # Create regexes that *should* in be in the urls for specifics mime types. - # Notices are thrown if the pattern doesn't match the url for the file type. + ## Mapping between mime types and regular expressions for URLs + ## The :bro:enum:`HTTP::Incorrect_File_Type` notice is generated if the pattern + ## doesn't match the mime type that was discovered. const mime_types_extensions: table[string] of pattern = { ["application/x-dosexec"] = /\.([eE][xX][eE]|[dD][lL][lL])/, } &redef; diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro index 59107bb4c7..6571548145 100644 --- a/scripts/base/protocols/http/main.bro +++ b/scripts/base/protocols/http/main.bro @@ -1,3 +1,7 @@ +##! Implements base functionality for HTTP analysis. The logging model is +##! to log request/response pairs and all relevant metadata together in +##! a single record. + @load base/utils/numbers @load base/utils/files @@ -8,6 +12,7 @@ export { ## Indicate a type of attack or compromise in the record to be logged. type Tags: enum { + ## Placeholder. EMPTY }; @@ -15,64 +20,69 @@ export { const default_capture_password = F &redef; type Info: record { - ts: time &log; - uid: string &log; - id: conn_id &log; - ## This represents the pipelined depth into the connection of this + ## Timestamp for when the request happened. + ts: time &log; + uid: string &log; + id: conn_id &log; + ## Represents the pipelined depth into the connection of this ## request/response transaction. - trans_depth: count &log; - ## The verb used in the HTTP request (GET, POST, HEAD, etc.). - method: string &log &optional; - ## The value of the HOST header. - host: string &log &optional; - ## The URI used in the request. - uri: string &log &optional; - ## The value of the "referer" header. The comment is deliberately + trans_depth: count &log; + ## Verb used in the HTTP request (GET, POST, HEAD, etc.). + method: string &log &optional; + ## Value of the HOST header. + host: string &log &optional; + ## URI used in the request. + uri: string &log &optional; + ## Value of the "referer" header. The comment is deliberately ## misspelled like the standard declares, but the name used here is ## "referrer" spelled correctly. - referrer: string &log &optional; - ## The value of the User-Agent header from the client. - user_agent: string &log &optional; - ## The actual uncompressed content size of the data transferred from + referrer: string &log &optional; + ## Value of the User-Agent header from the client. + user_agent: string &log &optional; + ## Actual uncompressed content size of the data transferred from ## the client. - request_body_len: count &log &default=0; - ## The actual uncompressed content size of the data transferred from + request_body_len: count &log &default=0; + ## Actual uncompressed content size of the data transferred from ## the server. response_body_len: count &log &default=0; - ## The status code returned by the server. + ## Status code returned by the server. status_code: count &log &optional; - ## The status message returned by the server. + ## Status message returned by the server. status_msg: string &log &optional; - ## The last 1xx informational reply code returned by the server. + ## Last seen 1xx informational reply code returned by the server. info_code: count &log &optional; - ## The last 1xx informational reply message returned by the server. + ## Last seen 1xx informational reply message returned by the server. info_msg: string &log &optional; - ## The filename given in the Content-Disposition header - ## sent by the server. + ## Filename given in the Content-Disposition header sent by the server. filename: string &log &optional; - ## This is a set of indicators of various attributes discovered and + ## A set of indicators of various attributes discovered and ## related to a particular request/response pair. tags: set[Tags] &log; - ## The username if basic-auth is performed for the request. + ## Username if basic-auth is performed for the request. username: string &log &optional; - ## The password if basic-auth is performed for the request. + ## Password if basic-auth is performed for the request. password: string &log &optional; - ## This determines if the password will be captured for this request. + ## Determines if the password will be captured for this request. capture_password: bool &default=default_capture_password; ## All of the headers that may indicate if the request was proxied. proxied: set[string] &log &optional; }; + ## Structure to maintain state for an HTTP connection with multiple + ## requests and responses. type State: record { + ## Pending requests. pending: table[count] of Info; - current_response: count &default=0; + ## Current request in the pending queue. current_request: count &default=0; + ## Current response in the pending queue. + current_response: count &default=0; }; - ## The list of HTTP headers typically used to indicate a proxied request. + ## A list of HTTP headers typically used to indicate proxied requests. const proxy_headers: set[string] = { "FORWARDED", "X-FORWARDED-FOR", @@ -83,6 +93,8 @@ export { "PROXY-CONNECTION", } &redef; + ## Event that can be handled to access the HTTP record as it is sent on + ## to the logging framework. global log_http: event(rec: Info); } diff --git a/scripts/base/protocols/http/utils.bro b/scripts/base/protocols/http/utils.bro index 6e2583bc75..0f2666fade 100644 --- a/scripts/base/protocols/http/utils.bro +++ b/scripts/base/protocols/http/utils.bro @@ -5,8 +5,31 @@ module HTTP; export { + ## Given a string containing a series of key-value pairs separated by "=", + ## this function can be used to parse out all of the key names. + ## + ## data: The raw data, such as a URL or cookie value. + ## + ## kv_splitter: A regular expression representing the separator between + ## key-value pairs. + ## + ## Returns: A vector of strings containing the keys. global extract_keys: function(data: string, kv_splitter: pattern): string_vec; + + ## Creates a URL from an :bro:type:`HTTP::Info` record. This should handle + ## edge cases such as proxied requests appropriately. + ## + ## rec: An :bro:type:`HTTP::Info` record. + ## + ## Returns: A URL, not prefixed by "http://". global build_url: function(rec: Info): string; + + ## Creates a URL from an :bro:type:`HTTP::Info` record. This should handle + ## edge cases such as proxied requests appropriately. + ## + ## rec: An :bro:type:`HTTP::Info` record. + ## + ## Returns: A URL prefixed with "http://". global build_url_http: function(rec: Info): string; } diff --git a/scripts/base/protocols/irc/dcc-send.bro b/scripts/base/protocols/irc/dcc-send.bro index b2a48a472a..9604848777 100644 --- a/scripts/base/protocols/irc/dcc-send.bro +++ b/scripts/base/protocols/irc/dcc-send.bro @@ -5,8 +5,9 @@ ##! but that connection will actually be between B and C which could be ##! analyzed on a different worker. ##! -##! Example line from IRC server indicating that the DCC SEND is about to start: -##! PRIVMSG my_nick :^ADCC SEND whateverfile.zip 3640061780 1026 41709^A + +# Example line from IRC server indicating that the DCC SEND is about to start: +# PRIVMSG my_nick :^ADCC SEND whateverfile.zip 3640061780 1026 41709^A @load ./main @load base/utils/files @@ -14,24 +15,25 @@ module IRC; export { - redef enum Tag += { EXTRACTED_FILE }; - ## Pattern of file mime types to extract from IRC DCC file transfers. const extract_file_types = /NO_DEFAULT/ &redef; - ## The on-disk prefix for files to be extracted from IRC DCC file transfers. + ## On-disk prefix for files to be extracted from IRC DCC file transfers. const extraction_prefix = "irc-dcc-item" &redef; redef record Info += { - dcc_file_name: string &log &optional; - dcc_file_size: count &log &optional; - dcc_mime_type: string &log &optional; + ## DCC filename requested. + dcc_file_name: string &log &optional; + ## Size of the DCC transfer as indicated by the sender. + dcc_file_size: count &log &optional; + ## Sniffed mime type of the file. + dcc_mime_type: string &log &optional; ## The file handle for the file to be extracted - extraction_file: file &log &optional; + extraction_file: file &log &optional; - ## A boolean to indicate if the current file transfer should be extraced. - extract_file: bool &default=F; + ## A boolean to indicate if the current file transfer should be extracted. + extract_file: bool &default=F; ## The count of the number of file that have been extracted during the session. num_extracted_files: count &default=0; @@ -54,8 +56,10 @@ event file_transferred(c: connection, prefix: string, descr: string, if ( extract_file_types == irc$dcc_mime_type ) { irc$extract_file = T; - add irc$tags[EXTRACTED_FILE]; + } + if ( irc$extract_file ) + { local suffix = fmt("%d.dat", ++irc$num_extracted_files); local fname = generate_extraction_filename(extraction_prefix, c, suffix); irc$extraction_file = open(fname); @@ -76,7 +80,7 @@ event file_transferred(c: connection, prefix: string, descr: string, Log::write(IRC::LOG, irc); irc$command = tmp; - if ( irc$extract_file && irc?$extraction_file ) + if ( irc?$extraction_file ) set_contents_file(id, CONTENTS_RESP, irc$extraction_file); # Delete these values in case another DCC transfer diff --git a/scripts/base/protocols/irc/main.bro b/scripts/base/protocols/irc/main.bro index 3736569910..2bf2a9bbb9 100644 --- a/scripts/base/protocols/irc/main.bro +++ b/scripts/base/protocols/irc/main.bro @@ -1,36 +1,38 @@ -##! This is the script that implements the core IRC analysis support. It only -##! logs a very limited subset of the IRC protocol by default. The points -##! that it logs at are NICK commands, USER commands, and JOIN commands. It -##! log various bits of meta data as indicated in the :bro:type:`IRC::Info` -##! record along with the command at the command arguments. +##! Implements the core IRC analysis support. The logging model is to log +##! IRC commands along with the associated response and some additional +##! metadata about the connection if it's available. module IRC; export { + redef enum Log::ID += { LOG }; - type Tag: enum { - EMPTY - }; - type Info: record { + ## Timestamp when the command was seen. ts: time &log; uid: string &log; id: conn_id &log; + ## Nick name given for the connection. nick: string &log &optional; + ## User name given for the connection. user: string &log &optional; - channels: set[string] &log &optional; - + + ## Command given by the client. command: string &log &optional; + ## Value for the command given by the client. value: string &log &optional; + ## Any additional data for the command. addl: string &log &optional; - tags: set[Tag] &log; }; + ## Event that can be handled to access the IRC record as it is sent on + ## to the logging framework. global irc_log: event(rec: Info); } redef record connection += { + ## IRC session information. irc: Info &optional; }; diff --git a/scripts/policy/protocols/http/detect-MHR.bro b/scripts/policy/protocols/http/detect-MHR.bro index fca3bb1ac9..a0e3cb50fb 100644 --- a/scripts/policy/protocols/http/detect-MHR.bro +++ b/scripts/policy/protocols/http/detect-MHR.bro @@ -1,8 +1,8 @@ -##! This script takes MD5 sums of files transferred over HTTP and checks them with -##! Team Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/). +##! Detect file downloads over HTTP that have MD5 sums matching files in Team +##! Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/). ##! By default, not all file transfers will have MD5 sums calculated. Read the -##! documentation for the :doc:base/protocols/http/file-hash.bro script to see how to -##! configure which transfers will have hashes calculated. +##! documentation for the :doc:base/protocols/http/file-hash.bro script to see +##! how to configure which transfers will have hashes calculated. @load base/frameworks/notice @load base/protocols/http diff --git a/scripts/policy/protocols/http/detect-intel.bro b/scripts/policy/protocols/http/detect-intel.bro index 6da4d8d1e1..281d705c13 100644 --- a/scripts/policy/protocols/http/detect-intel.bro +++ b/scripts/policy/protocols/http/detect-intel.bro @@ -1,4 +1,4 @@ -##! Intelligence based HTTP detections. +##! Intelligence based HTTP detections. Not yet working! @load base/protocols/http/main @load base/protocols/http/utils diff --git a/scripts/policy/protocols/http/detect-sqli.bro b/scripts/policy/protocols/http/detect-sqli.bro index e7d81e3b0f..a92565c63a 100644 --- a/scripts/policy/protocols/http/detect-sqli.bro +++ b/scripts/policy/protocols/http/detect-sqli.bro @@ -16,7 +16,9 @@ export { }; redef enum Metrics::ID += { + ## Metric to track SQL injection attackers. SQLI_ATTACKER, + ## Metrics to track SQL injection victims. SQLI_VICTIM, }; @@ -30,7 +32,7 @@ export { COOKIE_SQLI, }; - ## This defines the threshold that determines if an SQL injection attack + ## Defines the threshold that determines if an SQL injection attack ## is ongoing based on the number of requests that appear to be SQL ## injection attacks. const sqli_requests_threshold = 50 &redef; @@ -40,7 +42,7 @@ export { ## At the end of each interval the counter is reset. const sqli_requests_interval = 5min &redef; - ## This regular expression is used to match URI based SQL injections + ## Regular expression is used to match URI based SQL injections. const match_sql_injection_uri = /[\?&][^[:blank:]\x00-\x37\|]+?=[\-[:alnum:]%]+([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]?([[:blank:]\x00-\x37]|\/\*.*?\*\/|\)?;)+.*?([hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[iI][nN][sS][eE][rR][tT])([[:blank:]\x00-\x37]|\/\*.*?\*\/)+/ | /[\?&][^[:blank:]\x00-\x37\|]+?=[\-0-9%]+([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]?([[:blank:]\x00-\x37]|\/\*.*?\*\/|\)?;)+([xX]?[oO][rR]|[nN]?[aA][nN][dD])([[:blank:]\x00-\x37]|\/\*.*?\*\/)+['"]?(([^a-zA-Z&]+)?=|[eE][xX][iI][sS][tT][sS])/ diff --git a/scripts/policy/protocols/http/detect-webapps.bro b/scripts/policy/protocols/http/detect-webapps.bro index 4a94d1adbd..a13d64679a 100644 --- a/scripts/policy/protocols/http/detect-webapps.bro +++ b/scripts/policy/protocols/http/detect-webapps.bro @@ -1,3 +1,5 @@ +##! Detect and log web applications through the software framework. + @load base/frameworks/signatures @load base/frameworks/software @load base/protocols/http @@ -10,10 +12,12 @@ redef Signatures::ignored_ids += /^webapp-/; export { redef enum Software::Type += { + ## Identifier for web applications in the software framework. WEB_APPLICATION, }; redef record Software::Info += { + ## Most root URL where the software was discovered. url: string &optional &log; }; } diff --git a/scripts/policy/protocols/http/software-browser-plugins.bro b/scripts/policy/protocols/http/software-browser-plugins.bro index db9eafd1a7..a035a2dcfc 100644 --- a/scripts/policy/protocols/http/software-browser-plugins.bro +++ b/scripts/policy/protocols/http/software-browser-plugins.bro @@ -1,5 +1,5 @@ -##! This script take advantage of a few ways that installed plugin information -##! leaks from web browsers. +##! Detect browser plugins as they leak through requests to Omniture +##! advertising servers. @load base/protocols/http @load base/frameworks/software @@ -13,6 +13,7 @@ export { }; redef enum Software::Type += { + ## Identifier for browser plugins in the software framework. BROWSER_PLUGIN }; } diff --git a/scripts/policy/protocols/http/software.bro b/scripts/policy/protocols/http/software.bro index 8732634359..8907d0853e 100644 --- a/scripts/policy/protocols/http/software.bro +++ b/scripts/policy/protocols/http/software.bro @@ -6,8 +6,11 @@ module HTTP; export { redef enum Software::Type += { + ## Identifier for web servers in the software framework. SERVER, + ## Identifier for app servers in the software framework. APPSERVER, + ## Identifier for web browsers in the software framework. BROWSER, }; diff --git a/scripts/policy/protocols/http/var-extraction-cookies.bro b/scripts/policy/protocols/http/var-extraction-cookies.bro index 2b3f282b03..610c6e1381 100644 --- a/scripts/policy/protocols/http/var-extraction-cookies.bro +++ b/scripts/policy/protocols/http/var-extraction-cookies.bro @@ -1,4 +1,4 @@ -##! This script extracts and logs variables from cookies sent by clients +##! Extracts and logs variables names from cookies sent by clients. @load base/protocols/http/main @load base/protocols/http/utils @@ -6,6 +6,7 @@ module HTTP; redef record Info += { + ## Variable names extracted from all cookies. cookie_vars: vector of string &optional &log; }; diff --git a/scripts/policy/protocols/http/var-extraction-uri.bro b/scripts/policy/protocols/http/var-extraction-uri.bro index b03474bb94..27ee89d6f2 100644 --- a/scripts/policy/protocols/http/var-extraction-uri.bro +++ b/scripts/policy/protocols/http/var-extraction-uri.bro @@ -1,10 +1,12 @@ -##! This script extracts and logs variables from the requested URI +##! Extracts and log variables from the requested URI in the default HTTP +##! logging stream. @load base/protocols/http module HTTP; redef record Info += { + ## Variable names from the URI. uri_vars: vector of string &optional &log; }; diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log index 39ff897fae..b5c137bcf8 100644 --- a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log +++ b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path irc -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user channels command value addl tags dcc_file_name dcc_file_size extraction_file -#types time string addr port addr port string string table[string] string string string table[enum] string count file -1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - - NICK bloed - (empty) - - - -1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq (empty) - - - -1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - JOIN #easymovies (empty) (empty) - - - -1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - DCC #easymovies (empty) (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 - +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user command value addl dcc_file_name dcc_file_size extraction_file +#types time string addr port addr port string string string string string string count file +1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - NICK bloed - - - - +1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - +1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje JOIN #easymovies (empty) - - - +1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje DCC #easymovies (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 - diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log index 342923ba7b..7513bfb9b8 100644 --- a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log +++ b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path irc -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user channels command value addl tags dcc_file_name dcc_file_size dcc_mime_type extraction_file -#types time string addr port addr port string string table[string] string string string table[enum] string count string file -1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - - NICK bloed - (empty) - - - - -1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq (empty) - - - - -1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - JOIN #easymovies (empty) (empty) - - - - -1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - DCC #easymovies (empty) IRC::EXTRACTED_FILE ladyvampress-default(2011-07-07)-OS.zip 42208 FAKE_MIME irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user command value addl dcc_file_name dcc_file_size dcc_mime_type extraction_file +#types time string addr port addr port string string string string string string count string file +1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - NICK bloed - - - - - +1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - - +1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje JOIN #easymovies (empty) - - - - +1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje DCC #easymovies (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 FAKE_MIME irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat From f389fb42c3df4886a3527ba93b96cc48f034afe0 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 9 Jan 2012 13:23:14 -0500 Subject: [PATCH 235/964] Small updates to the default local.bro. - Removed the note from local-manager.bro about setting the notice policy there. The notice framework changed and this isn't necessary anymore. --- scripts/site/local-manager.bro | 10 +--------- scripts/site/local-proxy.bro | 1 - scripts/site/local.bro | 30 ++++++++++++++++++------------ 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/scripts/site/local-manager.bro b/scripts/site/local-manager.bro index c933207603..5e6005f21e 100644 --- a/scripts/site/local-manager.bro +++ b/scripts/site/local-manager.bro @@ -1,9 +1 @@ -##! Local site policy loaded only by the manager in a cluster. - -@load base/frameworks/notice - -# If you are running a cluster you should define your Notice::policy here -# so that notice processing occurs on the manager. -redef Notice::policy += { - -}; +##! Local site policy loaded only by the manager if Bro is running as a cluster. diff --git a/scripts/site/local-proxy.bro b/scripts/site/local-proxy.bro index 1b71cc1870..478ba6d048 100644 --- a/scripts/site/local-proxy.bro +++ b/scripts/site/local-proxy.bro @@ -1,2 +1 @@ ##! Local site policy loaded only by the proxies if Bro is running as a cluster. - diff --git a/scripts/site/local.bro b/scripts/site/local.bro index 8e1f4d802b..9681f7a75c 100644 --- a/scripts/site/local.bro +++ b/scripts/site/local.bro @@ -1,22 +1,29 @@ -##! Local site policy. Customize as appropriate. This file will not be -##! overwritten when upgrading or reinstalling. +##! Local site policy. Customize as appropriate. +##! +##! This file will not be overwritten when upgrading or reinstalling! -# Load the script to log which script were loaded during each run +# This script logs which scripts were loaded during each run. @load misc/loaded-scripts # Apply the default tuning scripts for common tuning settings. @load tuning/defaults -# Vulnerable versions of software to generate notices for when discovered. +# Generate notices when vulnerable versions of software are discovered. # The default is to only monitor software found in the address space defined # as "local". Refer to the software framework's documentation for more # information. @load frameworks/software/vulnerable + +# Example vulnerable software. This needs to be updated and maintained over +# time as new vulnerabilities are discovered. redef Software::vulnerable_versions += { ["Flash"] = [$major=10,$minor=2,$minor2=153,$addl="1"], ["Java"] = [$major=1,$minor=6,$minor2=0,$addl="22"], }; +# Detect software changing (e.g. attacker installing hacked SSHD). +@load frameworks/software/version-changes + # This adds signatures to detect cleartext forward and reverse windows shells. redef signature_files += "frameworks/signatures/detect-windows-shells.sig"; @@ -25,13 +32,15 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig"; # redef Notice::policy += { [$action = Notice::ACTION_ALARM, $priority = 0] }; # Load all of the scripts that detect software in various protocols. -@load protocols/http/software -#@load protocols/http/detect-webapps @load protocols/ftp/software @load protocols/smtp/software @load protocols/ssh/software +@load protocols/http/software +# The detect-webapps script could possibly cause performance trouble when +# running on live traffic. Enable it cautiously. +#@load protocols/http/detect-webapps -# Load the script to detect DNS results pointing toward your Site::local_nets +# This script detects DNS results pointing toward your Site::local_nets # where the name is not part of your local DNS zone and is being hosted # externally. Requires that the Site::local_zones variable is defined. @load protocols/dns/detect-external-names @@ -39,15 +48,12 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig"; # Script to detect various activity in FTP sessions. @load protocols/ftp/detect -# Detect software changing (e.g. attacker installing hacked SSHD). -@load frameworks/software/version-changes - # Scripts that do asset tracking. @load protocols/conn/known-hosts @load protocols/conn/known-services @load protocols/ssl/known-certs -# Load the script to enable SSL/TLS certificate validation. +# This script enables SSL/TLS certificate validation. @load protocols/ssl/validate-certs # If you have libGeoIP support built in, do some geographic detections and @@ -60,5 +66,5 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig"; # Detect MD5 sums in Team Cymru's Malware Hash Registry. @load protocols/http/detect-MHR -# Detect SQL injection attacks +# Detect SQL injection attacks. @load protocols/http/detect-sqli From 62d012e04aad212c7195f4ad3fd3ad43920efcb7 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 9 Jan 2012 14:23:24 -0600 Subject: [PATCH 236/964] Add Conn and DNS protocol script documentation. (fixes #731) --- scripts/base/protocols/conn/contents.bro | 22 +++--- scripts/base/protocols/conn/inactivity.bro | 2 +- scripts/base/protocols/conn/main.bro | 27 +++++-- scripts/base/protocols/dns/consts.bro | 71 +++++++++---------- scripts/base/protocols/dns/main.bro | 54 +++++++++++++- scripts/policy/protocols/conn/known-hosts.bro | 10 ++- .../policy/protocols/conn/known-services.bro | 26 +++++-- 7 files changed, 148 insertions(+), 64 deletions(-) diff --git a/scripts/base/protocols/conn/contents.bro b/scripts/base/protocols/conn/contents.bro index feabb1303c..2e6b547ab1 100644 --- a/scripts/base/protocols/conn/contents.bro +++ b/scripts/base/protocols/conn/contents.bro @@ -1,23 +1,27 @@ ##! This script can be used to extract either the originator's data or the ##! responders data or both. By default nothing is extracted, and in order ##! to actually extract data the ``c$extract_orig`` and/or the -##! ``c$extract_resp`` variable must be set to T. One way to achieve this -##! would be to handle the connection_established event elsewhere and set the -##! extract_orig and extract_resp options there. However, there may be trouble -##! with the timing due the event queue delay. -##! This script does not work well in a cluster context unless it has a -##! remotely mounted disk to write the content files to. +##! ``c$extract_resp`` variable must be set to ``T``. One way to achieve this +##! would be to handle the :bro:id:`connection_established` event elsewhere +##! and set the ``extract_orig`` and ``extract_resp`` options there. +##! However, there may be trouble with the timing due to event queue delay. +##! +##! .. note:: +##! +##! This script does not work well in a cluster context unless it has a +##! remotely mounted disk to write the content files to. @load base/utils/files module Conn; export { - ## The prefix given to files as they are opened on disk. + ## The prefix given to files containing extracted connections as they are + ## opened on disk. const extraction_prefix = "contents" &redef; - ## If this variable is set to T, then all contents of all files will be - ## extracted. + ## If this variable is set to ``T``, then all contents of all connections + ## will be extracted. const default_extract = F &redef; } diff --git a/scripts/base/protocols/conn/inactivity.bro b/scripts/base/protocols/conn/inactivity.bro index 04dab62470..28df192de3 100644 --- a/scripts/base/protocols/conn/inactivity.bro +++ b/scripts/base/protocols/conn/inactivity.bro @@ -4,7 +4,7 @@ module Conn; export { - ## Define inactivty timeouts by the service detected being used over + ## Define inactivity timeouts by the service detected being used over ## the connection. const analyzer_inactivity_timeouts: table[AnalyzerTag] of interval = { # For interactive services, allow longer periods of inactivity. diff --git a/scripts/base/protocols/conn/main.bro b/scripts/base/protocols/conn/main.bro index 751fe8f6cf..34ec12fa56 100644 --- a/scripts/base/protocols/conn/main.bro +++ b/scripts/base/protocols/conn/main.bro @@ -1,17 +1,33 @@ +##! This script manages the tracking/logging of general information regarding +##! TCP, UDP, and ICMP traffic. For UDP and ICMP, "connections" are to +##! be interpreted using flow semantics (sequence of packets from a source +##! host/post to a destination host/port). Further, ICMP "ports" are to +##! be interpreted as the source port meaning the ICMP message type and +##! the destination port being the ICMP message code. + @load base/utils/site module Conn; export { + ## The connection logging stream identifier. redef enum Log::ID += { LOG }; + ## The record type which contains column fields of the connection log. type Info: record { ## This is the time of the first packet. ts: time &log; + ## A unique identifier of a connection. uid: string &log; + ## The connection's 4-tuple of endpoint addresses/ports. id: conn_id &log; + ## The transport layer protocol of the connection. proto: transport_proto &log; + ## An identification of an application protocol being sent over the + ## the connection. service: string &log &optional; + ## How long the connection lasted. For 3-way or 4-way connection + ## tear-downs, this will not include the final ACK. duration: interval &log &optional; ## The number of payload bytes the originator sent. For TCP ## this is taken from sequence numbers and might be inaccurate @@ -51,8 +67,8 @@ export { ## have been completed prior to the packet loss. missed_bytes: count &log &default=0; - ## Records the state history of (TCP) connections as - ## a string of letters. + ## Records the state history of connections as a string of letters. + ## For TCP connections the meaning of those letters is: ## ## ====== ==================================================== ## Letter Meaning @@ -71,7 +87,8 @@ export { ## originator and lower case then means the responder. ## Also, there is compression. We only record one "d" in each direction, ## for instance. I.e., we just record that data went in that direction. - ## This history is not meant to encode how much data that happened to be. + ## This history is not meant to encode how much data that happened to + ## be. history: string &log &optional; ## Number of packets the originator sent. ## Only set if :bro:id:`use_conn_size_analyzer` = T @@ -85,7 +102,9 @@ export { ## Number IP level bytes the responder sent. See ``orig_pkts``. resp_ip_bytes: count &log &optional; }; - + + ## Event that can be handled to access the :bro:type:`Conn::Info` + ## record as it is sent on to the logging framework. global log_conn: event(rec: Info); } diff --git a/scripts/base/protocols/dns/consts.bro b/scripts/base/protocols/dns/consts.bro index b57170dded..fbf4aba008 100644 --- a/scripts/base/protocols/dns/consts.bro +++ b/scripts/base/protocols/dns/consts.bro @@ -4,9 +4,9 @@ module DNS; export { - const PTR = 12; - const EDNS = 41; - const ANY = 255; + const PTR = 12; ##< RR TYPE value for a domain name pointer. + const EDNS = 41; ##< An OPT RR TYPE value described by EDNS. + const ANY = 255; ##< A QTYPE value describing a request for all records. ## Mapping of DNS query type codes to human readable string representation. const query_types = { @@ -29,50 +29,43 @@ export { [ANY] = "*", } &default = function(n: count): string { return fmt("query-%d", n); }; - const code_types = { - [0] = "X0", - [1] = "Xfmt", - [2] = "Xsrv", - [3] = "Xnam", - [4] = "Ximp", - [5] = "X[", - } &default="?"; - ## Errors used for non-TSIG/EDNS types. const base_errors = { - [0] = "NOERROR", ##< No Error - [1] = "FORMERR", ##< Format Error - [2] = "SERVFAIL", ##< Server Failure - [3] = "NXDOMAIN", ##< Non-Existent Domain - [4] = "NOTIMP", ##< Not Implemented - [5] = "REFUSED", ##< Query Refused - [6] = "YXDOMAIN", ##< Name Exists when it should not - [7] = "YXRRSET", ##< RR Set Exists when it should not - [8] = "NXRRSet", ##< RR Set that should exist does not - [9] = "NOTAUTH", ##< Server Not Authoritative for zone - [10] = "NOTZONE", ##< Name not contained in zone - [11] = "unassigned-11", ##< available for assignment - [12] = "unassigned-12", ##< available for assignment - [13] = "unassigned-13", ##< available for assignment - [14] = "unassigned-14", ##< available for assignment - [15] = "unassigned-15", ##< available for assignment - [16] = "BADVERS", ##< for EDNS, collision w/ TSIG - [17] = "BADKEY", ##< Key not recognized - [18] = "BADTIME", ##< Signature out of time window - [19] = "BADMODE", ##< Bad TKEY Mode - [20] = "BADNAME", ##< Duplicate key name - [21] = "BADALG", ##< Algorithm not supported - [22] = "BADTRUNC", ##< draft-ietf-dnsext-tsig-sha-05.txt - [3842] = "BADSIG", ##< 16 <= number collision with EDNS(16); - ##< this is a translation from TSIG(16) + [0] = "NOERROR", # No Error + [1] = "FORMERR", # Format Error + [2] = "SERVFAIL", # Server Failure + [3] = "NXDOMAIN", # Non-Existent Domain + [4] = "NOTIMP", # Not Implemented + [5] = "REFUSED", # Query Refused + [6] = "YXDOMAIN", # Name Exists when it should not + [7] = "YXRRSET", # RR Set Exists when it should not + [8] = "NXRRSet", # RR Set that should exist does not + [9] = "NOTAUTH", # Server Not Authoritative for zone + [10] = "NOTZONE", # Name not contained in zone + [11] = "unassigned-11", # available for assignment + [12] = "unassigned-12", # available for assignment + [13] = "unassigned-13", # available for assignment + [14] = "unassigned-14", # available for assignment + [15] = "unassigned-15", # available for assignment + [16] = "BADVERS", # for EDNS, collision w/ TSIG + [17] = "BADKEY", # Key not recognized + [18] = "BADTIME", # Signature out of time window + [19] = "BADMODE", # Bad TKEY Mode + [20] = "BADNAME", # Duplicate key name + [21] = "BADALG", # Algorithm not supported + [22] = "BADTRUNC", # draft-ietf-dnsext-tsig-sha-05.txt + [3842] = "BADSIG", # 16 <= number collision with EDNS(16); + # this is a translation from TSIG(16) } &default = function(n: count): string { return fmt("rcode-%d", n); }; - # This deciphers EDNS Z field values. + ## This deciphers EDNS Z field values. const edns_zfield = { [0] = "NOVALUE", # regular entry [32768] = "DNS_SEC_OK", # accepts DNS Sec RRs } &default="?"; + ## Possible values of the CLASS field in resource records or QCLASS field + ## in query messages. const classes = { [1] = "C_INTERNET", [2] = "C_CSNET", @@ -81,4 +74,4 @@ export { [254] = "C_NONE", [255] = "C_ANY", } &default = function(n: count): string { return fmt("qclass-%d", n); }; -} \ No newline at end of file +} diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro index b8cfc7b44e..56107fd02d 100644 --- a/scripts/base/protocols/dns/main.bro +++ b/scripts/base/protocols/dns/main.bro @@ -1,38 +1,80 @@ +##! Base DNS analysis script which tracks and logs DNS queries along with +##! their responses. + @load ./consts module DNS; export { + ## The DNS logging stream identifier. redef enum Log::ID += { LOG }; + ## The record type which contains the column fields of the DNS log. type Info: record { + ## The earliest time at which a DNS protocol message over the + ## associated connection is observed. ts: time &log; + ## A unique identifier of the connection over which DNS messages + ## are being transferred. uid: string &log; + ## The connection's 4-tuple of endpoint addresses/ports. id: conn_id &log; + ## The transport layer protocol of the connection. proto: transport_proto &log; + ## A 16 bit identifier assigned by the program that generated the + ## DNS query. Also used in responses to match up replies to + ## outstanding queries. trans_id: count &log &optional; + ## The domain name that is the subject of the DNS query. query: string &log &optional; + ## The QCLASS value specifying the class of the query. qclass: count &log &optional; + ## A descriptive name for the class of the query. qclass_name: string &log &optional; + ## A QTYPE value specifying the type of the query. qtype: count &log &optional; + ## A descriptive name for the type of the query. qtype_name: string &log &optional; + ## The response code value in DNS response messages. rcode: count &log &optional; + ## A descriptive name for the response code value. rcode_name: string &log &optional; + ## Whether the message is a query (F) or response (T). QR: bool &log &default=F; + ## The Authoritative Answer bit for response messages specifies that + ## the responding name server is an authority for the domain name + ## in the question section. AA: bool &log &default=F; + ## The Truncation bit specifies that the message was truncated. TC: bool &log &default=F; + ## The Recursion Desired bit indicates to a name server to recursively + ## purse the query. RD: bool &log &default=F; + ## The Recursion Available bit in a response message indicates if + ## the name server supports recursive queries. RA: bool &log &default=F; + ## A reserved field that is currently supposed to be zero in all + ## queries and responses. Z: count &log &default=0; + ## The set of resource descriptions in answer of the query. answers: vector of string &log &optional; + ## The caching intervals of the associated RRs described by the + ## ``answers`` field. TTLs: vector of interval &log &optional; - ## This value indicates if this request/response pair is ready to be logged. + ## This value indicates if this request/response pair is ready to be + ## logged. ready: bool &default=F; + ## The total number of resource records in a reply message's answer + ## section. total_answers: count &optional; + ## The total number of resource records in a reply message's answer, + ## authority, and additional sections. total_replies: count &optional; }; + ## A record type which tracks the status of DNS queries for a given + ## :bro:type:`connection`. type State: record { ## Indexed by query id, returns Info record corresponding to ## query/response which haven't completed yet. @@ -44,11 +86,21 @@ export { finished_answers: set[count] &optional; }; + ## An event that can be handled to access the :bro:type:`DNS::Info` + ## record as it is sent to the logging framework. global log_dns: event(rec: Info); ## This is called by the specific dns_*_reply events with a "reply" which ## may not represent the full data available from the resource record, but ## it's generally considered a summarization of the response(s). + ## + ## c: The connection record for which to fill in DNS reply data. + ## + ## msg: The DNS message header information for the response. + ## + ## ans: The general information of a RR response. + ## + ## reply: The specific response information according to RR type/class. global do_reply: event(c: connection, msg: dns_msg, ans: dns_answer, reply: string); } diff --git a/scripts/policy/protocols/conn/known-hosts.bro b/scripts/policy/protocols/conn/known-hosts.bro index 017b6c8a25..8914a5a22a 100644 --- a/scripts/policy/protocols/conn/known-hosts.bro +++ b/scripts/policy/protocols/conn/known-hosts.bro @@ -8,8 +8,10 @@ module Known; export { + ## The known-hosts logging stream identifier. redef enum Log::ID += { HOSTS_LOG }; - + + ## The record type which contains the column fields of the known-hosts log. type HostsInfo: record { ## The timestamp at which the host was detected. ts: time &log; @@ -19,7 +21,7 @@ export { }; ## The hosts whose existence should be logged and tracked. - ## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS + ## See :bro:type:`Host` for possible choices. const host_tracking = LOCAL_HOSTS &redef; ## The set of all known addresses to store for preventing duplicate @@ -28,7 +30,9 @@ export { ## Maintain the list of known hosts for 24 hours so that the existence ## of each individual address is logged each day. global known_hosts: set[addr] &create_expire=1day &synchronized &redef; - + + ## An event that can be handled to access the :bro:type:`Known::HostsInfo` + ## record as it is sent on to the logging framework. global log_known_hosts: event(rec: HostsInfo); } diff --git a/scripts/policy/protocols/conn/known-services.bro b/scripts/policy/protocols/conn/known-services.bro index 9d58f3a9fb..f494a30f82 100644 --- a/scripts/policy/protocols/conn/known-services.bro +++ b/scripts/policy/protocols/conn/known-services.bro @@ -8,29 +8,41 @@ module Known; export { + ## The known-services logging stream identifier. redef enum Log::ID += { SERVICES_LOG }; - + + ## The record type which contains the column fields of the known-services + ## log. type ServicesInfo: record { + ## The time at which the service was detected. ts: time &log; + ## The host address on which the service is running. host: addr &log; + ## The port number on which the service is running. port_num: port &log; + ## The transport-layer protocol which the service uses. port_proto: transport_proto &log; + ## A set of protocols that match the service's connection payloads. service: set[string] &log; - - done: bool &default=F; }; ## The hosts whose services should be tracked and logged. + ## See :bro:type:`Host` for possible choices. const service_tracking = LOCAL_HOSTS &redef; - + + ## Tracks the set of daily-detected services for preventing the logging + ## of duplicates, but can also be inspected by other scripts for + ## different purposes. global known_services: set[addr, port] &create_expire=1day &synchronized; - + + ## Event that can be handled to access the :bro:type:`Known::ServicesInfo` + ## record as it is sent on to the logging framework. global log_known_services: event(rec: ServicesInfo); } redef record connection += { - ## This field is to indicate whether or not the processing for detecting - ## and logging the service for this connection is complete. + # This field is to indicate whether or not the processing for detecting + # and logging the service for this connection is complete. known_services_done: bool &default=F; }; From acf5537acfa865d27aea7013b8b99444f874a5fd Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Mon, 9 Jan 2012 15:26:34 -0600 Subject: [PATCH 237/964] Add ssl and syslog script documentation --- scripts/base/protocols/ssl/main.bro | 3 +++ scripts/base/protocols/syslog/main.bro | 1 + 2 files changed, 4 insertions(+) diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 4b2fa39696..4c6721fcd8 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -1,3 +1,5 @@ +##! Base SSL analysis script. + @load ./consts module SSL; @@ -6,6 +8,7 @@ export { redef enum Log::ID += { LOG }; type Info: record { + ## Time when the SSL connection began. ts: time &log; uid: string &log; id: conn_id &log; diff --git a/scripts/base/protocols/syslog/main.bro b/scripts/base/protocols/syslog/main.bro index 2acc843ea8..f4afa6d34d 100644 --- a/scripts/base/protocols/syslog/main.bro +++ b/scripts/base/protocols/syslog/main.bro @@ -8,6 +8,7 @@ export { redef enum Log::ID += { LOG }; type Info: record { + ## Timestamp of when the syslog message was seen. ts: time &log; uid: string &log; id: conn_id &log; From b8778026a66ddf470cd80746c47ab7be0145ca2c Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 9 Jan 2012 16:11:41 -0600 Subject: [PATCH 238/964] Add summary documentation to bif files. --- src/bro.bif | 6 +++++- src/const.bif | 4 +++- src/event.bif | 5 +++++ src/strings.bif | 3 ++- src/types.bif | 5 +++-- 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index c941d19c3b..c9a843e743 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1,4 +1,8 @@ -# Definitions of Bro built-in functions. +##! A collection of built-in functions that implement a variety of things +##! such as general programming algorithms, string processing, math functions, +##! introspection, type conversion, file/directory manipulation, packet +##! filtering, inter-process communication and controlling protocol analyzer +##! behavior. %%{ // C segment #include diff --git a/src/const.bif b/src/const.bif index 96630e300b..bc960caeb6 100644 --- a/src/const.bif +++ b/src/const.bif @@ -1,4 +1,6 @@ -# Documentation and default values for these are located in policy/bro.init. +##! Declaration of various scripting-layer constants that the Bro core uses +##! internally. Documentation and default values for the scripting-layer +##! variables themselves are found in :doc:`/scripts/base/init-bare`. const ignore_keep_alive_rexmit: bool; const skip_http_data: bool; diff --git a/src/event.bif b/src/event.bif index f3354ea2e2..a54010f2e4 100644 --- a/src/event.bif +++ b/src/event.bif @@ -1,3 +1,8 @@ +##! The events that the C/C++ core of Bro can generate. This is mostly +##! consisting the high-level network events that protocol analyzers detect, +##! but there are also several general-utilit yevents generated by internal +##! Bro frameworks. + # # Documentation conventions: # diff --git a/src/strings.bif b/src/strings.bif index 5301dfcf5e..9c9bd576b1 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -1,4 +1,5 @@ -# Definitions of Bro built-in functions related to strings. +##! Definitions of built-in functions related to string processing and +##! manipulation. %%{ // C segment diff --git a/src/types.bif b/src/types.bif index da6bd6e031..4182c797e8 100644 --- a/src/types.bif +++ b/src/types.bif @@ -1,3 +1,4 @@ +##! Declaration of various types that the Bro core uses internally. enum dce_rpc_ptype %{ DCE_RPC_REQUEST, @@ -134,8 +135,8 @@ enum createmode_t %{ EXCLUSIVE = 2, %} -# Decleare record types that we want to access from the even engine. These are -# defined in bro.init. +# Decleare record types that we want to access from the event engine. These are +# defined in init-bare.bro. type info_t: record; type fattr_t: record; type diropargs_t: record; From fd74eb8e308915fd6766ec221acfd15633361568 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Mon, 9 Jan 2012 16:59:29 -0600 Subject: [PATCH 239/964] fixed a couple typos in comments --- src/event.bif | 2 +- src/types.bif | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/event.bif b/src/event.bif index a54010f2e4..aa40202f64 100644 --- a/src/event.bif +++ b/src/event.bif @@ -1,6 +1,6 @@ ##! The events that the C/C++ core of Bro can generate. This is mostly ##! consisting the high-level network events that protocol analyzers detect, -##! but there are also several general-utilit yevents generated by internal +##! but there are also several general-utility events generated by internal ##! Bro frameworks. # diff --git a/src/types.bif b/src/types.bif index 4182c797e8..4657584a90 100644 --- a/src/types.bif +++ b/src/types.bif @@ -135,7 +135,7 @@ enum createmode_t %{ EXCLUSIVE = 2, %} -# Decleare record types that we want to access from the event engine. These are +# Declare record types that we want to access from the event engine. These are # defined in init-bare.bro. type info_t: record; type fattr_t: record; From 26183a133987e7577690a130531ce360829ad80f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 9 Jan 2012 16:11:02 -0800 Subject: [PATCH 240/964] Updating submodule(s). [nomail] --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- cmake | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aux/binpac b/aux/binpac index e94d92b01f..3741d4f229 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit e94d92b01f327655fd2061157942b95ae75b5f0f +Subproject commit 3741d4f22933fa64d5194ebe4686d487c66458fa diff --git a/aux/bro-aux b/aux/bro-aux index f6b92bf573..bce1059c71 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit f6b92bf5732c26e54eb4387efadc612663980389 +Subproject commit bce1059c717685fd862928404efbbfe62b0a95c5 diff --git a/aux/broccoli b/aux/broccoli index c5cee3d574..20d0a88708 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit c5cee3d5746ed3d5c14348c1f264d19404caa761 +Subproject commit 20d0a887082837549aa530749c963db4ad204abd diff --git a/aux/broctl b/aux/broctl index d3d5934310..4d57e071be 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit d3d5934310a94452b1dddabb2e75f6c5c86b4860 +Subproject commit 4d57e071bec29a118d5629d1f15d673754c06b8b diff --git a/cmake b/cmake index 0c0a469768..ca4ed1a237 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 0c0a4697687df7f17c09391a1d0d95b25297a662 +Subproject commit ca4ed1a237215765ce9a7f2bc4b57b56958039ef From cbfe16db0e231967b3ef597aab6945cb7696fe49 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 9 Jan 2012 18:00:50 -0800 Subject: [PATCH 241/964] Updating baseline. --- CHANGES | 4 ++++ VERSION | 2 +- .../coverage.bare-load-baseline/canonified_loaded_scripts.log | 1 + .../canonified_loaded_scripts.log | 1 + 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index f53241d662..313ef02400 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.0-beta-185 | 2012-01-09 18:00:50 -0800 + + * Tweaks for OpenBSD support. (Jon Siwek) + 2.0-beta-181 | 2012-01-08 20:49:04 -0800 * Add SFTP log postprocessor that transfers logs to remote hosts. diff --git a/VERSION b/VERSION index 03c798ba2b..49980740c9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0-beta-181 +2.0-beta-185 diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 8fab67304e..d43367f300 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -17,5 +17,6 @@ scripts/base/init-bare.bro build/src/base/logging.bif.bro scripts/base/frameworks/logging/./postprocessors/__load__.bro scripts/base/frameworks/logging/./postprocessors/./scp.bro + scripts/base/frameworks/logging/./postprocessors/./sftp.bro scripts/base/frameworks/logging/./writers/ascii.bro scripts/policy/misc/loaded-scripts.bro diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 3f77797df8..92deb62edb 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -17,6 +17,7 @@ scripts/base/init-bare.bro build/src/base/logging.bif.bro scripts/base/frameworks/logging/./postprocessors/__load__.bro scripts/base/frameworks/logging/./postprocessors/./scp.bro + scripts/base/frameworks/logging/./postprocessors/./sftp.bro scripts/base/frameworks/logging/./writers/ascii.bro scripts/base/init-default.bro scripts/base/utils/site.bro From b10e02b691776b91ba735966aff5064d5b6531b8 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 9 Jan 2012 18:04:34 -0800 Subject: [PATCH 242/964] A set of script-reference polishing. Includes: - Quite a bit of CSS tweaking. - Some small changes to BroDoc.cc to change the order of sections. - A bit of tweaking some reST here and there. --- COPYING | 2 +- INSTALL | 7 +- aux/broccoli | 2 +- aux/broctl | 2 +- doc/.gitignore | 1 + doc/_static/960.css | 1 + doc/_static/basic.css | 513 +++++++++++++++++++++++++++++++++ doc/_static/broxygen-extra.css | 136 +++++++++ doc/_static/broxygen-extra.js | 0 doc/_static/broxygen.css | 437 ++++++++++++++++++++++++++++ doc/_static/download.js | 3 - doc/_static/pygments.css | 58 ++++ doc/_templates/layout.html | 10 +- doc/conf.py.in | 6 +- doc/ext/adapt-toc.py | 29 ++ doc/quickstart.rst | 38 ++- src/BroDoc.cc | 46 +-- src/bro.bif | 2 +- src/const.bif | 3 +- src/event.bif | 2 + src/strings.bif | 2 +- src/types.bif | 1 + 22 files changed, 1248 insertions(+), 53 deletions(-) create mode 100644 doc/_static/960.css create mode 100644 doc/_static/basic.css create mode 100644 doc/_static/broxygen-extra.js create mode 100644 doc/_static/broxygen.css delete mode 100644 doc/_static/download.js create mode 100644 doc/_static/pygments.css create mode 100644 doc/ext/adapt-toc.py diff --git a/COPYING b/COPYING index 5ae3c62e7a..7b0a94a03b 100644 --- a/COPYING +++ b/COPYING @@ -1,4 +1,4 @@ -Copyright (c) 1995-2011, The Regents of the University of California +Copyright (c) 1995-2012, The Regents of the University of California through the Lawrence Berkeley National Laboratory and the International Computer Science Institute. All rights reserved. diff --git a/INSTALL b/INSTALL index deb71c7b84..94e0993c3a 100644 --- a/INSTALL +++ b/INSTALL @@ -28,6 +28,7 @@ installation time: Bro also needs the following tools, but on most systems they will already come preinstalled: + * Bash (For Bro Control). * BIND8 (headers and libraries) * Bison (GNU Parser Generator) * Flex (Fast Lexical Analyzer) @@ -71,10 +72,8 @@ Running Bro =========== Bro is a complex program and it takes a bit of time to get familiar -with it. A good place for newcomers to start is the Quickstart Guide at - - http://www.bro-ids.org/documentation/quickstart.bro.html - +with it. A good place for newcomers to start is the Quickstart Guide +at http://www.bro-ids.org/documentation/quickstart.bro.html. For developers that wish to run Bro directly from the ``build/`` directory (i.e., without performing ``make install``), they will have diff --git a/aux/broccoli b/aux/broccoli index c5cee3d574..d7b8a43759 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit c5cee3d5746ed3d5c14348c1f264d19404caa761 +Subproject commit d7b8a43759bfcbe1381d132d8ab388937e52a6d4 diff --git a/aux/broctl b/aux/broctl index d3d5934310..a42e4d133b 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit d3d5934310a94452b1dddabb2e75f6c5c86b4860 +Subproject commit a42e4d133b94622c612055047f8534d5122e6e88 diff --git a/doc/.gitignore b/doc/.gitignore index 1936cc1d44..15972ee82a 100644 --- a/doc/.gitignore +++ b/doc/.gitignore @@ -1 +1,2 @@ html +*.pyc diff --git a/doc/_static/960.css b/doc/_static/960.css new file mode 100644 index 0000000000..22c5e18180 --- /dev/null +++ b/doc/_static/960.css @@ -0,0 +1 @@ +body{min-width:960px}.container_12,.container_16{margin-left:auto;margin-right:auto;width:960px}.grid_1,.grid_2,.grid_3,.grid_4,.grid_5,.grid_6,.grid_7,.grid_8,.grid_9,.grid_10,.grid_11,.grid_12,.grid_13,.grid_14,.grid_15,.grid_16{display:inline;float:left;margin-left:10px;margin-right:10px}.push_1,.pull_1,.push_2,.pull_2,.push_3,.pull_3,.push_4,.pull_4,.push_5,.pull_5,.push_6,.pull_6,.push_7,.pull_7,.push_8,.pull_8,.push_9,.pull_9,.push_10,.pull_10,.push_11,.pull_11,.push_12,.pull_12,.push_13,.pull_13,.push_14,.pull_14,.push_15,.pull_15{position:relative}.container_12 .grid_3,.container_16 .grid_4{width:220px}.container_12 .grid_6,.container_16 .grid_8{width:460px}.container_12 .grid_9,.container_16 .grid_12{width:700px}.container_12 .grid_12,.container_16 .grid_16{width:940px}.alpha{margin-left:0}.omega{margin-right:0}.container_12 .grid_1{width:60px}.container_12 .grid_2{width:140px}.container_12 .grid_4{width:300px}.container_12 .grid_5{width:380px}.container_12 .grid_7{width:540px}.container_12 .grid_8{width:620px}.container_12 .grid_10{width:780px}.container_12 .grid_11{width:860px}.container_16 .grid_1{width:40px}.container_16 .grid_2{width:100px}.container_16 .grid_3{width:160px}.container_16 .grid_5{width:280px}.container_16 .grid_6{width:340px}.container_16 .grid_7{width:400px}.container_16 .grid_9{width:520px}.container_16 .grid_10{width:580px}.container_16 .grid_11{width:640px}.container_16 .grid_13{width:760px}.container_16 .grid_14{width:820px}.container_16 .grid_15{width:880px}.container_12 .prefix_3,.container_16 .prefix_4{padding-left:240px}.container_12 .prefix_6,.container_16 .prefix_8{padding-left:480px}.container_12 .prefix_9,.container_16 .prefix_12{padding-left:720px}.container_12 .prefix_1{padding-left:80px}.container_12 .prefix_2{padding-left:160px}.container_12 .prefix_4{padding-left:320px}.container_12 .prefix_5{padding-left:400px}.container_12 .prefix_7{padding-left:560px}.container_12 .prefix_8{padding-left:640px}.container_12 .prefix_10{padding-left:800px}.container_12 .prefix_11{padding-left:880px}.container_16 .prefix_1{padding-left:60px}.container_16 .prefix_2{padding-left:120px}.container_16 .prefix_3{padding-left:180px}.container_16 .prefix_5{padding-left:300px}.container_16 .prefix_6{padding-left:360px}.container_16 .prefix_7{padding-left:420px}.container_16 .prefix_9{padding-left:540px}.container_16 .prefix_10{padding-left:600px}.container_16 .prefix_11{padding-left:660px}.container_16 .prefix_13{padding-left:780px}.container_16 .prefix_14{padding-left:840px}.container_16 .prefix_15{padding-left:900px}.container_12 .suffix_3,.container_16 .suffix_4{padding-right:240px}.container_12 .suffix_6,.container_16 .suffix_8{padding-right:480px}.container_12 .suffix_9,.container_16 .suffix_12{padding-right:720px}.container_12 .suffix_1{padding-right:80px}.container_12 .suffix_2{padding-right:160px}.container_12 .suffix_4{padding-right:320px}.container_12 .suffix_5{padding-right:400px}.container_12 .suffix_7{padding-right:560px}.container_12 .suffix_8{padding-right:640px}.container_12 .suffix_10{padding-right:800px}.container_12 .suffix_11{padding-right:880px}.container_16 .suffix_1{padding-right:60px}.container_16 .suffix_2{padding-right:120px}.container_16 .suffix_3{padding-right:180px}.container_16 .suffix_5{padding-right:300px}.container_16 .suffix_6{padding-right:360px}.container_16 .suffix_7{padding-right:420px}.container_16 .suffix_9{padding-right:540px}.container_16 .suffix_10{padding-right:600px}.container_16 .suffix_11{padding-right:660px}.container_16 .suffix_13{padding-right:780px}.container_16 .suffix_14{padding-right:840px}.container_16 .suffix_15{padding-right:900px}.container_12 .push_3,.container_16 .push_4{left:240px}.container_12 .push_6,.container_16 .push_8{left:480px}.container_12 .push_9,.container_16 .push_12{left:720px}.container_12 .push_1{left:80px}.container_12 .push_2{left:160px}.container_12 .push_4{left:320px}.container_12 .push_5{left:400px}.container_12 .push_7{left:560px}.container_12 .push_8{left:640px}.container_12 .push_10{left:800px}.container_12 .push_11{left:880px}.container_16 .push_1{left:60px}.container_16 .push_2{left:120px}.container_16 .push_3{left:180px}.container_16 .push_5{left:300px}.container_16 .push_6{left:360px}.container_16 .push_7{left:420px}.container_16 .push_9{left:540px}.container_16 .push_10{left:600px}.container_16 .push_11{left:660px}.container_16 .push_13{left:780px}.container_16 .push_14{left:840px}.container_16 .push_15{left:900px}.container_12 .pull_3,.container_16 .pull_4{left:-240px}.container_12 .pull_6,.container_16 .pull_8{left:-480px}.container_12 .pull_9,.container_16 .pull_12{left:-720px}.container_12 .pull_1{left:-80px}.container_12 .pull_2{left:-160px}.container_12 .pull_4{left:-320px}.container_12 .pull_5{left:-400px}.container_12 .pull_7{left:-560px}.container_12 .pull_8{left:-640px}.container_12 .pull_10{left:-800px}.container_12 .pull_11{left:-880px}.container_16 .pull_1{left:-60px}.container_16 .pull_2{left:-120px}.container_16 .pull_3{left:-180px}.container_16 .pull_5{left:-300px}.container_16 .pull_6{left:-360px}.container_16 .pull_7{left:-420px}.container_16 .pull_9{left:-540px}.container_16 .pull_10{left:-600px}.container_16 .pull_11{left:-660px}.container_16 .pull_13{left:-780px}.container_16 .pull_14{left:-840px}.container_16 .pull_15{left:-900px}.clear{clear:both;display:block;overflow:hidden;visibility:hidden;width:0;height:0}.clearfix:before,.clearfix:after{content:'\0020';display:block;overflow:hidden;visibility:hidden;width:0;height:0}.clearfix:after{clear:both}.clearfix{zoom:1} diff --git a/doc/_static/basic.css b/doc/_static/basic.css new file mode 100644 index 0000000000..1332c7b048 --- /dev/null +++ b/doc/_static/basic.css @@ -0,0 +1,513 @@ +/* + * basic.css + * ~~~~~~~~~ + * + * Sphinx stylesheet -- basic theme. + * + * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. + * :license: BSD, see LICENSE for details. + * + */ + +/* -- main layout ----------------------------------------------------------- */ + +div.clearer { + clear: both; +} + +/* -- relbar ---------------------------------------------------------------- */ + +div.related { + width: 100%; + font-size: 90%; +} + +div.related h3 { + display: none; +} + +div.related ul { + margin: 0; + padding: 0 0 0 10px; + list-style: none; +} + +div.related li { + display: inline; +} + +div.related li.right { + float: right; + margin-right: 5px; +} + +/* -- sidebar --------------------------------------------------------------- */ + +div.sphinxsidebarwrapper { + padding: 10px 5px 0 10px; +} + +div.sphinxsidebar { + float: left; + width: 230px; + margin-left: -100%; + font-size: 90%; +} + +div.sphinxsidebar ul { + list-style: none; +} + +div.sphinxsidebar ul ul, +div.sphinxsidebar ul.want-points { + margin-left: 20px; + list-style: square; +} + +div.sphinxsidebar ul ul { + margin-top: 0; + margin-bottom: 0; +} + +div.sphinxsidebar form { + margin-top: 10px; +} + +div.sphinxsidebar input { + border: 1px solid #98dbcc; + font-family: sans-serif; + font-size: 1em; +} + +div.sphinxsidebar input[type="text"] { + width: 170px; +} + +div.sphinxsidebar input[type="submit"] { + width: 30px; +} + +img { + border: 0; +} + +/* -- search page ----------------------------------------------------------- */ + +ul.search { + margin: 10px 0 0 20px; + padding: 0; +} + +ul.search li { + padding: 5px 0 5px 20px; + background-image: url(file.png); + background-repeat: no-repeat; + background-position: 0 7px; +} + +ul.search li a { + font-weight: bold; +} + +ul.search li div.context { + color: #888; + margin: 2px 0 0 30px; + text-align: left; +} + +ul.keywordmatches li.goodmatch a { + font-weight: bold; +} + +/* -- index page ------------------------------------------------------------ */ + +table.contentstable { + width: 90%; +} + +table.contentstable p.biglink { + line-height: 150%; +} + +a.biglink { + font-size: 1.3em; +} + +span.linkdescr { + font-style: italic; + padding-top: 5px; + font-size: 90%; +} + +/* -- general index --------------------------------------------------------- */ + +table.indextable { + width: 100%; +} + +table.indextable td { + text-align: left; + vertical-align: top; +} + +table.indextable dl, table.indextable dd { + margin-top: 0; + margin-bottom: 0; +} + +table.indextable tr.pcap { + height: 10px; +} + +table.indextable tr.cap { + margin-top: 10px; + background-color: #f2f2f2; +} + +img.toggler { + margin-right: 3px; + margin-top: 3px; + cursor: pointer; +} + +div.modindex-jumpbox { + border-top: 1px solid #ddd; + border-bottom: 1px solid #ddd; + margin: 1em 0 1em 0; + padding: 0.4em; +} + +div.genindex-jumpbox { + border-top: 1px solid #ddd; + border-bottom: 1px solid #ddd; + margin: 1em 0 1em 0; + padding: 0.4em; +} + +/* -- general body styles --------------------------------------------------- */ + +a.headerlink { + visibility: hidden; +} + +div.body p.caption { + text-align: inherit; +} + +div.body td { + text-align: left; +} + +.field-list ul { + padding-left: 1em; +} + +.first { + margin-top: 0 !important; +} + +p.rubric { + margin-top: 30px; + font-weight: bold; +} + +img.align-left, .figure.align-left, object.align-left { + clear: left; + float: left; + margin-right: 1em; +} + +img.align-right, .figure.align-right, object.align-right { + clear: right; + float: right; + margin-left: 1em; +} + +img.align-center, .figure.align-center, object.align-center { + display: block; + margin-left: auto; + margin-right: auto; +} + +.align-left { + text-align: left; +} + +.align-center { + text-align: center; +} + +.align-right { + text-align: right; +} + +/* -- sidebars -------------------------------------------------------------- */ + +div.sidebar { + margin: 0 0 0.5em 1em; + border: 1px solid #ddb; + padding: 7px 7px 0 7px; + background-color: #ffe; + width: 40%; + float: right; +} + +p.sidebar-title { + font-weight: bold; +} + +/* -- topics ---------------------------------------------------------------- */ + +div.topic { + border: 1px solid #ccc; + padding: 7px 7px 0 7px; + margin: 10px 0 10px 0; +} + +p.topic-title { + font-size: 1.1em; + font-weight: bold; + margin-top: 10px; +} + +/* -- admonitions ----------------------------------------------------------- */ + +div.admonition { + margin-top: 10px; + margin-bottom: 10px; + padding: 7px; +} + +div.admonition dt { + font-weight: bold; +} + +div.admonition dl { + margin-bottom: 0; +} + +p.admonition-title { + margin: 0px 10px 5px 0px; + font-weight: bold; +} + +div.body p.centered { + text-align: center; + margin-top: 25px; +} + +/* -- tables ---------------------------------------------------------------- */ + +table.field-list td, table.field-list th { + border: 0 !important; +} + +table.footnote td, table.footnote th { + border: 0 !important; +} + +th { + text-align: left; + padding-right: 5px; +} + +table.citation { + border-left: solid 1px gray; + margin-left: 1px; +} + +table.citation td { + border-bottom: none; +} + +/* -- other body styles ----------------------------------------------------- */ + +ol.arabic { + list-style: decimal; +} + +ol.loweralpha { + list-style: lower-alpha; +} + +ol.upperalpha { + list-style: upper-alpha; +} + +ol.lowerroman { + list-style: lower-roman; +} + +ol.upperroman { + list-style: upper-roman; +} + +dd p { + margin-top: 0px; +} + +dd ul, dd table { + margin-bottom: 10px; +} + +dd { + margin-top: 3px; + margin-bottom: 10px; + margin-left: 30px; +} + +dt:target, .highlighted { + background-color: #fbe54e; +} + +dl.glossary dt { + font-weight: bold; + font-size: 1.1em; +} + +.field-list ul { + margin: 0; + padding-left: 1em; +} + +.field-list p { + margin: 0; +} + +.refcount { + color: #060; +} + +.optional { + font-size: 1.3em; +} + +.versionmodified { + font-style: italic; +} + +.system-message { + background-color: #fda; + padding: 5px; + border: 3px solid red; +} + +.footnote:target { + background-color: #ffa; +} + +.line-block { + display: block; + margin-top: 1em; + margin-bottom: 1em; +} + +.line-block .line-block { + margin-top: 0; + margin-bottom: 0; + margin-left: 1.5em; +} + +.guilabel, .menuselection { + font-family: sans-serif; +} + +.accelerator { + text-decoration: underline; +} + +.classifier { + font-style: oblique; +} + +abbr, acronym { + border-bottom: dotted 1px; + cursor: help; +} + +/* -- code displays --------------------------------------------------------- */ + +pre { + overflow: auto; + overflow-y: hidden; /* fixes display issues on Chrome browsers */ +} + +td.linenos pre { + padding: 5px 0px; + border: 0; + background-color: transparent; + color: #aaa; +} + +table.highlighttable { + margin-left: 0.5em; +} + +table.highlighttable td { + padding: 0 0.5em 0 0.5em; +} + +tt.descname { + background-color: transparent; + font-weight: bold; +# font-size: 1.2em; +} + +tt.descclassname { + background-color: transparent; +} + +tt.xref, a tt { + background-color: transparent; +# font-weight: bold; +} + +h1 tt, h2 tt, h3 tt, h4 tt, h5 tt, h6 tt { + background-color: transparent; +} + +.viewcode-link { + float: right; +} + +.viewcode-back { + float: right; + font-family: sans-serif; +} + +div.viewcode-block:target { + margin: -1px -10px; + padding: 0 10px; +} + +/* -- math display ---------------------------------------------------------- */ + +img.math { + vertical-align: middle; +} + +div.body div.math p { + text-align: center; +} + +span.eqno { + float: right; +} + +/* -- printout stylesheet --------------------------------------------------- */ + +@media print { + div.document, + div.documentwrapper, + div.bodywrapper { + margin: 0 !important; + width: 100%; + } + + div.sphinxsidebar, + div.related, + div.footer, + #top-link { + display: none; + } +} diff --git a/doc/_static/broxygen-extra.css b/doc/_static/broxygen-extra.css index ec240cec7b..543d25ced2 100644 --- a/doc/_static/broxygen-extra.css +++ b/doc/_static/broxygen-extra.css @@ -1,3 +1,17 @@ + +a.toc-backref { + color: #333; +} + +h1, h2, h3, h4, h5, h6, +h1 a, h2 a, h3 a, h4 a, h5 a, h6 a { + padding:0 0 0px 0; +} + +ul { + padding-bottom: 0px; +} + h1 { font-weight: bold; font-size: 32px; @@ -14,3 +28,125 @@ th.field-name { white-space:nowrap; } + +h2 { + margin-top: 50px; + padding-bottom: 5px; + margin-bottom: 30px; + border-bottom: 1px solid; + border-color: #aaa; + font-style: normal; +} + +h3 { + font-size: 20px; + margin-top: 40px; + margin-bottom: 0¡px; + font-weight: normal; + font-style: normal; +} + +h3.widgettitle { + font-style: normal; +} + +h4 { + font-size:18px; + font-style: normal; + margin-bottom: 0em; + margin-top: 40px; + font-style: italic; +} + +h5 { + font-size:16px; +} + +h6 { + font-size:15px; +} + +.toc-backref { + color: #333; +} + +.contents ul { + padding-bottom: 1em; +} + +dl.namespace { + display: none; +} + +table.docutils tbody { + margin: 1em 1em 1em 1em; +} + +table.docutils td { + padding: 5pt 5pt 5pt 5pt; + font-size: 14px; + border-left: 0; + border-right: 0; +} + +dl pre { + font-size: 14px; +} + +table.docutils th { + padding: 5pt 5pt 5pt 5pt; + font-size: 14px; + font-style: normal; + border-left: 0; + border-right: 0; +} + +table.docutils tr:first-child td { + #border-top: 1px solid #aaa; +} + +.download { + font-family:"Courier New", Courier, mono; + font-weight: normal; +} + +dt:target, .highlighted { + background-color: #ccc; +} + +p { + padding-bottom: 0px; +} + +p.last { + margin-bottom: 0px; +} + +dl { + padding: 1em 1em 1em 1em; + background: #fffff0; + border: 1px solid #aaa; + +} + +dl { + margin-bottom: 10px; +} + + +table.docutils { + background: #fffff0; + border-collapse: collapse; + border: 1px solid #ddd; +} + +dl table.docutils { + border: 0; +} + +table.docutils dl { + border: 1px dashed #666; +} + + + diff --git a/doc/_static/broxygen-extra.js b/doc/_static/broxygen-extra.js new file mode 100644 index 0000000000..e69de29bb2 diff --git a/doc/_static/broxygen.css b/doc/_static/broxygen.css new file mode 100644 index 0000000000..967dcd6eaa --- /dev/null +++ b/doc/_static/broxygen.css @@ -0,0 +1,437 @@ +/* Automatically generated. Do not edit. */ + + + + + +#bro-main, #bro-standalone-main { + padding: 0 0 0 0; + position:relative; + z-index:1; +} + +#bro-main { + margin-bottom: 2em; + } + +#bro-standalone-main { + margin-bottom: 0em; + padding-left: 50px; + padding-right: 50px; + } + +#bro-outer { + color: #333; + background: #ffffff; +} + +#bro-title { + font-weight: bold; + font-size: 32px; + line-height:32px; + text-align: center; + padding-top: 3px; + margin-bottom: 30px; + font-family: Palatino,'Palatino Linotype',Georgia,serif;; + color: #000; + } + +.opening:first-letter { + font-size: 24px; + font-weight: bold; + letter-spacing: 0.05em; + } + +.opening { + font-size: 17px; +} + +.version { + text-align: right; + font-size: 12px; + color: #aaa; + line-height: 0; + height: 0; +} + +.git-info-version { + position: relative; + height: 2em; + top: -1em; + color: #ccc; + float: left; + font-size: 12px; +} + +.git-info-date { + position: relative; + height: 2em; + top: -1em; + color: #ccc; + float: right; + font-size: 12px; +} + +body { + font-family:Arial, Helvetica, sans-serif; + font-size:15px; + line-height:22px; + color: #333; + margin: 0px; +} + +h1, h2, h3, h4, h5, h6, +h1 a, h2 a, h3 a, h4 a, h5 a, h6 a { + padding:0 0 20px 0; + font-weight:bold; + text-decoration:none; +} + +div.section h3, div.section h4, div.section h5, div.section h6 { + font-style: italic; +} + +h1, h2 { + font-size:27px; + letter-spacing:-1px; +} + +h3 { + margin-top: 1em; + font-size:18px; +} + +h4 { + font-size:16px; +} + +h5 { + font-size:15px; +} + +h6 { + font-size:12px; +} + +p { + padding:0 0 20px 0; +} + +hr { + background:none; + height:1px; + line-height:1px; + border:0; + margin:0 0 20px 0; +} + +ul, ol { + margin:0 20px 20px 0; + padding-left:40px; +} + +ul.simple, ol.simple { + margin:0 0px 0px 0; +} + +blockquote { + margin:0 0 0 40px; +} + +strong, dfn { + font-weight:bold; +} + +em, dfn { + font-style:italic; +} + +sup, sub { + line-height:0; +} + +pre { + white-space:pre; +} + +pre, code, tt { + font-family:"Courier New", Courier, mono; +} + +dl { + margin: 0 0 20px 0; +} + +dl dt { + font-weight: bold; +} + +dd { + margin:0 0 20px 20px; +} + +small { + font-size:75%; +} + +a:link, +a:visited, +a:active +{ + color: #2a85a7; +} + +a:hover +{ + color:#c24444; +} + +h1, h2, h3, h4, h5, h6, +h1 a, h2 a, h3 a, h4 a, h5 a, h6 a +{ + color: #333; +} + +hr { + border-bottom:1px solid #ddd; +} + +pre { + color: #333; + background: #FFFAE2; + padding: 7px 5px 3px 5px; + margin-bottom: 25px; + margin-top: 0px; +} + +ul { + padding-bottom: 5px; + } + +h1, h2 { + margin-top: 30px; + } + +h1 { + margin-bottom: 50px; + margin-bottom: 20px; + padding-bottom: 5px; + border-bottom: 1px solid; + border-color: #aaa; + } + +h2 { + font-size: 24px; + } + +pre { + -moz-box-shadow:0 0 6px #ddd; + -webkit-box-shadow:0 0 6px #ddd; + box-shadow:0 0 6px #ddd; +} + +a { + text-decoration:none; + } + +p { + padding-bottom: 15px; + } + +p, dd, li { + text-align: justify; + } + +li { + margin-bottom: 5px; + } + + + +#footer .widget_links ul a, +#footer .widget_links ol a +{ + color: #ddd; +} + +#footer .widget_links ul a:hover, +#footer .widget_links ol a:hover +{ + color:#c24444; +} + + +#footer .widget li { + padding-bottom:10px; +} + +#footer .widget_links li { + padding-bottom:1px; +} + +#footer .widget li:last-child { + padding-bottom:0; +} + +#footer .widgettitle { + color: #ddd; +} + + +.widget { + margin:0 0 40px 0; +} + +.widget, .widgettitle { + font-size:12px; + line-height:18px; +} + +.widgettitle { + font-weight:bold; + text-transform:uppercase; + padding:0 0 10px 0; + margin:0 0 20px 0; + line-height:100%; +} + +.widget UL, .widget OL { + list-style-type:none; + margin:0; + padding:0; +} + +.widget p { + padding:0; +} + +.widget li { + padding-bottom:10px; +} + +.widget a { + text-decoration:none; +} + +#bro-main .widgettitle, +{ + color: #333; +} + + +.widget img.left { + padding:5px 10px 10px 0; +} + +.widget img.right { + padding:5px 0 10px 10px; +} + +.ads .widgettitle { + margin-right:16px; +} + +.widget { + margin-left: 1em; +} + +.widgettitle { + color: #333; +} + +.widgettitle { + border-bottom:1px solid #ddd; +} + + +.sidebar-toc ul li { + padding-bottom: 0px; + text-align: left; + list-style-type: square; + list-style-position: inside; + padding-left: 1em; + text-indent: -1em; + } + +.sidebar-toc ul li li { + margin-left: 1em; + margin-bottom: 0px; + list-style-type: square; + } + +.sidebar-toc ul li li a { + font-size: 8pt; +} + +.contents { + padding: 10px; + background: #FFFAE2; + margin: 20px; + } + +.topic-title { + font-size: 20px; + font-weight: bold; + padding: 0px 0px 5px 0px; + text-align: center; + padding-top: .5em; +} + +.contents li { + margin-bottom: 0px; + list-style-type: square; +} + +.contents ul ul li { + margin-left: 0px; + padding-left: 0px; + padding-top: 0em; + font-size: 90%; + list-style-type: square; + font-weight: normal; +} + +.contents ul ul ul li { + list-style-type: none; +} + +.contents ul ul ul ul li { + display:none; +} + +.contents ul li { + padding-top: 1em; + list-style-type: none; + font-weight: bold; +} + +.contents ul { + margin-left: 0px; + padding-left: 2em; + margin: 0px 0px 0px 0px; +} + +.note, .warning, .error { + margin-left: 2em; + margin-right: 2em; + margin-top: 1.5em; + margin-bottom: 1.5em; + padding: 0.5em 1em 0.5em 1em; + overflow: auto; + border-left: solid 3px #aaa; + font-size: 15px; + color: #333; +} + +.admonition p { + margin-left: 1em; + } + +.admonition-title { + font-size: 16px; + font-weight: bold; + color: #000; + padding-bottom: 0em; + margin-bottom: .5em; + margin-top: 0em; +} \ No newline at end of file diff --git a/doc/_static/download.js b/doc/_static/download.js deleted file mode 100644 index 82bfe502cb..0000000000 --- a/doc/_static/download.js +++ /dev/null @@ -1,3 +0,0 @@ -$(document).ready(function() { - $('.docutils.download').removeClass('download'); -}); diff --git a/doc/_static/pygments.css b/doc/_static/pygments.css new file mode 100644 index 0000000000..3c96f6ae4e --- /dev/null +++ b/doc/_static/pygments.css @@ -0,0 +1,58 @@ +.hll { background-color: #ffffcc } +.c { color: #aaaaaa; font-style: italic } /* Comment */ +.err { color: #F00000; background-color: #F0A0A0 } /* Error */ +.k { color: #0000aa } /* Keyword */ +.cm { color: #aaaaaa; font-style: italic } /* Comment.Multiline */ +.cp { color: #4c8317 } /* Comment.Preproc */ +.c1 { color: #aaaaaa; font-style: italic } /* Comment.Single */ +.cs { color: #0000aa; font-style: italic } /* Comment.Special */ +.gd { color: #aa0000 } /* Generic.Deleted */ +.ge { font-style: italic } /* Generic.Emph */ +.gr { color: #aa0000 } /* Generic.Error */ +.gh { color: #000080; font-weight: bold } /* Generic.Heading */ +.gi { color: #00aa00 } /* Generic.Inserted */ +.go { color: #888888 } /* Generic.Output */ +.gp { color: #555555 } /* Generic.Prompt */ +.gs { font-weight: bold } /* Generic.Strong */ +.gu { color: #800080; font-weight: bold } /* Generic.Subheading */ +.gt { color: #aa0000 } /* Generic.Traceback */ +.kc { color: #0000aa } /* Keyword.Constant */ +.kd { color: #0000aa } /* Keyword.Declaration */ +.kn { color: #0000aa } /* Keyword.Namespace */ +.kp { color: #0000aa } /* Keyword.Pseudo */ +.kr { color: #0000aa } /* Keyword.Reserved */ +.kt { color: #00aaaa } /* Keyword.Type */ +.m { color: #009999 } /* Literal.Number */ +.s { color: #aa5500 } /* Literal.String */ +.na { color: #1e90ff } /* Name.Attribute */ +.nb { color: #00aaaa } /* Name.Builtin */ +.nc { color: #00aa00; text-decoration: underline } /* Name.Class */ +.no { color: #aa0000 } /* Name.Constant */ +.nd { color: #888888 } /* Name.Decorator */ +.ni { color: #800000; font-weight: bold } /* Name.Entity */ +.nf { color: #00aa00 } /* Name.Function */ +.nn { color: #00aaaa; text-decoration: underline } /* Name.Namespace */ +.nt { color: #1e90ff; font-weight: bold } /* Name.Tag */ +.nv { color: #aa0000 } /* Name.Variable */ +.ow { color: #0000aa } /* Operator.Word */ +.w { color: #bbbbbb } /* Text.Whitespace */ +.mf { color: #009999 } /* Literal.Number.Float */ +.mh { color: #009999 } /* Literal.Number.Hex */ +.mi { color: #009999 } /* Literal.Number.Integer */ +.mo { color: #009999 } /* Literal.Number.Oct */ +.sb { color: #aa5500 } /* Literal.String.Backtick */ +.sc { color: #aa5500 } /* Literal.String.Char */ +.sd { color: #aa5500 } /* Literal.String.Doc */ +.s2 { color: #aa5500 } /* Literal.String.Double */ +.se { color: #aa5500 } /* Literal.String.Escape */ +.sh { color: #aa5500 } /* Literal.String.Heredoc */ +.si { color: #aa5500 } /* Literal.String.Interpol */ +.sx { color: #aa5500 } /* Literal.String.Other */ +.sr { color: #009999 } /* Literal.String.Regex */ +.s1 { color: #aa5500 } /* Literal.String.Single */ +.ss { color: #0000aa } /* Literal.String.Symbol */ +.bp { color: #00aaaa } /* Name.Builtin.Pseudo */ +.vc { color: #aa0000 } /* Name.Variable.Class */ +.vg { color: #aa0000 } /* Name.Variable.Global */ +.vi { color: #aa0000 } /* Name.Variable.Instance */ +.il { color: #009999 } /* Literal.Number.Integer.Long */ diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html index a4775f5870..77d9d1de1c 100644 --- a/doc/_templates/layout.html +++ b/doc/_templates/layout.html @@ -1,11 +1,12 @@ {% extends "!layout.html" %} {% block extrahead %} - - - + + + - + + {% endblock %} {% block header %} @@ -47,6 +48,7 @@ Table of Contents

+

    {{toc}}

diff --git a/doc/conf.py.in b/doc/conf.py.in index 0e103b719f..2e93e82502 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -24,7 +24,7 @@ sys.path.insert(0, os.path.abspath('sphinx-sources/ext')) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['bro', 'rst_directive', 'sphinx.ext.todo'] +extensions = ['bro', 'rst_directive', 'sphinx.ext.todo', 'adapt-toc'] # Add any paths that contain templates here, relative to this directory. templates_path = ['sphinx-sources/_templates', 'sphinx-sources/_static'] @@ -40,7 +40,7 @@ master_doc = 'index' # General information about the project. project = u'Bro' -copyright = u'2011, The Bro Project' +copyright = u'2012, The Bro Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -169,6 +169,7 @@ html_sidebars = { # Output file base name for HTML help builder. htmlhelp_basename = 'Broxygen' +html_add_permalinks = None # -- Options for LaTeX output -------------------------------------------------- @@ -208,7 +209,6 @@ latex_documents = [ # If false, no module index is generated. #latex_domain_indices = True - # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples diff --git a/doc/ext/adapt-toc.py b/doc/ext/adapt-toc.py new file mode 100644 index 0000000000..12ee006977 --- /dev/null +++ b/doc/ext/adapt-toc.py @@ -0,0 +1,29 @@ + +import sys +import re + +# Removes the first TOC level, which is just the page title. +def process_html_toc(app, pagename, templatename, context, doctree): + + if not "toc" in context: + return + + toc = context["toc"] + + lines = toc.strip().split("\n") + lines = lines[2:-2] + + toc = "\n".join(lines) + toc = "
    " + toc + + context["toc"] = toc + + # print >>sys.stderr, pagename + # print >>sys.stderr, context["toc"] + # print >>sys.stderr, "-----" + # print >>sys.stderr, toc + # print >>sys.stderr, "====" + +def setup(app): + app.connect('html-page-context', process_html_toc) + diff --git a/doc/quickstart.rst b/doc/quickstart.rst index 0534186cde..da780e70df 100644 --- a/doc/quickstart.rst +++ b/doc/quickstart.rst @@ -31,15 +31,15 @@ See the `bro downloads page`_ for currently supported/targeted platforms. * RPM -.. console:: + .. console:: - sudo yum localinstall Bro-all*.rpm + sudo yum localinstall Bro-all*.rpm * DEB -.. console:: + .. console:: - sudo gdebi Bro-all-*.deb + sudo gdebi Bro-all-*.deb * MacOS Disk Image with Installer @@ -56,26 +56,32 @@ Building From Source Required Dependencies ~~~~~~~~~~~~~~~~~~~~~ +The following dependencies are required to build Bro: + * RPM/RedHat-based Linux: -.. console:: + .. console:: - sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel file-devel + sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel file-devel * DEB/Debian-based Linux: -.. console:: + .. console:: - sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev libmagic-dev + sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev libmagic-dev * FreeBSD Most required dependencies should come with a minimal FreeBSD install except for the following. -.. console:: + .. console:: - sudo pkg_add -r cmake swig bison python + sudo pkg_add -r bash cmake swig bison python + + Note that ``bash`` needs to be in ``PATH``, which by default it is + not. The FreeBSD package installs the binary into + ``/usr/local/bin``. * Mac OS X @@ -99,21 +105,21 @@ sending emails. * RPM/RedHat-based Linux: -.. console:: + .. console:: - sudo yum install GeoIP-devel sendmail + sudo yum install GeoIP-devel sendmail * DEB/Debian-based Linux: -.. console:: + .. console:: - sudo apt-get install libgeoip-dev sendmail + sudo apt-get install libgeoip-dev sendmail * Ports-based FreeBSD -.. console:: + .. console:: - sudo pkg_add -r GeoIP + sudo pkg_add -r GeoIP sendmail is typically already available. diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 91aed5ec62..b20db727ff 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -170,13 +170,26 @@ void BroDoc::WriteDocFile() const { WriteToDoc(".. Automatically generated. Do not edit.\n\n"); + WriteToDoc(":tocdepth: 3\n\n"); + WriteSectionHeading(doc_title.c_str(), '='); - WriteToDoc("\n:download:`Original Source File <%s>`\n\n", - downloadable_filename.c_str()); + WriteStringList(".. bro:namespace:: %s\n", modules); - WriteSectionHeading("Overview", '-'); - WriteStringList("%s\n", "%s\n\n", summary); + WriteToDoc("\n"); + + // WriteSectionHeading("Overview", '-'); + WriteStringList("%s\n", summary); + + WriteToDoc("\n"); + + if ( ! modules.empty() ) + { + WriteToDoc(":Namespace%s: ", (modules.size() > 1 ? "s" : "")); + // WriteStringList(":bro:namespace:`%s`", modules); + WriteStringList("``%s``, ", "``%s``", modules); + WriteToDoc("\n"); + } if ( ! imports.empty() ) { @@ -196,39 +209,38 @@ void BroDoc::WriteDocFile() const WriteToDoc("\n"); } + WriteToDoc(":Source File: :download:`%s`\n", + downloadable_filename.c_str()); + WriteToDoc("\n"); WriteInterface("Summary", '~', '#', true, true); - if ( ! modules.empty() ) - { - WriteSectionHeading("Namespaces", '~'); - WriteStringList(".. bro:namespace:: %s\n", modules); - WriteToDoc("\n"); - } - if ( ! notices.empty() ) - WriteBroDocObjList(notices, "Notices", '~'); + WriteBroDocObjList(notices, "Notices", '#'); - WriteInterface("Public Interface", '-', '~', true, false); + if ( port_analysis.size() || packet_filter.size() ) + WriteSectionHeading("Configuration Changes", '#'); if ( ! port_analysis.empty() ) { - WriteSectionHeading("Port Analysis", '-'); + WriteSectionHeading("Port Analysis", '^'); WriteToDoc("Loading this script makes the following changes to " ":bro:see:`dpd_config`.\n\n"); - WriteStringList("%s", port_analysis); + WriteStringList("%s, ", "%s", port_analysis); } if ( ! packet_filter.empty() ) { - WriteSectionHeading("Packet Filter", '-'); + WriteSectionHeading("Packet Filter", '^'); WriteToDoc("Loading this script makes the following changes to " ":bro:see:`capture_filters`.\n\n"); WriteToDoc("Filters added::\n\n"); WriteToDoc("%s\n", packet_filter.c_str()); } + WriteInterface("Detailed Interface", '~', '#', true, false); + #if 0 // Disabled for now. BroDocObjList::const_iterator it; bool hasPrivateIdentifiers = false; @@ -243,7 +255,7 @@ void BroDoc::WriteDocFile() const } if ( hasPrivateIdentifiers ) - WriteInterface("Private Interface", '-', '~', false, false); + WriteInterface("Private Interface", '~', '#', false, false); #endif } diff --git a/src/bro.bif b/src/bro.bif index c941d19c3b..4b1e19fe02 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1,4 +1,4 @@ -# Definitions of Bro built-in functions. +##! Defines most of the built-in functions accessible to the scripting layer. %%{ // C segment #include diff --git a/src/const.bif b/src/const.bif index 96630e300b..f1e59f185c 100644 --- a/src/const.bif +++ b/src/const.bif @@ -1,4 +1,5 @@ -# Documentation and default values for these are located in policy/bro.init. +##! Constants of used with built-in functions and events. Documented in +##! :doc:`/scripts/base/init-bare`. const ignore_keep_alive_rexmit: bool; const skip_http_data: bool; diff --git a/src/event.bif b/src/event.bif index abdf034719..3ae101a29e 100644 --- a/src/event.bif +++ b/src/event.bif @@ -1,3 +1,5 @@ +##! Events generated by Bro's internal event engine. + # # Documentation conventions: # diff --git a/src/strings.bif b/src/strings.bif index 5301dfcf5e..8fccf68690 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -1,4 +1,4 @@ -# Definitions of Bro built-in functions related to strings. +##! Built-in functions for working with strings. %%{ // C segment diff --git a/src/types.bif b/src/types.bif index da6bd6e031..401e102579 100644 --- a/src/types.bif +++ b/src/types.bif @@ -1,3 +1,4 @@ +##! Set of predefined types used with built-in functions and events. enum dce_rpc_ptype %{ DCE_RPC_REQUEST, From 5ff7afaadbd190ee347ba294920ee0cb65c79e81 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 9 Jan 2012 18:28:43 -0800 Subject: [PATCH 243/964] Tweaks. --- doc/_static/broxygen-extra.css | 10 +++++++++- src/event.bif | 31 +++++++------------------------ 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/doc/_static/broxygen-extra.css b/doc/_static/broxygen-extra.css index 543d25ced2..051e12e0be 100644 --- a/doc/_static/broxygen-extra.css +++ b/doc/_static/broxygen-extra.css @@ -38,11 +38,15 @@ h2 { font-style: normal; } +div.section h3 { + font-style: normal; + } + h3 { font-size: 20px; margin-top: 40px; margin-bottom: 0¡px; - font-weight: normal; + font-weight: bold; font-style: normal; } @@ -78,6 +82,10 @@ dl.namespace { display: none; } +dl dt { + font-weight: normal; +} + table.docutils tbody { margin: 1em 1em 1em 1em; } diff --git a/src/event.bif b/src/event.bif index 984d2ef307..001f0b84f1 100644 --- a/src/event.bif +++ b/src/event.bif @@ -4668,45 +4668,28 @@ event ssl_established%(c: connection%); ## defined as part of the SSL/TLS protocol. ## ## .. bro:see:: ssl_client_hello ssl_established ssl_extension ssl_server_hello -<<<<<<< HEAD ## ssl_session_ticket_handshake x509_certificate x509_error x509_extension event ssl_alert%(c: connection, is_orig: bool, level: count, desc: count%); -## Generated for SSL/TLS handshake. SSL/TLS sessions start with an unencrypted +## Generated for SSL/TLS handshake messages that are a part of the stateless-server +## session resumption mechanism. SSL/TLS sessions start with an unencrypted ## handshake, and Bro extracts as much information out of that as it can. This ## event is raised when an SSL/TLS server passes session ticket to the client that -## can later be used for resuming the session. +## can later be used for resuming the session. The mechanism is described in +## :rfc:`4507` ## ## See `Wikipedia `__ for ## more information about the SSL/TLS protocol. ## ## c: The connection. ## -## ticket_lifetime_hint: A hint to the client indicating how long the ticket should -## be valid. +## ticket_lifetime_hint: A hint from the server about how long the ticket +## should be stored by the client. ## -## ticket: The raw ticket. +## ticket: The raw ticket data. ## ## .. bro:see:: ssl_client_hello ssl_established ssl_extension ssl_server_hello ## x509_certificate x509_error x509_extension ssl_alert -======= -## x509_certificate x509_error x509_extension -event ssl_alert%(c: connection, is_orig: bool, level: count, desc: count%); - -## Generated for SSL/TLS session ticket handshake messages that are a part -## of the stateless-server session resumption mechanism as described by -## :rfc:`4507`. -## -## c: The connection. -## -## ticket_lifetime_hint: A hint from the server about how long the ticket -## should be stored by the client. -## -## ticket: The ticket data. -## -## .. bro:see:: ssl_client_hello ssl_established ssl_extension ssl_server_hello -## x509_certificate x509_error x509_extension ->>>>>>> fd74eb8e308915fd6766ec221acfd15633361568 event ssl_session_ticket_handshake%(c: connection, ticket_lifetime_hint: count, ticket: string%); ## Generated for x509 certificates seen in SSL/TLS connections. During the initial From 6d3d289d5cf9d692cd2c6585994f63f8adb7b721 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 9 Jan 2012 22:50:49 -0500 Subject: [PATCH 244/964] Fix some documentation errors. --- src/bro.bif | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index c941d19c3b..bc948dd562 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -2197,11 +2197,11 @@ function addr_to_count%(a: addr%): count return new Val(ntohl(addr), TYPE_COUNT); %} -## Converts a :bro:type:`addr` to a :bro:type:`count`. +## Converts a :bro:type:`port` to a :bro:type:`count`. ## -## a: The :bro:type:`addr` to convert. +## p: The :bro:type:`port` to convert. ## -## Returns: The :bro:type:`addr` *a* as :bro:type:`count`. +## Returns: The :bro:type:`port` *p* as :bro:type:`count`. ## ## .. bro:see:: count_to_port function port_to_count%(p: port%): count @@ -2211,9 +2211,9 @@ function port_to_count%(p: port%): count ## Converts a :bro:type:`count` and ``transport_proto`` to a :bro:type:`port`. ## -## a: The :bro:type:`addr` to convert. +## c: The :bro:type:`count` to convert. ## -## Returns: The :bro:type:`addr` *a* as :bro:type:`count`. +## Returns: The :bro:type:`count` *c* as :bro:type:`port`. ## ## .. bro:see:: port_to_count function count_to_port%(c: count, t: transport_proto%): port From 8ab372ccff474fbaffde8870f8cc47ec8900b94a Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 9 Jan 2012 22:53:53 -0500 Subject: [PATCH 245/964] Adding the draft SSL extension type next_protocol_negotiation. --- scripts/base/protocols/ssl/consts.bro | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/base/protocols/ssl/consts.bro b/scripts/base/protocols/ssl/consts.bro index 9d8bc68fd5..b0656eab58 100644 --- a/scripts/base/protocols/ssl/consts.bro +++ b/scripts/base/protocols/ssl/consts.bro @@ -69,6 +69,7 @@ export { [13] = "signature_algorithms", [14] = "use_srtp", [35] = "SessionTicket TLS", + [13172] = "next_protocol_negotiation", [65281] = "renegotiation_info" } &default=function(i: count):string { return fmt("unknown-%d", i); }; From 911d7d843618bcacb96792ff604ecfb565943df9 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 10 Jan 2012 00:56:12 -0500 Subject: [PATCH 246/964] Finished SSL & syslog autodocs. --- scripts/base/protocols/ssl/consts.bro | 29 ++++++++----- scripts/base/protocols/ssl/main.bro | 41 +++++++++++++------ scripts/base/protocols/syslog/consts.bro | 6 ++- scripts/base/protocols/syslog/main.bro | 10 +++-- scripts/policy/protocols/ssl/cert-hash.bro | 3 +- .../policy/protocols/ssl/expiring-certs.bro | 9 ++-- .../protocols/ssl/extract-certs-pem.bro | 10 ++--- scripts/policy/protocols/ssl/known-certs.bro | 5 ++- .../policy/protocols/ssl/validate-certs.bro | 3 +- 9 files changed, 74 insertions(+), 42 deletions(-) diff --git a/scripts/base/protocols/ssl/consts.bro b/scripts/base/protocols/ssl/consts.bro index b0656eab58..1568ff17d3 100644 --- a/scripts/base/protocols/ssl/consts.bro +++ b/scripts/base/protocols/ssl/consts.bro @@ -1,23 +1,27 @@ module SSL; export { - const SSLv2 = 0x0002; const SSLv3 = 0x0300; const TLSv10 = 0x0301; const TLSv11 = 0x0302; + ## Mapping between the constants and string values for SSL/TLS versions. const version_strings: table[count] of string = { [SSLv2] = "SSLv2", [SSLv3] = "SSLv3", [TLSv10] = "TLSv10", [TLSv11] = "TLSv11", } &default="UNKNOWN"; - + + ## Mapping between numeric codes and human readable strings for alert + ## levels. const alert_levels: table[count] of string = { [1] = "warning", [2] = "fatal", } &default=function(i: count):string { return fmt("unknown-%d", i); }; - + + ## Mapping between numeric codes and human readable strings for alert + ## descriptions.. const alert_descriptions: table[count] of string = { [0] = "close_notify", [10] = "unexpected_message", @@ -50,8 +54,11 @@ export { [114] = "bad_certificate_hash_value", [115] = "unknown_psk_identity", } &default=function(i: count):string { return fmt("unknown-%d", i); }; - - # http://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xml + + ## Mapping between numeric codes and human readable strings for SSL/TLS + ## extensions. + ## ..note: More information can be found here: + ## http://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xml const extensions: table[count] of string = { [0] = "server_name", [1] = "max_fragment_length", @@ -299,11 +306,10 @@ export { const SSL_RSA_WITH_DES_CBC_MD5 = 0xFF82; const SSL_RSA_WITH_3DES_EDE_CBC_MD5 = 0xFF83; const TLS_EMPTY_RENEGOTIATION_INFO_SCSV = 0x00FF; - - # --- This is a table of all known cipher specs. - # --- It can be used for detecting unknown ciphers and for - # --- converting the cipher spec constants into a human readable format. - + + ## This is a table of all known cipher specs. It can be used for + ## detecting unknown ciphers and for converting the cipher spec constants + ## into a human readable format. const cipher_desc: table[count] of string = { # --- sslv20 --- [SSLv20_CK_RC4_128_EXPORT40_WITH_MD5] = @@ -530,7 +536,8 @@ export { [SSL_RSA_FIPS_WITH_DES_CBC_SHA_2] = "SSL_RSA_FIPS_WITH_DES_CBC_SHA_2", [SSL_RSA_FIPS_WITH_3DES_EDE_CBC_SHA_2] = "SSL_RSA_FIPS_WITH_3DES_EDE_CBC_SHA_2", } &default="UNKNOWN"; - + + ## Mapping between the constants and string values for SSL/TLS errors. const x509_errors: table[count] of string = { [0] = "ok", [1] = "unable to get issuer cert", diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 4c6721fcd8..0b280a6bcf 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -1,4 +1,5 @@ -##! Base SSL analysis script. +##! Base SSL analysis script. This script logs information about the SSL/TLS +##! handshaking and encryption establishment process. @load ./consts @@ -12,44 +13,53 @@ export { ts: time &log; uid: string &log; id: conn_id &log; + ## SSL/TLS version the server offered. version: string &log &optional; + ## SSL/TLS cipher suite the server chose. cipher: string &log &optional; + ## Value of the Server Name Indicator SSL/TLS extension. It + ## indicates the server name that the client was requesting. server_name: string &log &optional; + ## Session ID offered by the client for session resumption. session_id: string &log &optional; + ## Subject of the X.509 certificate offered by the server. subject: string &log &optional; + ## NotValidBefore field value from the server certificate. not_valid_before: time &log &optional; + ## NotValidAfter field value from the serve certificate. not_valid_after: time &log &optional; + ## Last alert that was seen during the connection. last_alert: string &log &optional; - + + ## Full binary server certificate stored in DER format. cert: string &optional; + ## Chain of certificates offered by the server to validate its + ## complete signing chain. cert_chain: vector of string &optional; - ## This stores the analyzer id used for the analyzer instance attached + ## The analyzer ID used for the analyzer instance attached ## to each connection. It is not used for logging since it's a ## meaningless arbitrary number. analyzer_id: count &optional; }; - - ## This is where the default root CA bundle is defined. By loading the + + ## The default root CA bundle. By loading the ## mozilla-ca-list.bro script it will be set to Mozilla's root CA list. const root_certs: table[string] of string = {} &redef; - + ## If true, detach the SSL analyzer from the connection to prevent ## continuing to process encrypted traffic. Helps with performance ## (especially with large file transfers). const disable_analyzer_after_detection = T &redef; - + ## The openssl command line utility. If it's in the path the default ## value will work, otherwise a full path string can be supplied for the ## utility. const openssl_util = "openssl" &redef; - + + ## Event that can be handled to access the SSL + ## record as it is sent on to the logging framework. global log_ssl: event(rec: Info); - - const ports = { - 443/tcp, 563/tcp, 585/tcp, 614/tcp, 636/tcp, - 989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp - } &redef; } redef record connection += { @@ -76,6 +86,11 @@ redef capture_filters += { ["xmpps"] = "tcp port 5223", }; +const ports = { + 443/tcp, 563/tcp, 585/tcp, 614/tcp, 636/tcp, + 989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp +}; + redef dpd_config += { [[ANALYZER_SSL]] = [$ports = ports] }; diff --git a/scripts/base/protocols/syslog/consts.bro b/scripts/base/protocols/syslog/consts.bro index f08e7f71d7..dce1877ecf 100644 --- a/scripts/base/protocols/syslog/consts.bro +++ b/scripts/base/protocols/syslog/consts.bro @@ -1,6 +1,9 @@ +##! Constants definitions for syslog. + module Syslog; export { + ## Mapping between the constants and string values for syslog facilities. const facility_codes: table[count] of string = { [0] = "KERN", [1] = "USER", @@ -27,7 +30,8 @@ export { [22] = "LOCAL6", [23] = "LOCAL7", } &default=function(c: count): string { return fmt("?-%d", c); }; - + + ## Mapping between the constants and string values for syslog severities. const severity_codes: table[count] of string = { [0] = "EMERG", [1] = "ALERT", diff --git a/scripts/base/protocols/syslog/main.bro b/scripts/base/protocols/syslog/main.bro index f4afa6d34d..79f89d5e71 100644 --- a/scripts/base/protocols/syslog/main.bro +++ b/scripts/base/protocols/syslog/main.bro @@ -1,4 +1,5 @@ -##! Core script support for logging syslog messages. +##! Core script support for logging syslog messages. This script represents +##! one syslog message as one logged record. @load ./consts @@ -12,16 +13,19 @@ export { ts: time &log; uid: string &log; id: conn_id &log; + ## Protocol over which the message was seen. proto: transport_proto &log; + ## Syslog facility for the message. facility: string &log; + ## Syslog severity for the message. severity: string &log; + ## The plain text message. message: string &log; }; - - const ports = { 514/udp } &redef; } redef capture_filters += { ["syslog"] = "port 514" }; +const ports = { 514/udp } &redef; redef dpd_config += { [ANALYZER_SYSLOG_BINPAC] = [$ports = ports] }; redef likely_server_ports += { 514/udp }; diff --git a/scripts/policy/protocols/ssl/cert-hash.bro b/scripts/policy/protocols/ssl/cert-hash.bro index 1e47ccac2e..32a165a946 100644 --- a/scripts/policy/protocols/ssl/cert-hash.bro +++ b/scripts/policy/protocols/ssl/cert-hash.bro @@ -1,4 +1,4 @@ -##! This script calculates MD5 sums for server DER formatted certificates. +##! Calculate MD5 sums for server DER formatted certificates. @load base/protocols/ssl @@ -6,6 +6,7 @@ module SSL; export { redef record Info += { + ## MD5 sum of the raw server certificate. cert_hash: string &log &optional; }; } diff --git a/scripts/policy/protocols/ssl/expiring-certs.bro b/scripts/policy/protocols/ssl/expiring-certs.bro index 374065c286..80616e6a99 100644 --- a/scripts/policy/protocols/ssl/expiring-certs.bro +++ b/scripts/policy/protocols/ssl/expiring-certs.bro @@ -1,6 +1,6 @@ -##! This script can be used to generate notices when X.509 certificates over -##! SSL/TLS are expired or going to expire based on the date and time values -##! stored within the certificate. +##! Generate notices when X.509 certificates over SSL/TLS are expired or +##! going to expire soon based on the date and time values stored within the +##! certificate. @load base/protocols/ssl @load base/frameworks/notice @@ -24,7 +24,8 @@ export { ## The category of hosts you would like to be notified about which have ## certificates that are going to be expiring soon. By default, these - ## notices will be suppressed by the notice framework for 1 day. + ## notices will be suppressed by the notice framework for 1 day after + ## a particular certificate has had a notice generated. ## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS const notify_certs_expiration = LOCAL_HOSTS &redef; diff --git a/scripts/policy/protocols/ssl/extract-certs-pem.bro b/scripts/policy/protocols/ssl/extract-certs-pem.bro index e6a740c215..420c60a4fd 100644 --- a/scripts/policy/protocols/ssl/extract-certs-pem.bro +++ b/scripts/policy/protocols/ssl/extract-certs-pem.bro @@ -2,7 +2,7 @@ ##! after being converted to PEM files. The certificates will be stored in ##! a single file, one for local certificates and one for remote certificates. ##! -##! A couple of things to think about with this script:: +##! ..note:: ##! ##! - It doesn't work well on a cluster because each worker will write its ##! own certificate files and no duplicate checking is done across @@ -20,15 +20,15 @@ module SSL; export { - ## Setting to control if host certificates offered by the defined hosts + ## Control if host certificates offered by the defined hosts ## will be written to the PEM certificates file. ## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS const extract_certs_pem = LOCAL_HOSTS &redef; } -## This is an internally maintained variable to prevent relogging of -## certificates that have already been seen. It is indexed on an md5 sum of -## the certificate. +# This is an internally maintained variable to prevent relogging of +# certificates that have already been seen. It is indexed on an md5 sum of +# the certificate. global extracted_certs: set[string] = set() &read_expire=1hr &redef; event ssl_established(c: connection) &priority=5 diff --git a/scripts/policy/protocols/ssl/known-certs.bro b/scripts/policy/protocols/ssl/known-certs.bro index 669432e4d9..3986a9aa1e 100644 --- a/scripts/policy/protocols/ssl/known-certs.bro +++ b/scripts/policy/protocols/ssl/known-certs.bro @@ -1,5 +1,4 @@ -##! This script can be used to log information about certificates while -##! attempting to avoid duplicate logging. +##! Log information about certificates while attempting to avoid duplicate logging. @load base/utils/directions-and-hosts @load base/protocols/ssl @@ -36,6 +35,8 @@ export { ## in the set is for storing the DER formatted certificate's MD5 hash. global certs: set[addr, string] &create_expire=1day &synchronized &redef; + ## Event that can be handled to access the loggable record as it is sent + ## on to the logging framework. global log_known_certs: event(rec: CertsInfo); } diff --git a/scripts/policy/protocols/ssl/validate-certs.bro b/scripts/policy/protocols/ssl/validate-certs.bro index 5a663864d2..03624eac84 100644 --- a/scripts/policy/protocols/ssl/validate-certs.bro +++ b/scripts/policy/protocols/ssl/validate-certs.bro @@ -14,8 +14,7 @@ export { }; redef record Info += { - ## This stores and logs the result of certificate validation for - ## this connection. + ## Result of certificate validation for this connection. validation_status: string &log &optional; }; From 9b6373584ce8242943242599fda1de68f64f0726 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 10 Jan 2012 01:09:35 -0500 Subject: [PATCH 247/964] Forgot to add protocol identifier support for TLS 1.2 --- scripts/base/protocols/ssl/consts.bro | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/base/protocols/ssl/consts.bro b/scripts/base/protocols/ssl/consts.bro index 1568ff17d3..e15a8f169a 100644 --- a/scripts/base/protocols/ssl/consts.bro +++ b/scripts/base/protocols/ssl/consts.bro @@ -5,12 +5,14 @@ export { const SSLv3 = 0x0300; const TLSv10 = 0x0301; const TLSv11 = 0x0302; + const TLSv12 = 0x0303; ## Mapping between the constants and string values for SSL/TLS versions. const version_strings: table[count] of string = { [SSLv2] = "SSLv2", [SSLv3] = "SSLv3", [TLSv10] = "TLSv10", [TLSv11] = "TLSv11", + [TLSv12] = "TLSv12", } &default="UNKNOWN"; ## Mapping between numeric codes and human readable strings for alert From 4de670a10e144f25dd87fd96b6acd0d6f60d75e0 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 10 Jan 2012 01:30:55 -0500 Subject: [PATCH 248/964] Fixing some doc warnings. --- scripts/base/frameworks/notice/main.bro | 2 +- scripts/base/protocols/ssl/consts.bro | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index 7fb8d6df77..e9b29e7392 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -78,7 +78,7 @@ export { ## or p is specified. proto: transport_proto &log &optional; - ## The :bro:enum:`Notice::Type` of the notice. + ## The :bro:type:`Notice::Type` of the notice. note: Type &log; ## The human readable message for the notice. msg: string &log &optional; diff --git a/scripts/base/protocols/ssl/consts.bro b/scripts/base/protocols/ssl/consts.bro index e15a8f169a..9d16ab18ba 100644 --- a/scripts/base/protocols/ssl/consts.bro +++ b/scripts/base/protocols/ssl/consts.bro @@ -59,8 +59,8 @@ export { ## Mapping between numeric codes and human readable strings for SSL/TLS ## extensions. - ## ..note: More information can be found here: - ## http://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xml + # More information can be found here: + # http://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xml const extensions: table[count] of string = { [0] = "server_name", [1] = "max_fragment_length", @@ -82,7 +82,7 @@ export { [65281] = "renegotiation_info" } &default=function(i: count):string { return fmt("unknown-%d", i); }; - ## SSLv2 + # SSLv2 const SSLv20_CK_RC4_128_WITH_MD5 = 0x010080; const SSLv20_CK_RC4_128_EXPORT40_WITH_MD5 = 0x020080; const SSLv20_CK_RC2_128_CBC_WITH_MD5 = 0x030080; @@ -91,7 +91,7 @@ export { const SSLv20_CK_DES_64_CBC_WITH_MD5 = 0x060040; const SSLv20_CK_DES_192_EDE3_CBC_WITH_MD5 = 0x0700C0; - ## TLS + # TLS const TLS_NULL_WITH_NULL_NULL = 0x0000; const TLS_RSA_WITH_NULL_MD5 = 0x0001; const TLS_RSA_WITH_NULL_SHA = 0x0002; @@ -313,7 +313,6 @@ export { ## detecting unknown ciphers and for converting the cipher spec constants ## into a human readable format. const cipher_desc: table[count] of string = { - # --- sslv20 --- [SSLv20_CK_RC4_128_EXPORT40_WITH_MD5] = "SSLv20_CK_RC4_128_EXPORT40_WITH_MD5", [SSLv20_CK_RC4_128_WITH_MD5] = "SSLv20_CK_RC4_128_WITH_MD5", @@ -325,7 +324,6 @@ export { "SSLv20_CK_DES_192_EDE3_CBC_WITH_MD5", [SSLv20_CK_DES_64_CBC_WITH_MD5] = "SSLv20_CK_DES_64_CBC_WITH_MD5", - # --- TLS --- [TLS_NULL_WITH_NULL_NULL] = "TLS_NULL_WITH_NULL_NULL", [TLS_RSA_WITH_NULL_MD5] = "TLS_RSA_WITH_NULL_MD5", [TLS_RSA_WITH_NULL_SHA] = "TLS_RSA_WITH_NULL_SHA", From 048516c6057dd55b4ff9d30587c826d575fce3df Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 10 Jan 2012 09:10:45 -0500 Subject: [PATCH 249/964] Adding back the stats.bro file. Closes #656 --- scripts/policy/misc/stats.bro | 83 +++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 scripts/policy/misc/stats.bro diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro new file mode 100644 index 0000000000..b188bf3af4 --- /dev/null +++ b/scripts/policy/misc/stats.bro @@ -0,0 +1,83 @@ +##| Log memory/packet/lag statistics. Differs from profiling.bro in that this +##| is lighter-weight (much less info, and less load to generate). + +@load base/frameworks/notice + +module Stats; + +export { + redef enum Log::ID += { LOG }; + + ## How often stats are reported. + const stats_report_interval = 1min &redef; + + type Info: record { + ## Timestamp for the measurement. + ts: time &log; + ## Peer that generated this log. Mostly for clusters. + peer: string &log; + ## Amount of memory currently in use in MB. + mem: count &log; + ## Number of packets processed since the last stats interval. + pkts_proc: count &log; + ## Number of events that been processed since the last stats interval. + events_proc: count &log; + ## Number of events that have been queued since the last stats interval. + events_queued: count &log; + + ## Lag between the wall clock and packet timestamps if reading live traffic. + lag: interval &log &optional; + ## Number of packets received since the last stats interval if reading + ## live traffic. + pkts_recv: count &log &optional; + ## Number of packets dropped since the last stats interval if reading + ## live traffic. + pkts_dropped: count &log &optional; + ## Number of packets seen on the link since the last stats interval + ## if reading live traffic. + pkts_link: count &log &optional; + }; + + ## Event to catch stats as they are written to the logging stream. + global log_stats: event(rec: Info); +} + +event bro_init() &priority=5 + { + Log::create_stream(Stats::LOG, [$columns=Info, $ev=log_stats]); + } + +event check_stats(last_ts: time, last_ns: NetStats, last_res: bro_resources) + { + local now = current_time(); + local ns = net_stats(); + local res = resource_usage(); + + if ( bro_is_terminating() ) + # No more stats will be written or scheduled when Bro is + # shutting down. + return; + + local info: Info = [$ts=now, $peer=peer_description, $mem=res$mem/1000000, + $pkts_proc=res$num_packets - last_res$num_packets, + $events_proc=res$num_events_dispatched - last_res$num_events_dispatched, + $events_queued=res$num_events_queued - last_res$num_events_queued]; + + if ( reading_live_traffic() ) + { + info$lag = now - network_time(); + # Someone's going to have to explain what this is and add a field to the Info record. + # info$util = 100.0*((res$user_time + res$system_time) - (last_res$user_time + last_res$system_time))/(now-last_ts); + info$pkts_recv = ns$pkts_recvd - last_ns$pkts_recvd; + info$pkts_dropped = ns$pkts_dropped - last_ns$pkts_dropped; + info$pkts_link = ns$pkts_link - last_ns$pkts_link; + } + + Log::write(Stats::LOG, info); + schedule stats_report_interval { check_stats(now, ns, res) }; + } + +event bro_init() + { + schedule stats_report_interval { check_stats(current_time(), net_stats(), resource_usage()) }; + } From 727e626bb4445719fea15baa8529c11398f57469 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 10 Jan 2012 10:38:12 -0500 Subject: [PATCH 250/964] Added an option for filtering out urls before they are turned into HTTP::Incorrect_File_Type notices --- scripts/base/protocols/http/file-ident.bro | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/base/protocols/http/file-ident.bro b/scripts/base/protocols/http/file-ident.bro index c2d858852b..de209d2785 100644 --- a/scripts/base/protocols/http/file-ident.bro +++ b/scripts/base/protocols/http/file-ident.bro @@ -39,6 +39,12 @@ export { const mime_types_extensions: table[string] of pattern = { ["application/x-dosexec"] = /\.([eE][xX][eE]|[dD][lL][lL])/, } &redef; + + ## A pattern for filtering out :bro:enum:`HTTP::Incorrect_File_Type` urls + ## that are not noteworthy before a notice is created. Each + ## pattern added should match the complete URL (the matched URLs include + ## "http://" at the beginning). + const ignored_incorrect_file_type_urls = /^$/ &redef; } event signature_match(state: signature_state, msg: string, data: string) &priority=5 @@ -59,6 +65,10 @@ event signature_match(state: signature_state, msg: string, data: string) &priori c$http?$uri && mime_types_extensions[msg] !in c$http$uri ) { local url = build_url_http(c$http); + + if ( url == ignored_incorrect_file_type_urls ) + return; + local message = fmt("%s %s %s", msg, c$http$method, url); NOTICE([$note=Incorrect_File_Type, $msg=message, From f921a4d5db4912d446910aed79ea84b5159c092f Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 10 Jan 2012 09:38:17 -0600 Subject: [PATCH 251/964] Change SFTP/SCP log rotators to use 4-digit year in filenames (fixes #745). --- scripts/base/frameworks/logging/postprocessors/scp.bro | 10 +++++++++- .../base/frameworks/logging/postprocessors/sftp.bro | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/scripts/base/frameworks/logging/postprocessors/scp.bro b/scripts/base/frameworks/logging/postprocessors/scp.bro index f27e748ae5..ee709ebd5e 100644 --- a/scripts/base/frameworks/logging/postprocessors/scp.bro +++ b/scripts/base/frameworks/logging/postprocessors/scp.bro @@ -25,6 +25,10 @@ export { ## function queries this table upon log rotation and performs a secure ## copy of the rotated-log to each destination in the set. global scp_destinations: table[Writer, string] of set[SCPDestination]; + + ## Default naming format for timestamps embedded into log filenames + ## that use the SCP rotator. + const scp_rotation_date_format = "%Y-%m-%d-%H-%M-%S" &redef; } function scp_postprocessor(info: Log::RotationInfo): bool @@ -34,7 +38,11 @@ function scp_postprocessor(info: Log::RotationInfo): bool local command = ""; for ( d in scp_destinations[info$writer, info$path] ) - command += fmt("scp %s %s@%s:%s;", info$fname, d$user, d$host, d$path); + { + local dst = fmt("%s/%s.%s.log", d$path, info$path, + strftime(Log::scp_rotation_date_format, info$open)); + command += fmt("scp %s %s@%s:%s;", info$fname, d$user, d$host, dst); + } command += fmt("/bin/rm %s", info$fname); system(command); diff --git a/scripts/base/frameworks/logging/postprocessors/sftp.bro b/scripts/base/frameworks/logging/postprocessors/sftp.bro index c0423bb1c4..5a31853063 100644 --- a/scripts/base/frameworks/logging/postprocessors/sftp.bro +++ b/scripts/base/frameworks/logging/postprocessors/sftp.bro @@ -47,6 +47,10 @@ export { ## transfer of the rotated-log to each destination in the set. This ## table can be modified at run-time. global sftp_destinations: table[Writer, string] of set[SFTPDestination]; + + ## Default naming format for timestamps embedded into log filenames + ## that use the SFTP rotator. + const sftp_rotation_date_format = "%Y-%m-%d-%H-%M-%S" &redef; } function sftp_postprocessor(info: Log::RotationInfo): bool @@ -56,8 +60,12 @@ function sftp_postprocessor(info: Log::RotationInfo): bool local command = ""; for ( d in sftp_destinations[info$writer, info$path] ) - command += fmt("echo put %s %s | sftp -b - %s@%s;", info$fname, d$path, + { + local dst = fmt("%s/%s.%s.log", d$path, info$path, + strftime(Log::sftp_rotation_date_format, info$open)); + command += fmt("echo put %s %s | sftp -b - %s@%s;", info$fname, dst, d$user, d$host); + } command += fmt("/bin/rm %s", info$fname); system(command); From 86a1cbca82b53d67cfbe45a3293c90f2a08093c8 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 10 Jan 2012 10:49:10 -0500 Subject: [PATCH 252/964] A few more tiny documentation updates commited to the wrong branch. :) --- scripts/policy/misc/capture-loss.bro | 2 +- scripts/policy/misc/loaded-scripts.bro | 2 +- scripts/policy/misc/profiling.bro | 9 ++++----- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/policy/misc/capture-loss.bro b/scripts/policy/misc/capture-loss.bro index d966708762..87fc80be67 100644 --- a/scripts/policy/misc/capture-loss.bro +++ b/scripts/policy/misc/capture-loss.bro @@ -17,7 +17,7 @@ export { redef enum Notice::Type += { ## Report if the detected capture loss exceeds the percentage - ## threshold + ## threshold. Too_Much_Loss }; diff --git a/scripts/policy/misc/loaded-scripts.bro b/scripts/policy/misc/loaded-scripts.bro index 27275156ea..468478e682 100644 --- a/scripts/policy/misc/loaded-scripts.bro +++ b/scripts/policy/misc/loaded-scripts.bro @@ -1,4 +1,4 @@ -##! +##! Log the loaded scripts. module LoadedScripts; diff --git a/scripts/policy/misc/profiling.bro b/scripts/policy/misc/profiling.bro index 457675b1d6..31451f1a55 100644 --- a/scripts/policy/misc/profiling.bro +++ b/scripts/policy/misc/profiling.bro @@ -2,14 +2,13 @@ module Profiling; +## Set the profiling output file. redef profiling_file = open_log_file("prof"); -export { - ## Cheap profiling every 15 seconds. - redef profiling_interval = 15 secs &redef; -} +## Set the cheap profiling interval. +redef profiling_interval = 15 secs; -# Expensive profiling every 5 minutes. +## Set the expensive profiling interval. redef expensive_profiling_multiple = 20; event bro_init() From 4232e67db34c4892fb11a05305845f1889184c9c Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 10 Jan 2012 11:25:53 -0500 Subject: [PATCH 253/964] A few updates for the FAQ. --- doc/faq.rst | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/doc/faq.rst b/doc/faq.rst index 510d03c5af..6ddef4d8ea 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -59,34 +59,30 @@ Usage How can I identify backscatter? ------------------------------- -Identifying backscatter via connections labeled as ``OTH`` is not -a reliable means to detect backscatter. Use rather the following -procedure: - -* Enable connection history via ``redef record_state_history=T`` to - track all control/data packet types in connection logs. - -* Backscatter is now visible in terms of connections that never had an - initial ``SYN`` but started instead with a ``SYN-ACK`` or ``RST`` - (though this latter generally is just discarded). +Identifying backscatter via connections labeled as ``OTH`` is not a reliable +means to detect backscatter. Backscatter is however visible by interpreting +the contents of the ``history`` field in the ``conn.log`` file. The basic idea +is to watch for connections that never had an initial ``SYN`` but started +instead with a ``SYN-ACK`` or ``RST`` (though this latter generally is just +discarded). Here are some history fields which provide backscatter examples: +``hAFf``, ``r``. Refer to the conn protocol analysis scripts to interpret the +individual character meanings in the history field. Is there help for understanding Bro's resource consumption? ----------------------------------------------------------- There are two scripts that collect statistics on resource usage: -``stats.bro`` and ``profiling.bro``. The former is quite lightweight, -while the latter should only be used for debugging. Furthermore, -there's also ``print-globals.bro``, which prints the size of all -global script variable at termination. +``misc/stats.bro`` and ``misc/profiling.bro``. The former is quite +lightweight, while the latter should only be used for debugging. How can I capture packets as an unprivileged user? -------------------------------------------------- -Normally, unprivileged users cannot capture packets from a network -interface, which means they would not be able to use Bro to read/analyze -live traffic. However, there are ways to enable packet capture -permission for non-root users, which is worth doing in the context of -using Bro to monitor live traffic +Normally, unprivileged users cannot capture packets from a network interface, +which means they would not be able to use Bro to read/analyze live traffic. +However, there are operating system specific ways to enable packet capture +permission for non-root users, which is worth doing in the context of using +Bro to monitor live traffic. With Linux Capabilities ^^^^^^^^^^^^^^^^^^^^^^^ From 2348d794b6e0476a944b119e08319cae4bffc7b9 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 10 Jan 2012 12:25:33 -0600 Subject: [PATCH 254/964] Fix ref counting bug in BIFs that call internal_type. (fixes #740) --- src/NetVar.cc | 4 ++++ src/NetVar.h | 2 ++ src/bro.bif | 12 ++++++++---- src/strings.bif | 6 +++--- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/NetVar.cc b/src/NetVar.cc index 25e4f7a0bc..5aed213508 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -16,7 +16,9 @@ RecordType* pcap_packet; RecordType* signature_state; EnumType* transport_proto; TableType* string_set; +TableType* string_array; TableType* count_set; +VectorType* string_vec; int watchdog_interval; @@ -328,6 +330,8 @@ void init_net_var() pcap_packet = internal_type("pcap_packet")->AsRecordType(); transport_proto = internal_type("transport_proto")->AsEnumType(); string_set = internal_type("string_set")->AsTableType(); + string_array = internal_type("string_array")->AsTableType(); + string_vec = internal_type("string_vec")->AsVectorType(); ignore_checksums = opt_internal_int("ignore_checksums"); partial_connection_ok = opt_internal_int("partial_connection_ok"); diff --git a/src/NetVar.h b/src/NetVar.h index f8def230c0..4a513a8a53 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -19,7 +19,9 @@ extern RecordType* SYN_packet; extern RecordType* pcap_packet; extern EnumType* transport_proto; extern TableType* string_set; +extern TableType* string_array; extern TableType* count_set; +extern VectorType* string_vec; extern int watchdog_interval; diff --git a/src/bro.bif b/src/bro.bif index a2f97356a7..c78ae5bffe 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -363,19 +363,23 @@ function cat%(...%): string function record_type_to_vector%(rt: string%): string_vec %{ - VectorVal* result = - new VectorVal(internal_type("string_vec")->AsVectorType()); + VectorVal* result = new VectorVal(string_vec); - RecordType *type = internal_type(rt->CheckString())->AsRecordType(); + ID* id = lookup_ID(rt->CheckString(), GLOBAL_MODULE_NAME); - if ( type ) + if ( id ) { + RecordType *type = id->Type()->AsRecordType(); for ( int i = 0; i < type->NumFields(); ++i ) { StringVal* val = new StringVal(type->FieldName(i)); result->Assign(i+1, val, 0); } + Unref(id); } + else + reporter->InternalError("internal record type %s missing", + rt->CheckString()); return result; %} diff --git a/src/strings.bif b/src/strings.bif index 7e9885e296..52b6089bcc 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -129,7 +129,7 @@ function sort_string_array%(a: string_array%): string_array } // sort(vs.begin(), vs.end(), Bstr_cmp); - TableVal* b = new TableVal(internal_type("string_array")->AsTableType()); + TableVal* b = new TableVal(string_array); vs_to_string_array(vs, b, 1, n); return b; %} @@ -216,7 +216,7 @@ static int match_prefix(int s_len, const char* s, int t_len, const char* t) Val* do_split(StringVal* str_val, RE_Matcher* re, TableVal* other_sep, int incl_sep, int max_num_sep) { - TableVal* a = new TableVal(internal_type("string_array")->AsTableType()); + TableVal* a = new TableVal(string_array); ListVal* other_strings = 0; if ( other_sep && other_sep->Size() > 0 ) @@ -679,7 +679,7 @@ function str_shell_escape%(source: string%): string # empty set if none). function find_all%(str: string, re: pattern%) : string_set %{ - TableVal* a = new TableVal(internal_type("string_set")->AsTableType()); + TableVal* a = new TableVal(string_set); const u_char* s = str->Bytes(); const u_char* e = s + str->Len(); From 66be86da61c76f9ce226dd13faaa2c46339e7cd3 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 10 Jan 2012 10:42:35 -0800 Subject: [PATCH 255/964] Fixing coverage failures. Friendly reminder: please run test-suite before pushing things upstream for merges ... --- doc/scripts/DocSourcesList.cmake | 2 ++ scripts/test-all-policy.bro | 1 + 2 files changed, 3 insertions(+) diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index 9d99effc02..ade0add875 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -34,6 +34,7 @@ rest_target(${psd} base/frameworks/dpd/main.bro) rest_target(${psd} base/frameworks/intel/main.bro) rest_target(${psd} base/frameworks/logging/main.bro) rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro) +rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro) rest_target(${psd} base/frameworks/logging/writers/ascii.bro) rest_target(${psd} base/frameworks/metrics/cluster.bro) rest_target(${psd} base/frameworks/metrics/main.bro) @@ -102,6 +103,7 @@ rest_target(${psd} policy/misc/analysis-groups.bro) rest_target(${psd} policy/misc/capture-loss.bro) rest_target(${psd} policy/misc/loaded-scripts.bro) rest_target(${psd} policy/misc/profiling.bro) +rest_target(${psd} policy/misc/stats.bro) rest_target(${psd} policy/misc/trim-trace-file.bro) rest_target(${psd} policy/protocols/conn/known-hosts.bro) rest_target(${psd} policy/protocols/conn/known-services.bro) diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index f653220cbc..415468a801 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -26,6 +26,7 @@ @load misc/capture-loss.bro @load misc/loaded-scripts.bro @load misc/profiling.bro +@load misc/stats.bro @load misc/trim-trace-file.bro @load protocols/conn/known-hosts.bro @load protocols/conn/known-services.bro From 03e12c31eba91a794912a3f1e5f8e4b6d4c4c185 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 10 Jan 2012 12:58:24 -0600 Subject: [PATCH 256/964] Fix some failing coverage unit tests. --- doc/scripts/DocSourcesList.cmake | 2 ++ scripts/test-all-policy.bro | 1 + testing/external/scripts/diff-all | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index 9d99effc02..ade0add875 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -34,6 +34,7 @@ rest_target(${psd} base/frameworks/dpd/main.bro) rest_target(${psd} base/frameworks/intel/main.bro) rest_target(${psd} base/frameworks/logging/main.bro) rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro) +rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro) rest_target(${psd} base/frameworks/logging/writers/ascii.bro) rest_target(${psd} base/frameworks/metrics/cluster.bro) rest_target(${psd} base/frameworks/metrics/main.bro) @@ -102,6 +103,7 @@ rest_target(${psd} policy/misc/analysis-groups.bro) rest_target(${psd} policy/misc/capture-loss.bro) rest_target(${psd} policy/misc/loaded-scripts.bro) rest_target(${psd} policy/misc/profiling.bro) +rest_target(${psd} policy/misc/stats.bro) rest_target(${psd} policy/misc/trim-trace-file.bro) rest_target(${psd} policy/protocols/conn/known-hosts.bro) rest_target(${psd} policy/protocols/conn/known-services.bro) diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index f653220cbc..415468a801 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -26,6 +26,7 @@ @load misc/capture-loss.bro @load misc/loaded-scripts.bro @load misc/profiling.bro +@load misc/stats.bro @load misc/trim-trace-file.bro @load protocols/conn/known-hosts.bro @load protocols/conn/known-services.bro diff --git a/testing/external/scripts/diff-all b/testing/external/scripts/diff-all index 1fd633b1e9..e84416c088 100755 --- a/testing/external/scripts/diff-all +++ b/testing/external/scripts/diff-all @@ -22,7 +22,7 @@ files_cwd=`ls $@` files_baseline=`cd $TEST_BASELINE && ls $@` for i in `echo $files_cwd $files_baseline | sort | uniq`; do - if [[ "$i" != "loaded_scripts.log" && "$i" != "prof.log" && "$i" != "debug.log" ]]; then + if [[ "$i" != "loaded_scripts.log" && "$i" != "prof.log" && "$i" != "debug.log" && "$i" != "stats.log" ]]; then if [[ "$i" == "reporter.log" ]]; then # Do not diff the reporter.log if it only complains about missing From 38f58df729ae3f29d222f5ba18788e1072eb2e0c Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 10 Jan 2012 14:41:42 -0600 Subject: [PATCH 257/964] Small fix to binary package quickstart docs. --- doc/quickstart.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/quickstart.rst b/doc/quickstart.rst index 0534186cde..710921ead1 100644 --- a/doc/quickstart.rst +++ b/doc/quickstart.rst @@ -33,17 +33,17 @@ See the `bro downloads page`_ for currently supported/targeted platforms. .. console:: - sudo yum localinstall Bro-all*.rpm + sudo yum localinstall Bro-*.rpm * DEB .. console:: - sudo gdebi Bro-all-*.deb + sudo gdebi Bro-*.deb * MacOS Disk Image with Installer - Just open the ``Bro-all-*.dmg`` and then run the ``.pkg`` installer. + Just open the ``Bro-*.dmg`` and then run the ``.pkg`` installer. Everything installed by the package will go into ``/opt/bro``. The primary install prefix for binary packages is ``/opt/bro``. From 46de66090929a5e2fe200f48fb318e0d2120e85c Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 10 Jan 2012 17:27:22 -0800 Subject: [PATCH 258/964] Updating submodules. --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aux/binpac b/aux/binpac index 3741d4f229..aa1aa85ddc 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 3741d4f22933fa64d5194ebe4686d487c66458fa +Subproject commit aa1aa85ddcf524ffcfcf9efa5277bfac341871f7 diff --git a/aux/bro-aux b/aux/bro-aux index bce1059c71..22a2c5249c 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit bce1059c717685fd862928404efbbfe62b0a95c5 +Subproject commit 22a2c5249c56b46290f5bde366f69e1eeacbfe0a diff --git a/aux/broccoli b/aux/broccoli index 20d0a88708..71da6a319f 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 20d0a887082837549aa530749c963db4ad204abd +Subproject commit 71da6a319f8c2578b87cf1bb6337b9fcc2724a66 diff --git a/aux/broctl b/aux/broctl index 4d57e071be..6ae53077c1 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 4d57e071bec29a118d5629d1f15d673754c06b8b +Subproject commit 6ae53077c1b389011e4b49f10e3dbd8d3cf2d0eb From 981c0ebf408ce53e8b5f487c1fa1139942e7dc13 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 10 Jan 2012 17:37:49 -0800 Subject: [PATCH 259/964] Couple submodule notes for NEWS. --- NEWS | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index aca9a820b6..96544d887e 100644 --- a/NEWS +++ b/NEWS @@ -43,8 +43,20 @@ final release are: * Some NOTICES were renamed, and the signatures of some SSL events have changed. + * bro-cut got some new capabilities: + + - If no field names are given on the command line, we now pass + through all fields. + + - New options -u/-U for time output in UTC. + + - New option -F to give output field separator. + + * Broccoli supports more types internally, allowing to send + complex records. + * Many smaller bug fixes, portability improvements, and general - polishing. + polishing across all modules. From a7a33ab1469b6659f5fda130ffb51c3824d1e142 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 10 Jan 2012 17:40:18 -0800 Subject: [PATCH 260/964] Updating submodule. --- aux/btest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/btest b/aux/btest index 38890e8514..5350e4652b 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 38890e851416fa9fc827a1d36f06c4cb9f7d4e69 +Subproject commit 5350e4652b44ce1fbd9fffe1228d097fb04247cd From 8b06fdfc677088577f9f57431b4579c541d2dec7 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 10 Jan 2012 17:51:34 -0800 Subject: [PATCH 261/964] Updating CHANGES and VERSION. --- CHANGES | 4 ++++ VERSION | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 462e487224..af6123aa85 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.0 | 2012-01-10 17:40:18 -0800 + + * Adding script reference documentation. (The Team). + 2.0-beta-194 | 2012-01-10 10:44:32 -0800 * Added an option for filtering out URLs before they are turned into diff --git a/VERSION b/VERSION index 37f4da983a..cd5ac039d6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0-beta-194 +2.0 From 71eaae8539ea45602128292d1de0fbe6cebb99c7 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 10 Jan 2012 19:09:30 -0800 Subject: [PATCH 262/964] Fixing typos --- scripts/policy/misc/stats.bro | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index 79c3da46ea..d7866fd136 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -1,5 +1,5 @@ -##| Log memory/packet/lag statistics. Differs from profiling.bro in that this -##| is lighter-weight (much less info, and less load to generate). +##! Log memory/packet/lag statistics. Differs from profiling.bro in that this +##! is lighter-weight (much less info, and less load to generate). @load base/frameworks/notice From 4dd898131e632bf28135ecf1e87f3f19192b9fb0 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 10 Jan 2012 19:17:22 -0800 Subject: [PATCH 263/964] Slight reordering on main doc page. --- doc/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/index.rst b/doc/index.rst index 4b362db62e..ec67b76fd8 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -41,10 +41,10 @@ Script Reference .. toctree:: :maxdepth: 1 - scripts/builtins - scripts/bifs scripts/packages scripts/index + scripts/builtins + scripts/bifs Other Bro Components -------------------- From eac4690f83375304a696dbe8d0cc4aceab27d30d Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 10 Jan 2012 19:17:53 -0800 Subject: [PATCH 264/964] Updating submodule(s). [nomail] --- CHANGES | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index af6123aa85..f9e9b7eee9 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,5 @@ -2.0 | 2012-01-10 17:40:18 -0800 +2.0 | 2012-01-10 19:17:53 -0800 * Adding script reference documentation. (The Team). diff --git a/aux/broccoli b/aux/broccoli index 71da6a319f..722ca1f266 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 71da6a319f8c2578b87cf1bb6337b9fcc2724a66 +Subproject commit 722ca1f266d8cf2f79a89e24b6b0a15174365ccd diff --git a/aux/broctl b/aux/broctl index 6ae53077c1..84636789bf 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 6ae53077c1b389011e4b49f10e3dbd8d3cf2d0eb +Subproject commit 84636789bf48376d7278b31e0bd7be19ff72c566 From d9dccd9e65dafaee9dc29d0604c7de8fa9ac5e1f Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 11 Jan 2012 13:02:12 -0600 Subject: [PATCH 265/964] corrected some minor typos --- Makefile | 2 +- NEWS | 7 ++++--- README | 2 +- doc/README | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 4ec6d04a95..455fa6ed88 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # A simple static wrapper for a number of standard Makefile targets, # mostly just forwarding to build/Makefile. This is provided only for # convenience and supports only a subset of what CMake's Makefile -# to offer. For more, execute that one directly. +# offers. For more, execute that one directly. # BUILD=build diff --git a/NEWS b/NEWS index aca9a820b6..4586410afc 100644 --- a/NEWS +++ b/NEWS @@ -9,7 +9,7 @@ Bro 2.0 ------- As the version number jump suggests, Bro 2.0 is a major upgrade and -lots of things have changed. We have assembled a separate upprade +lots of things have changed. We have assembled a separate upgrade guide with the most important changes compared to Bro 1.5 at http://www.bro-ids.org/documentation/upgrade.bro.html. You can find the offline version of that document in ``doc/upgrade.rst.``. @@ -32,8 +32,9 @@ final release are: be redefined on the command line, e.g. ``bro -i eth0 snaplen=65535``. - * Reintroduced the BRO_LOG_SUFFIX environment that the ASCII - logger now respects to add a suffix to the log files it creates. + * Reintroduced the BRO_LOG_SUFFIX environment variable that the + ASCII logger now respects to add a suffix to the log files it + creates. * The ASCII logs now include further header information, and fields set to an empty value are now logged as ``(empty)`` by diff --git a/README b/README index 37cee2f333..c837afaf92 100644 --- a/README +++ b/README @@ -4,7 +4,7 @@ Bro Network Security Monitor Bro is a powerful framework for network analysis and security monitoring. Please see the INSTALL file for installation instructions -and pointers for getting started. NEWS contains releases notes for the +and pointers for getting started. NEWS contains release notes for the current version, and CHANGES has the complete history of changes. Please see COPYING for licensing information. diff --git a/doc/README b/doc/README index 57d569db84..0ba0a8587f 100644 --- a/doc/README +++ b/doc/README @@ -2,7 +2,7 @@ Documentation ============= -This directory contains Bro documentation in reStructured text format +This directory contains Bro documentation in reStructuredText format (see http://docutils.sourceforge.net/rst.html). It is the root of a Sphinx source tree and can be modified to add more From b6ea4c6026912913cbfc7a88a49583ed7e0b445e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 11 Jan 2012 12:21:49 -0800 Subject: [PATCH 266/964] Fixing links. --- INSTALL | 2 +- NEWS | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/INSTALL b/INSTALL index a5af647b87..73b824b2b7 100644 --- a/INSTALL +++ b/INSTALL @@ -76,7 +76,7 @@ Running Bro Bro is a complex program and it takes a bit of time to get familiar with it. A good place for newcomers to start is the Quickstart Guide -at http://www.bro-ids.org/documentation/quickstart.bro.html. +at http://www.bro-ids.org/documentation/quickstart.html. For developers that wish to run Bro directly from the ``build/`` directory (i.e., without performing ``make install``), they will have diff --git a/NEWS b/NEWS index 96544d887e..765d9b688e 100644 --- a/NEWS +++ b/NEWS @@ -11,7 +11,7 @@ Bro 2.0 As the version number jump suggests, Bro 2.0 is a major upgrade and lots of things have changed. We have assembled a separate upprade guide with the most important changes compared to Bro 1.5 at -http://www.bro-ids.org/documentation/upgrade.bro.html. You can find +http://www.bro-ids.org/documentation/upgrade.html. You can find the offline version of that document in ``doc/upgrade.rst.``. Compared to the earlier 2.0 Beta version, the major changes in the From 6a2af5cd63d904985e143cfe77ed74101d34c67c Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 11 Jan 2012 12:23:11 -0800 Subject: [PATCH 267/964] Updating submodule(s). [nomail] --- CHANGES | 2 +- aux/bro-aux | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index f9e9b7eee9..8a3eebe3d7 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,5 @@ -2.0 | 2012-01-10 19:17:53 -0800 +2.0 | 2012-01-11 12:23:11 -0800 * Adding script reference documentation. (The Team). diff --git a/aux/bro-aux b/aux/bro-aux index 22a2c5249c..1ac5080de0 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 22a2c5249c56b46290f5bde366f69e1eeacbfe0a +Subproject commit 1ac5080de06bb29f2bc9fc73bd7dbbf945c7dc81 From 9aefeec4ce8f4a3952dc9874a99b570310fe8b39 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 11 Jan 2012 16:30:25 -0600 Subject: [PATCH 268/964] Integrate Bro script coverage profiling with the btest suite. --- src/Brofiler.cc | 20 +++++++-------- src/Brofiler.h | 8 ++++-- testing/btest/.gitignore | 1 + testing/btest/Makefile | 2 ++ testing/btest/btest.cfg | 5 +++- testing/scripts/btest-bg-run | 7 ++++++ testing/scripts/coverage-calc | 46 +++++++++++++++++++++++++++++++++++ 7 files changed, 75 insertions(+), 14 deletions(-) create mode 100755 testing/scripts/btest-bg-run create mode 100755 testing/scripts/coverage-calc diff --git a/src/Brofiler.cc b/src/Brofiler.cc index fc23f10a7d..8db5861d20 100644 --- a/src/Brofiler.cc +++ b/src/Brofiler.cc @@ -13,16 +13,12 @@ Brofiler::~Brofiler() { } -void Brofiler::ReadStats() +bool Brofiler::ReadStats() { char* bf = getenv("BROFILER_FILE"); - if ( ! bf ) return; + if ( ! bf ) return false; FILE* f = fopen(bf, "r"); - if ( ! f ) - { - fprintf(stderr, "Failed to open Brofiler file '%s' for reading\n", bf); - return; - } + if ( ! f ) return false; char line[16384]; string delimiter; @@ -40,18 +36,19 @@ void Brofiler::ReadStats() } fclose(f); + return true; } -void Brofiler::WriteStats() +bool Brofiler::WriteStats() { char* bf = getenv("BROFILER_FILE"); - if ( ! bf ) return; + if ( ! bf ) return false; FILE* f = fopen(bf, "w"); if ( ! f ) { - fprintf(stderr, "Failed to open Brofiler file '%s' for writing\n", bf); - return; + reporter->Error("Failed to open Brofiler file '%s' for writing\n", bf); + return false; } for ( list::const_iterator it = stmts.begin(); @@ -78,5 +75,6 @@ void Brofiler::WriteStats() } fclose(f); + return true; } diff --git a/src/Brofiler.h b/src/Brofiler.h index 6ded906698..0a284c62c8 100644 --- a/src/Brofiler.h +++ b/src/Brofiler.h @@ -18,15 +18,19 @@ public: /** * Imports Bro script Stmt usage information from file pointed to by * environment variable BROFILER_FILE. + * + * @return: true if usage info was read, otherwise false. */ - void ReadStats(); + bool ReadStats(); /** * Combines usage stats from current run with any read from ReadStats(), * then writes information to file pointed to by environment variable * BROFILER_FILE. + * + * @return: true when usage info is written, otherwise false. */ - void WriteStats(); + bool WriteStats(); void SetDelim(char d) { delim = d; } diff --git a/testing/btest/.gitignore b/testing/btest/.gitignore index 0c143f664e..5282177d90 100644 --- a/testing/btest/.gitignore +++ b/testing/btest/.gitignore @@ -1,2 +1,3 @@ .tmp diag.log +coverage.log diff --git a/testing/btest/Makefile b/testing/btest/Makefile index 7489d761fb..2ebd66edd2 100644 --- a/testing/btest/Makefile +++ b/testing/btest/Makefile @@ -5,7 +5,9 @@ BTEST=../../aux/btest/btest all: # Showing all tests. @rm -f $(DIAG) + @rm -f .tmp/script-coverage* @$(BTEST) -f $(DIAG) + @../scripts/coverage-calc ".tmp/script-coverage*" coverage.log `pwd` brief: # Brief output showing only failed tests. diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 7d8283587c..b37b3063be 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -10,9 +10,12 @@ BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev` BRO_SEED_FILE=%(testbase)s/random.seed TZ=UTC LC_ALL=C -PATH=%(testbase)s/../../build/src:%(testbase)s/../../aux/btest:%(default_path)s +BTEST_PATH=%(testbase)s/../../aux/btest +PATH=%(testbase)s/../../build/src:%(testbase)s/../scripts:%(testbase)s/../../aux/btest:%(default_path)s TRACES=%(testbase)s/Traces SCRIPTS=%(testbase)s/../scripts DIST=%(testbase)s/../.. BUILD=%(testbase)s/../../build TEST_DIFF_CANONIFIER=$SCRIPTS/diff-canonifier +TMPDIR=%(testbase)s/.tmp +BROFILER_FILE=%(testbase)s/.tmp/script-coverage diff --git a/testing/scripts/btest-bg-run b/testing/scripts/btest-bg-run new file mode 100755 index 0000000000..462ae23fa2 --- /dev/null +++ b/testing/scripts/btest-bg-run @@ -0,0 +1,7 @@ +#! /usr/bin/env bash + +# This is a wrapper script to btest's real btest-bg-run. It's used +# when collecting Bro script coverage statistics so that two independent +# Bro processing don't try to write those usage statistics to the same file. + +BROFILER_FILE=`mktemp -t script-coverage` $BTEST_PATH/btest-bg-run $@ diff --git a/testing/scripts/coverage-calc b/testing/scripts/coverage-calc new file mode 100755 index 0000000000..a146667595 --- /dev/null +++ b/testing/scripts/coverage-calc @@ -0,0 +1,46 @@ +#! /usr/bin/env python + +# This script aggregates many files containing Bro script coverage information +# into a single file and reports the overall coverage information. Usage: +# +# coverage-calc +# +# The last argument is used to ignore Bro scripts that are part of the test +# suite itself as those should not count towards the coverage calculation. + +import os +import sys +import glob + +stats = {} +inputglob = sys.argv[1] +outputfile = sys.argv[2] +ignoredir = os.path.abspath(sys.argv[3]) + +for filename in glob.glob(inputglob): + with open(filename, 'r') as f: + for line in f.read().splitlines(): + parts = line.split("\t") + exec_count = int(parts[0]) + location = os.path.normpath(parts[1]) + # ignore scripts that don't appear to be part of Bro distribution + if location.startswith(ignoredir) or not location.startswith("/"): + continue + desc = parts[2] + key = location + desc + if key in stats: + stats[key][0] += exec_count + else: + stats[key] = [exec_count, location, desc] + +with open(outputfile, 'w') as f: + for k in sorted(stats, key=lambda i: stats[i][1]): + f.write("%s\t%s\t%s\n" % (stats[k][0], stats[k][1], stats[k][2])) + +num_covered = 0 +for k in stats: + if stats[k][0] > 0: + num_covered += 1 + +if len(stats) > 0: + print "%s/%s (%.1f%%) Bro script statements covered." % (num_covered, len(stats), float(num_covered)/len(stats)*100) From 9c42f02082c41b088ae9313680091eebc28a2d47 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 11 Jan 2012 16:57:09 -0600 Subject: [PATCH 269/964] fixed some broken links --- doc/reporting-problems.rst | 2 +- doc/signatures.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/reporting-problems.rst b/doc/reporting-problems.rst index a1105708eb..fa3f32a620 100644 --- a/doc/reporting-problems.rst +++ b/doc/reporting-problems.rst @@ -29,7 +29,7 @@ following: * The output you're seeing along with a description what you'd expect Bro to do instead. -* A *small* trace in `libpcap format `__ +* A *small* trace in `libpcap format `__ demonstrating the effect (assuming the problem doesn't happen right at startup already). diff --git a/doc/signatures.rst b/doc/signatures.rst index a1e70f8e6f..c44e1b571a 100644 --- a/doc/signatures.rst +++ b/doc/signatures.rst @@ -338,7 +338,7 @@ Things to keep in mind when writing signatures signature engine and can be matched with ``\r`` and ``\n``, respectively. Generally, Bro follows `flex's regular expression syntax - `_. + `_. See the DPD signatures in ``base/frameworks/dpd/dpd.sig`` for some examples of fairly complex payload patterns. From b6c3567ba40e513cbdb303099d921f693ff65914 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 12 Jan 2012 11:58:13 -0600 Subject: [PATCH 270/964] Test coverage integration for external tests and complete suite. coverage.log files are output to each testing dir and reflect the number of times statement locations of Bro scripts are executed. --- testing/.gitignore | 1 + testing/Makefile | 5 +++++ testing/external/subdir-btest.cfg | 1 + 3 files changed, 7 insertions(+) create mode 100644 testing/.gitignore diff --git a/testing/.gitignore b/testing/.gitignore new file mode 100644 index 0000000000..a664c1d684 --- /dev/null +++ b/testing/.gitignore @@ -0,0 +1 @@ +coverage.log diff --git a/testing/Makefile b/testing/Makefile index 7f03a55f49..4b4e87d82f 100644 --- a/testing/Makefile +++ b/testing/Makefile @@ -3,6 +3,11 @@ DIRS=btest external all: @for repo in $(DIRS); do (cd $$repo && make ); done + @cp btest/coverage.log `mktemp brocov.tmp.XXX` + @for f in external/*/coverage.log; do cp $$f `mktemp brocov.tmp.XXX`; done + @echo "Complete test suite code coverage:" + @./scripts/coverage-calc "brocov.tmp.*" coverage.log `pwd` + @rm -f brocov.tmp.* brief: @for repo in $(DIRS); do (cd $$repo && make brief ); done diff --git a/testing/external/subdir-btest.cfg b/testing/external/subdir-btest.cfg index dd9a57c879..e24f89255a 100644 --- a/testing/external/subdir-btest.cfg +++ b/testing/external/subdir-btest.cfg @@ -17,3 +17,4 @@ TRACES=%(testbase)s/Traces SCRIPTS=%(testbase)s/../scripts DIST=%(testbase)s/../../.. BUILD=%(testbase)s/../../../build +BROFILER_FILE=%(testbase)s/.tmp/script-coverage From eb53a3d1c8bb9b7a590fc4f5ba70a5ea5017b6ee Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 12 Jan 2012 11:51:12 -0800 Subject: [PATCH 271/964] make input framework compile with brov6 --- src/InputReaderAscii.cc | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 6da161a2bf..257cb4cf71 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -253,19 +253,37 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { val->val.subnet_val.width = atoi(width.c_str()); string addr = s.substr(0, pos); s = addr; - // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // NOTE: dotted_to_addr BREAKS THREAD SAFETY! it uses reporter. // Solve this some other time.... +#ifdef BROv6 + if ( s.find(':') != s.npos ) { + uint32* addr = dotted_to_addr6(s.c_str()); + copy_addr(val->val.subnet_val.net, addr); + delete addr; + } else { + val->val.subnet_val.net[0] = val->val.subnet_val.net[1] = val->val.subnet_val.net[2] = 0; + val->val.subnet_val.net[3] = dotted_to_addr(s.c_str()); + } +#else val->val.subnet_val.net = dotted_to_addr(s.c_str()); +#endif break; } case TYPE_ADDR: { // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. // Solve this some other time.... - addr_type t = dotted_to_addr(s.c_str()); #ifdef BROv6 - copy_addr(t, val->val.addr_val); + if ( s.find(':') != s.npos ) { + uint32* addr = dotted_to_addr6(s.c_str()); + copy_addr(val->val.addr_val, addr); + delete addr; + } else { + val->val.addr_val[0] = val->val.addr_val[1] = val->val.addr_val[2] = 0; + val->val.addr_val[3] = dotted_to_addr(s.c_str()); + } #else + uint32 t = dotted_to_addr(s.c_str()); copy_addr(&t, val->val.addr_val); #endif break; From 713e3ac5d0ff63313fa2b6b8df25e41b1045519a Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 13 Jan 2012 14:52:58 -0600 Subject: [PATCH 272/964] Add "# @no-test" tag to blacklist statements from test coverage analysis. It can apply to either single statements: print "don't cover"; # @no-test or a block of statements: if ( F ) { # @no-test ... } --- src/Brofiler.cc | 2 +- src/Brofiler.h | 14 ++- src/parse.y | 89 +++++++++++-------- src/scan.l | 2 + .../coverage.coverage-blacklist/output | 5 ++ testing/btest/coverage/coverage-blacklist.bro | 29 ++++++ 6 files changed, 99 insertions(+), 42 deletions(-) create mode 100644 testing/btest/Baseline/coverage.coverage-blacklist/output create mode 100644 testing/btest/coverage/coverage-blacklist.bro diff --git a/src/Brofiler.cc b/src/Brofiler.cc index 8db5861d20..75c054a681 100644 --- a/src/Brofiler.cc +++ b/src/Brofiler.cc @@ -5,7 +5,7 @@ #include "util.h" Brofiler::Brofiler() - : delim('\t') + : delim('\t'), ignoring(0) { } diff --git a/src/Brofiler.h b/src/Brofiler.h index 0a284c62c8..58c23f06b1 100644 --- a/src/Brofiler.h +++ b/src/Brofiler.h @@ -34,14 +34,24 @@ public: void SetDelim(char d) { delim = d; } + void IncIgnoreDepth() { ignoring++; } + void DecIgnoreDepth() { ignoring--; } + + void AddStmt(const Stmt* s) { if ( ignoring == 0 ) stmts.push_back(s); } + +private: /** * The current, global Brofiler instance creates this list at parse-time. */ list stmts; -private: /** - * + * Indicates whether new statments will not be considered as part of + * coverage statistics because it was marked with the @no-test tag. + */ + unsigned int ignoring; + + /** * This maps Stmt location-desc pairs to the total number of times that * Stmt has been executed. The map can be initialized from a file at * startup time and modified at shutdown time before writing back diff --git a/src/parse.y b/src/parse.y index fb731d9555..35eae155b5 100644 --- a/src/parse.y +++ b/src/parse.y @@ -29,6 +29,8 @@ %token TOK_DOC TOK_POST_DOC +%token TOK_NO_TEST + %left ',' '|' %right '=' TOK_ADD_TO TOK_REMOVE_FROM %right '?' ':' TOK_USING @@ -42,6 +44,7 @@ %right '!' %left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR +%type opt_no_test opt_no_test_block %type TOK_ID TOK_PATTERN_TEXT single_pattern TOK_DOC TOK_POST_DOC %type opt_doc_list opt_post_doc_list %type local_id global_id def_global_id event_id global_or_event_id resolve_id begin_func @@ -196,6 +199,7 @@ static std::list* concat_opt_docs (std::list* pre, %} %union { + bool b; char* str; std::list* str_l; ID* id; @@ -1315,121 +1319,117 @@ attr: ; stmt: - '{' stmt_list '}' + '{' opt_no_test_block stmt_list '}' { - set_location(@1, @3); - $$ = $2; + set_location(@1, @4); + $$ = $3; + if ( $2 ) brofiler.DecIgnoreDepth(); } - | TOK_PRINT expr_list ';' + | TOK_PRINT expr_list ';' opt_no_test { set_location(@1, @3); $$ = new PrintStmt($2); - brofiler.stmts.push_back($$); + if ( ! $4 ) brofiler.AddStmt($$); } - | TOK_EVENT event ';' + | TOK_EVENT event ';' opt_no_test { set_location(@1, @3); $$ = new EventStmt($2); - brofiler.stmts.push_back($$); + if ( ! $4 ) brofiler.AddStmt($$); } | TOK_IF '(' expr ')' stmt { set_location(@1, @4); $$ = new IfStmt($3, $5, new NullStmt()); - //brofiler.stmts.push_back($$); } | TOK_IF '(' expr ')' stmt TOK_ELSE stmt { set_location(@1, @4); $$ = new IfStmt($3, $5, $7); - //brofiler.stmts.push_back($$); } | TOK_SWITCH expr '{' case_list '}' { set_location(@1, @2); $$ = new SwitchStmt($2, $4); - //brofiler.stmts.push_back($$); } | for_head stmt { $1->AsForStmt()->AddBody($2); - //brofiler.stmts.push_back($1); } - | TOK_NEXT ';' + | TOK_NEXT ';' opt_no_test { set_location(@1, @2); $$ = new NextStmt; - brofiler.stmts.push_back($$); + if ( ! $3 ) brofiler.AddStmt($$); } - | TOK_BREAK ';' + | TOK_BREAK ';' opt_no_test { set_location(@1, @2); $$ = new BreakStmt; - brofiler.stmts.push_back($$); + if ( ! $3 ) brofiler.AddStmt($$); } - | TOK_RETURN ';' + | TOK_RETURN ';' opt_no_test { set_location(@1, @2); $$ = new ReturnStmt(0); - brofiler.stmts.push_back($$); + if ( ! $3 ) brofiler.AddStmt($$); } - | TOK_RETURN expr ';' + | TOK_RETURN expr ';' opt_no_test { set_location(@1, @2); $$ = new ReturnStmt($2); - brofiler.stmts.push_back($$); + if ( ! $4 ) brofiler.AddStmt($$); } - | TOK_ADD expr ';' + | TOK_ADD expr ';' opt_no_test { set_location(@1, @3); $$ = new AddStmt($2); - brofiler.stmts.push_back($$); + if ( ! $4 ) brofiler.AddStmt($$); } - | TOK_DELETE expr ';' + | TOK_DELETE expr ';' opt_no_test { set_location(@1, @3); $$ = new DelStmt($2); - brofiler.stmts.push_back($$); + if ( ! $4 ) brofiler.AddStmt($$); } - | TOK_LOCAL local_id opt_type init_class opt_init opt_attr ';' + | TOK_LOCAL local_id opt_type init_class opt_init opt_attr ';' opt_no_test { set_location(@1, @7); $$ = add_local($2, $3, $4, $5, $6, VAR_REGULAR); - brofiler.stmts.push_back($$); + if ( ! $8 ) brofiler.AddStmt($$); } - | TOK_CONST local_id opt_type init_class opt_init opt_attr ';' + | TOK_CONST local_id opt_type init_class opt_init opt_attr ';' opt_no_test { set_location(@1, @6); $$ = add_local($2, $3, $4, $5, $6, VAR_CONST); - brofiler.stmts.push_back($$); + if ( ! $8 ) brofiler.AddStmt($$); } | TOK_WHEN '(' expr ')' stmt { set_location(@3, @5); $$ = new WhenStmt($3, $5, 0, 0, false); - brofiler.stmts.push_back($$); } - | TOK_WHEN '(' expr ')' stmt TOK_TIMEOUT expr '{' stmt_list '}' + | TOK_WHEN '(' expr ')' stmt TOK_TIMEOUT expr '{' opt_no_test_block stmt_list '}' { - set_location(@3, @8); - $$ = new WhenStmt($3, $5, $9, $7, false); - brofiler.stmts.push_back($$); + set_location(@3, @9); + $$ = new WhenStmt($3, $5, $10, $7, false); + if ( $9 ) brofiler.DecIgnoreDepth(); } @@ -1437,21 +1437,20 @@ stmt: { set_location(@4, @6); $$ = new WhenStmt($4, $6, 0, 0, true); - brofiler.stmts.push_back($$); } - | TOK_RETURN TOK_WHEN '(' expr ')' stmt TOK_TIMEOUT expr '{' stmt_list '}' + | TOK_RETURN TOK_WHEN '(' expr ')' stmt TOK_TIMEOUT expr '{' opt_no_test_block stmt_list '}' { - set_location(@4, @9); - $$ = new WhenStmt($4, $6, $10, $8, true); - brofiler.stmts.push_back($$); + set_location(@4, @10); + $$ = new WhenStmt($4, $6, $11, $8, true); + if ( $10 ) brofiler.DecIgnoreDepth(); } - | expr ';' + | expr ';' opt_no_test { set_location(@1, @2); $$ = new ExprStmt($1); - brofiler.stmts.push_back($$); + if ( ! $3 ) brofiler.AddStmt($$); } | ';' @@ -1649,6 +1648,18 @@ opt_doc_list: { $$ = 0; } ; +opt_no_test: + TOK_NO_TEST + { $$ = true; } + | + { $$ = false; } + +opt_no_test_block: + TOK_NO_TEST + { $$ = true; brofiler.IncIgnoreDepth(); } + | + { $$ = false; } + %% int yyerror(const char msg[]) diff --git a/src/scan.l b/src/scan.l index 623e0d2ed6..4914783c44 100644 --- a/src/scan.l +++ b/src/scan.l @@ -216,6 +216,8 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) } } +#{OWS}@no-test.* return TOK_NO_TEST; + #.* /* eat comments */ {WS} /* eat whitespace */ diff --git a/testing/btest/Baseline/coverage.coverage-blacklist/output b/testing/btest/Baseline/coverage.coverage-blacklist/output new file mode 100644 index 0000000000..6d3d243220 --- /dev/null +++ b/testing/btest/Baseline/coverage.coverage-blacklist/output @@ -0,0 +1,5 @@ +1 /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 13 print cover me; +1 /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 17 print always executed; +0 /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 26 print also impossible, but included in code coverage analysis; +1 /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 29 print success; +1 /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 5 print first; diff --git a/testing/btest/coverage/coverage-blacklist.bro b/testing/btest/coverage/coverage-blacklist.bro new file mode 100644 index 0000000000..04983a921f --- /dev/null +++ b/testing/btest/coverage/coverage-blacklist.bro @@ -0,0 +1,29 @@ +# @TEST-EXEC: BROFILER_FILE=coverage bro -b %INPUT +# @TEST-EXEC: grep %INPUT coverage | sort -k2 >output +# @TEST-EXEC: btest-diff output + +print "first"; + +if ( F ) + { # @no-test + print "hello"; + print "world"; + } + +print "cover me"; + +if ( T ) + { + print "always executed"; + } + +print "don't cover me"; # @no-test + +if ( 0 + 0 == 1 ) print "impossible"; # @no-test + +if ( 1 == 0 ) + { + print "also impossible, but included in code coverage analysis"; + } + +print "success"; From ec6560a6ed9235deb6d3976c544688abd494bab9 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 13 Jan 2012 16:06:44 -0600 Subject: [PATCH 273/964] Make communication log baseline test more reliable. --- .../send.log | 26 ++++++++++--------- .../communication_log_baseline.bro | 5 ++-- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log index e5dfb59592..d3c14c8603 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log +++ b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log @@ -5,15 +5,17 @@ #path communication #fields ts peer src_name connected_peer_desc connected_peer_addr connected_peer_port level message #types time string string string addr port string string -1324314302.411344 bro parent - - - info [#1/127.0.0.1:47757] added peer -1324314302.414978 bro child - - - info [#1/127.0.0.1:47757] connected -1324314302.415099 bro parent - - - info [#1/127.0.0.1:47757] peer connected -1324314302.415099 bro parent - - - info [#1/127.0.0.1:47757] phase: version -1324314302.417446 bro script - - - info connection established -1324314302.417446 bro script - - - info requesting events matching /^?(NOTHING)$?/ -1324314302.417446 bro script - - - info accepting state -1324314302.418003 bro parent - - - info [#1/127.0.0.1:47757] phase: handshake -1324314302.418003 bro parent - - - info warning: no events to request -1324314302.418003 bro parent - - - info terminating... -1324314302.418003 bro parent - - - info [#1/127.0.0.1:47757] peer_description is bro -1324314302.418003 bro parent - - - info [#1/127.0.0.1:47757] closing connection +1326492291.485390 bro parent - - - info [#1/127.0.0.1:47757] added peer +1326492291.491731 bro child - - - info [#1/127.0.0.1:47757] connected +1326492291.492024 bro parent - - - info [#1/127.0.0.1:47757] peer connected +1326492291.492024 bro parent - - - info [#1/127.0.0.1:47757] phase: version +1326492291.492740 bro script - - - info connection established +1326492291.492740 bro script - - - info requesting events matching /^?(NOTHING)$?/ +1326492291.492740 bro script - - - info accepting state +1326492291.493800 bro parent - - - info [#1/127.0.0.1:47757] phase: handshake +1326492291.493800 bro parent - - - info warning: no events to request +1326492291.494161 bro parent - - - info [#1/127.0.0.1:47757] peer_description is bro +1326492291.494404 bro parent - - - info [#1/127.0.0.1:47757] peer supports keep-in-cache; using that +1326492291.494404 bro parent - - - info [#1/127.0.0.1:47757] phase: running +1326492291.494404 bro parent - - - info terminating... +1326492291.494404 bro parent - - - info [#1/127.0.0.1:47757] closing connection diff --git a/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro b/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro index 3a4c1253eb..dc3b43ad67 100644 --- a/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro +++ b/testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro @@ -15,7 +15,7 @@ redef Communication::nodes += { ["foo"] = [$host = 127.0.0.1, $events = /NOTHING/, $connect=T] }; -event remote_connection_established(p: event_peer) +event remote_connection_handshake_done(p: event_peer) { terminate_communication(); terminate(); @@ -29,8 +29,9 @@ event remote_connection_established(p: event_peer) @load frameworks/communication/listen -event remote_connection_closed(p: event_peer) +event remote_connection_handshake_done(p: event_peer) { + terminate_communication(); terminate(); } From 0287f7adc2b6c43f3cd3e199d44a47b72bfe7a6e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 18 Jan 2012 11:45:53 -0600 Subject: [PATCH 274/964] Fix superfluous/duplicate data getting in to testing coverage log. --- testing/Makefile | 2 +- testing/btest/Makefile | 2 +- testing/scripts/coverage-calc | 17 +++++++++++------ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/testing/Makefile b/testing/Makefile index 4b4e87d82f..9a9a02fe47 100644 --- a/testing/Makefile +++ b/testing/Makefile @@ -6,7 +6,7 @@ all: @cp btest/coverage.log `mktemp brocov.tmp.XXX` @for f in external/*/coverage.log; do cp $$f `mktemp brocov.tmp.XXX`; done @echo "Complete test suite code coverage:" - @./scripts/coverage-calc "brocov.tmp.*" coverage.log `pwd` + @./scripts/coverage-calc "brocov.tmp.*" coverage.log `pwd`/../scripts @rm -f brocov.tmp.* brief: diff --git a/testing/btest/Makefile b/testing/btest/Makefile index 2ebd66edd2..e764dd2b15 100644 --- a/testing/btest/Makefile +++ b/testing/btest/Makefile @@ -7,7 +7,7 @@ all: @rm -f $(DIAG) @rm -f .tmp/script-coverage* @$(BTEST) -f $(DIAG) - @../scripts/coverage-calc ".tmp/script-coverage*" coverage.log `pwd` + @../scripts/coverage-calc ".tmp/script-coverage*" coverage.log `pwd`/../../scripts brief: # Brief output showing only failed tests. diff --git a/testing/scripts/coverage-calc b/testing/scripts/coverage-calc index a146667595..53e818fc32 100755 --- a/testing/scripts/coverage-calc +++ b/testing/scripts/coverage-calc @@ -3,10 +3,12 @@ # This script aggregates many files containing Bro script coverage information # into a single file and reports the overall coverage information. Usage: # -# coverage-calc +# coverage-calc