From 8fce51bf831536c8ce6007d466b343360fc98a1d Mon Sep 17 00:00:00 2001 From: FlyingWithJerome Date: Wed, 29 Sep 2021 10:24:16 -0400 Subject: [PATCH 001/210] initial commit for SVCB/HTTPS records --- .vscode/c_cpp_properties.json | 18 +++++++++ .vscode/settings.json | 5 +++ auxil/highwayhash | 2 +- doc | 2 +- scripts/base/init-bare.zeek | 6 +++ scripts/base/protocols/dns/consts.zeek | 11 ++++++ src/analyzer/protocol/dns/DNS.cc | 53 ++++++++++++++++++++++++++ src/analyzer/protocol/dns/DNS.h | 31 +++++++++++++++ src/analyzer/protocol/dns/events.bif | 31 +++++++++++++++ 9 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 .vscode/c_cpp_properties.json create mode 100644 .vscode/settings.json diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json new file mode 100644 index 0000000000..e48e2b73cd --- /dev/null +++ b/.vscode/c_cpp_properties.json @@ -0,0 +1,18 @@ +{ + "configurations": [ + { + "name": "Mac", + "includePath": [ + "${default}", + "${workspaceFolder}/**" + ], + "defines": [], + "macFrameworkPath": [], + "compilerPath": "/usr/local/bin/gcc-11", + "cStandard": "gnu17", + "cppStandard": "gnu++17", + "intelliSenseMode": "macos-gcc-x64" + } + ], + "version": 4 +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000..ed208baa59 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "files.associations": { + "thread": "cpp" + } +} \ No newline at end of file diff --git a/auxil/highwayhash b/auxil/highwayhash index ca0ff296b1..2361494e04 160000 --- a/auxil/highwayhash +++ b/auxil/highwayhash @@ -1 +1 @@ -Subproject commit ca0ff296b116354957276cef2c61be171ac2f897 +Subproject commit 2361494e0400d52eb76d2c6c62db72168ebe69d0 diff --git a/doc b/doc index fefd7e6ceb..f562d75aef 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit fefd7e6ceb67dd011c268c658171967f1281b970 +Subproject commit f562d75aef2e7c401913de122e4f888d42ae5f8e diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index dc6d8ce802..57ddd514db 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -3884,6 +3884,12 @@ type dns_loc_rr: record { is_query: count; ##< The RR is a query/Response. }; +type dns_svcb_rr: record { + svc_priority: count; ##< Service priority. (AliasMode? ServiceMode?) + target_name: string; ##< Target name, the hostname of the service endpoint. + svc_params: table[count] of vector of string; ##< service parameters as key-value pairs +} + # DNS answer types. # # .. zeek:see:: dns_answerr diff --git a/scripts/base/protocols/dns/consts.zeek b/scripts/base/protocols/dns/consts.zeek index a1e0f53a0b..283c2fc0b7 100644 --- a/scripts/base/protocols/dns/consts.zeek +++ b/scripts/base/protocols/dns/consts.zeek @@ -172,4 +172,15 @@ export { [4] = "SHA384", } &default = function(n: count): string { return fmt("digest-%d", n); }; + ## SVCB/HTTPS SvcParam keys, + ## as defined in https://www.ietf.org/archive/id/draft-ietf-dnsop-svcb-https-07.txt, sec 14.3.2 + const svcparam_keys = { + [0] = "mandatory", + [1] = "alpn", + [2] = "no-default-alpn", + [3] = "port", + [4] = "ipv4hint", + [5] = "ech", + [6] = "ipv6hint", + } &default = function(n: count): string { return fmt("key%d", n); }; } diff --git a/src/analyzer/protocol/dns/DNS.cc b/src/analyzer/protocol/dns/DNS.cc index fe5424c00e..72c88cc488 100644 --- a/src/analyzer/protocol/dns/DNS.cc +++ b/src/analyzer/protocol/dns/DNS.cc @@ -345,6 +345,14 @@ bool DNS_Interpreter::ParseAnswer(detail::DNS_MsgInfo* msg, const u_char*& data, status = ParseRR_LOC(msg, data, len, rdlength, msg_start); break; + case detail::TYPE_SVCB: + status = ParseRR_SVCB(msg, data, len, rdlength, msg_start, TYPE_SVCB); + break; + + case detail::TYPE_HTTPS: + status = ParseRR_SVCB(msg, data, len, rdlength, msg_start, TYPE_HTTPS); + break; + default: if ( dns_unknown_reply && ! msg->skip_event ) @@ -1687,6 +1695,39 @@ bool DNS_Interpreter::ParseRR_CAA(detail::DNS_MsgInfo* msg, const u_char*& data, return rdlength == 0; } +bool DNS_Interpreter::ParseRR_SVCB(detail::DNS_MsgInfo* msg, const u_char*& data, int& len, + int rdlength, const u_char* msg_start, const RR_Type& svcb_type) + { + unsigned short svc_priority = ExtractShort(data, len); + + u_char target_name[513]; + int name_len = sizeof(target_name) - 1; + u_char* name_end = ExtractName(data, len, target_name, name_len, msg_start, false); + if ( ! name_end ) + return false; + + SVCB_DATA svcb_data = { + .svc_priority = svc_priority, + .target_name = new String(target_name, name_end - target_name, true), + .svc_params = nullptr, + }; + + // TODO: parse svcparams + + switch( svcb_type ) + { + case detail::TYPE_SVCB: + analyzer->EnqueueConnEvent(dns_SVCB, analyzer->ConnVal(), msg->BuildHdrVal(), + msg->BuildAnswerVal(), msg->BuildSVCB_Val(&svcb_data)); + break; + case detail::TYPE_HTTPS: + analyzer->EnqueueConnEvent(dns_HTTPS, analyzer->ConnVal(), msg->BuildHdrVal(), + msg->BuildAnswerVal(), msg->BuildSVCB_Val(&svcb_data)); + break; + } + return true; + } + void DNS_Interpreter::SendReplyOrRejectEvent(detail::DNS_MsgInfo* msg, EventHandlerPtr event, const u_char*& data, int& len, String* question_name, String* original_name) @@ -1987,6 +2028,18 @@ RecordValPtr DNS_MsgInfo::BuildLOC_Val(LOC_DATA* loc) return r; } +RecordValPtr DNS_MsgInfo::BuildSVCB_Val(SVCB_DATA* svcb) + { + static auto dns_svcb_rr = id::find_type("dns_svcb_rr"); + auto r = make_intrusive(dns_svcb_rr); + + r->Assign(0, svcb->svc_priority); + r->Assign(1, make_intrusive(svcb->target_name)); + + // TODO: assign svcparams + return dns_svcb_rr; + } + } // namespace detail Contents_DNS::Contents_DNS(Connection* conn, bool orig, detail::DNS_Interpreter* arg_interp) diff --git a/src/analyzer/protocol/dns/DNS.h b/src/analyzer/protocol/dns/DNS.h index c1abaa06e2..0a0476c546 100644 --- a/src/analyzer/protocol/dns/DNS.h +++ b/src/analyzer/protocol/dns/DNS.h @@ -71,6 +71,8 @@ enum RR_Type TYPE_DS = 43, ///< Delegation signer (RFC 4034) TYPE_NSEC3 = 50, TYPE_NSEC3PARAM = 51, ///< Contains the NSEC3 parameters (RFC 5155) + TYPE_SVCB = 64, ///< SerViCe Binding (RFC draft: https://datatracker.ietf.org/doc/html/draft-ietf-dnsop-svcb-https-07#section-1.1) + TYPE_HTTPS = 65, ///< HTTPS record (HTTPS specific SVCB resource record) // Obsoleted TYPE_SPF = 99, ///< Alternative: storing SPF data in TXT records, using the same format (RFC ///< 4408). Support for it was discontinued in RFC 7208 @@ -148,6 +150,18 @@ enum DNSSEC_Digest SHA384 = 4, }; +///< all keys are defined in RFC draft https://datatracker.ietf.org/doc/html/draft-ietf-dnsop-svcb-https-07#section-14.3.2 +enum SVCPARAM_Key + { + mandatory = 0, + alpn = 1, + no_default_alpn = 2, + port = 3, + ipv4hint = 4, + ech = 5, + ipv6hint = 6, + }; + struct DNS_RawMsgHdr { unsigned short id; @@ -269,6 +283,20 @@ struct LOC_DATA unsigned long altitude; // 32 }; +struct SVCB_DATA + { + unsigned short svc_priority; // 2 + String* target_name; + SVCPARAM_KV* svc_params; + } + +struct SVCPARAM_KV + { + String* key; + String* value; + SVCPARAM_KV* next; + }; + class DNS_MsgInfo { public: @@ -288,6 +316,7 @@ public: RecordValPtr BuildDS_Val(struct DS_DATA*); RecordValPtr BuildBINDS_Val(struct BINDS_DATA*); RecordValPtr BuildLOC_Val(struct LOC_DATA*); + RecordValPtr BuildSVCB_Val(struct SVCB_DATA*); int id; int opcode; ///< query type, see DNS_Opcode @@ -393,6 +422,8 @@ protected: const u_char* msg_start); bool ParseRR_LOC(detail::DNS_MsgInfo* msg, const u_char*& data, int& len, int rdlength, const u_char* msg_start); + bool ParseRR_SVCB(detail::DNS_MsgInfo* msg, const u_char*& data, int& len, int rdlength, + const u_char* msg_start, const RR_Type& svcb_type); void SendReplyOrRejectEvent(detail::DNS_MsgInfo* msg, EventHandlerPtr event, const u_char*& data, int& len, String* question_name, String* original_name); diff --git a/src/analyzer/protocol/dns/events.bif b/src/analyzer/protocol/dns/events.bif index 40c6cf9f66..7e241c4d01 100644 --- a/src/analyzer/protocol/dns/events.bif +++ b/src/analyzer/protocol/dns/events.bif @@ -721,6 +721,37 @@ event dns_SSHFP%(c: connection, msg: dns_msg, ans: dns_answer, algo: count, fpty ## loc: The parsed RDATA of LOC type record. event dns_LOC%(c: connection, msg: dns_msg, ans: dns_answer, loc: dns_loc_rr%); +## Generated for DNS replies of type *SVCB*. +## See `RFC draft `__ +## for more information about the DNS SVCB/HTTPS resource records. +## For replies with multiple answers, an individual event of the corresponding type is raised for each. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## ans: The type-independent part of the parsed answer record. +## +## svcb: The parsed RDATA of SVCB type record. +event dns_SVCB%(c: connection, msg: dns_msg, ans: dns_answer, svcb: dns_svcb_rr%); + +## Generated for DNS replies of type *HTTPS*. +## See `RFC draft `__ +## for more information about the DNS SVCB/HTTPS resource records. +## Since SVCB and HTTPS record share the same wire format layout, the argument https is dns_svcb_rr. +## For replies with multiple answers, an individual event of the corresponding type is raised for each. +## +## c: The connection, which may be UDP or TCP depending on the type of the +## transport-layer session being analyzed. +## +## msg: The parsed DNS message header. +## +## ans: The type-independent part of the parsed answer record. +## +## https: The parsed RDATA of SVCB type record. +event dns_HTTPS%(c: connection, msg: dns_msg, ans: dns_answer, https: dns_svcb_rr%); + ## Generated at the end of processing a DNS packet. This event is the last ## ``dns_*`` event that will be raised for a DNS query/reply and signals that ## all resource records have been passed on. From 0849332eb959485ba587856671f517a472f2fe52 Mon Sep 17 00:00:00 2001 From: FlyingWithJerome Date: Wed, 29 Sep 2021 13:57:42 -0400 Subject: [PATCH 002/210] fix a few syntax errors --- conn.log | 10 ++++++++++ dns.log | 10 ++++++++++ packet_filter.log | 10 ++++++++++ scripts/base/init-bare.zeek | 2 +- src/analyzer/protocol/dns/DNS.cc | 4 ++-- src/analyzer/protocol/dns/DNS.h | 9 +-------- testing/btest/scripts/base/protocols/dns/svcb.zeek | 9 +++++++++ weird.log | 10 ++++++++++ 8 files changed, 53 insertions(+), 11 deletions(-) create mode 100644 conn.log create mode 100644 dns.log create mode 100644 packet_filter.log create mode 100644 testing/btest/scripts/base/protocols/dns/svcb.zeek create mode 100644 weird.log diff --git a/conn.log b/conn.log new file mode 100644 index 0000000000..227f0ebcbc --- /dev/null +++ b/conn.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2021-09-29-11-39-42 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] +1632928690.500620 CvVbVi3kNVC4LPU90f 192.168.1.99 62978 192.168.1.1 53 udp dns 0.027648 55 134 SF - - 0 Dd 1 83 1 162 - +#close 2021-09-29-11-39-42 diff --git a/dns.log b/dns.log new file mode 100644 index 0000000000..f519a96680 --- /dev/null +++ b/dns.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path dns +#open 2021-09-29-11-39-42 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id rtt query qclass qclass_name qtype qtype_name rcode rcode_name AA TC RD RA Z answers TTLs rejected auth addl +#types time string addr port addr port enum count interval string count string count string count string bool bool bool bool count vector[string] vector[interval] bool set[string] set[string] +1632928690.500620 CvVbVi3kNVC4LPU90f 192.168.1.99 62978 192.168.1.1 53 udp 62111 - cloudflare.com 1 C_INTERNET 65 HTTPS 0 NOERROR F F T F 2 - - F - - +#close 2021-09-29-11-39-42 diff --git a/packet_filter.log b/packet_filter.log new file mode 100644 index 0000000000..908eafb9ea --- /dev/null +++ b/packet_filter.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path packet_filter +#open 2021-09-29-11-39-42 +#fields ts node filter init success +#types time string string bool bool +1632929982.504499 zeek ip or not ip T T +#close 2021-09-29-11-39-42 diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 57ddd514db..2a9331fdc3 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -3888,7 +3888,7 @@ type dns_svcb_rr: record { svc_priority: count; ##< Service priority. (AliasMode? ServiceMode?) target_name: string; ##< Target name, the hostname of the service endpoint. svc_params: table[count] of vector of string; ##< service parameters as key-value pairs -} +}; # DNS answer types. # diff --git a/src/analyzer/protocol/dns/DNS.cc b/src/analyzer/protocol/dns/DNS.cc index 72c88cc488..b48e516707 100644 --- a/src/analyzer/protocol/dns/DNS.cc +++ b/src/analyzer/protocol/dns/DNS.cc @@ -1709,7 +1709,7 @@ bool DNS_Interpreter::ParseRR_SVCB(detail::DNS_MsgInfo* msg, const u_char*& data SVCB_DATA svcb_data = { .svc_priority = svc_priority, .target_name = new String(target_name, name_end - target_name, true), - .svc_params = nullptr, + .svc_params = Dictionary(), }; // TODO: parse svcparams @@ -2037,7 +2037,7 @@ RecordValPtr DNS_MsgInfo::BuildSVCB_Val(SVCB_DATA* svcb) r->Assign(1, make_intrusive(svcb->target_name)); // TODO: assign svcparams - return dns_svcb_rr; + return r; } } // namespace detail diff --git a/src/analyzer/protocol/dns/DNS.h b/src/analyzer/protocol/dns/DNS.h index 0a0476c546..cfc7b089c9 100644 --- a/src/analyzer/protocol/dns/DNS.h +++ b/src/analyzer/protocol/dns/DNS.h @@ -287,14 +287,7 @@ struct SVCB_DATA { unsigned short svc_priority; // 2 String* target_name; - SVCPARAM_KV* svc_params; - } - -struct SVCPARAM_KV - { - String* key; - String* value; - SVCPARAM_KV* next; + Dictionary svc_params; }; class DNS_MsgInfo diff --git a/testing/btest/scripts/base/protocols/dns/svcb.zeek b/testing/btest/scripts/base/protocols/dns/svcb.zeek new file mode 100644 index 0000000000..d7e9fa0590 --- /dev/null +++ b/testing/btest/scripts/base/protocols/dns/svcb.zeek @@ -0,0 +1,9 @@ +# @TEST-EXEC: zeek -C -r $TRACES/dns-https.pcap %INPUT > output +# @TEST-EXEC: btest-diff output + +@load policy/protocols/dns/auth-addl + +event dns_HTTPS(c: connection, msg: dns_msg, ans: dns_answer, https: dns_svcb_rr) + { + print https; + } \ No newline at end of file diff --git a/weird.log b/weird.log new file mode 100644 index 0000000000..ccab363330 --- /dev/null +++ b/weird.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#open 2021-09-29-11-39-42 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source +#types time string addr port addr port string string bool string string +1632928690.528268 CvVbVi3kNVC4LPU90f 192.168.1.99 62978 192.168.1.1 53 DNS_truncated_RR_rdlength_lt_len - F zeek DNS +#close 2021-09-29-11-39-42 From 8f7f75fcb4dd1a646264176a803d04ece37b884e Mon Sep 17 00:00:00 2001 From: FlyingWithJerome Date: Wed, 29 Sep 2021 13:59:40 -0400 Subject: [PATCH 003/210] remove test logs --- conn.log | 10 ---------- dns.log | 10 ---------- packet_filter.log | 10 ---------- weird.log | 10 ---------- 4 files changed, 40 deletions(-) delete mode 100644 conn.log delete mode 100644 dns.log delete mode 100644 packet_filter.log delete mode 100644 weird.log diff --git a/conn.log b/conn.log deleted file mode 100644 index 227f0ebcbc..0000000000 --- a/conn.log +++ /dev/null @@ -1,10 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path conn -#open 2021-09-29-11-39-42 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents -#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] -1632928690.500620 CvVbVi3kNVC4LPU90f 192.168.1.99 62978 192.168.1.1 53 udp dns 0.027648 55 134 SF - - 0 Dd 1 83 1 162 - -#close 2021-09-29-11-39-42 diff --git a/dns.log b/dns.log deleted file mode 100644 index f519a96680..0000000000 --- a/dns.log +++ /dev/null @@ -1,10 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path dns -#open 2021-09-29-11-39-42 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id rtt query qclass qclass_name qtype qtype_name rcode rcode_name AA TC RD RA Z answers TTLs rejected auth addl -#types time string addr port addr port enum count interval string count string count string count string bool bool bool bool count vector[string] vector[interval] bool set[string] set[string] -1632928690.500620 CvVbVi3kNVC4LPU90f 192.168.1.99 62978 192.168.1.1 53 udp 62111 - cloudflare.com 1 C_INTERNET 65 HTTPS 0 NOERROR F F T F 2 - - F - - -#close 2021-09-29-11-39-42 diff --git a/packet_filter.log b/packet_filter.log deleted file mode 100644 index 908eafb9ea..0000000000 --- a/packet_filter.log +++ /dev/null @@ -1,10 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path packet_filter -#open 2021-09-29-11-39-42 -#fields ts node filter init success -#types time string string bool bool -1632929982.504499 zeek ip or not ip T T -#close 2021-09-29-11-39-42 diff --git a/weird.log b/weird.log deleted file mode 100644 index ccab363330..0000000000 --- a/weird.log +++ /dev/null @@ -1,10 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path weird -#open 2021-09-29-11-39-42 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source -#types time string addr port addr port string string bool string string -1632928690.528268 CvVbVi3kNVC4LPU90f 192.168.1.99 62978 192.168.1.1 53 DNS_truncated_RR_rdlength_lt_len - F zeek DNS -#close 2021-09-29-11-39-42 From ca4b181d3516f948d85c9492c4301d2cfcc1d1f1 Mon Sep 17 00:00:00 2001 From: FlyingWithJerome Date: Wed, 29 Sep 2021 14:39:24 -0400 Subject: [PATCH 004/210] add a dns https test case --- src/analyzer/protocol/dns/DNS.cc | 10 ++++++++++ .../scripts.base.protocols.dns.https/output | 3 +++ testing/btest/Traces/dns-https.pcap | Bin 0 -> 329 bytes .../base/protocols/dns/{svcb.zeek => https.zeek} | 0 4 files changed, 13 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.protocols.dns.https/output create mode 100644 testing/btest/Traces/dns-https.pcap rename testing/btest/scripts/base/protocols/dns/{svcb.zeek => https.zeek} (100%) diff --git a/src/analyzer/protocol/dns/DNS.cc b/src/analyzer/protocol/dns/DNS.cc index b48e516707..c43888cb18 100644 --- a/src/analyzer/protocol/dns/DNS.cc +++ b/src/analyzer/protocol/dns/DNS.cc @@ -1706,6 +1706,15 @@ bool DNS_Interpreter::ParseRR_SVCB(detail::DNS_MsgInfo* msg, const u_char*& data if ( ! name_end ) return false; + // target name can be root - in this case the alternative endpoint is + // qname itself. make sure that we print "." instead of an empty string + if ( name_end - target_name == 0 ) + { + target_name[0] = '.'; + target_name[1] = '\0'; + name_end = target_name+1; + } + SVCB_DATA svcb_data = { .svc_priority = svc_priority, .target_name = new String(target_name, name_end - target_name, true), @@ -1724,6 +1733,7 @@ bool DNS_Interpreter::ParseRR_SVCB(detail::DNS_MsgInfo* msg, const u_char*& data analyzer->EnqueueConnEvent(dns_HTTPS, analyzer->ConnVal(), msg->BuildHdrVal(), msg->BuildAnswerVal(), msg->BuildSVCB_Val(&svcb_data)); break; + default: break; // unreachable. for suppressing compiler warnings. } return true; } diff --git a/testing/btest/Baseline/scripts.base.protocols.dns.https/output b/testing/btest/Baseline/scripts.base.protocols.dns.https/output new file mode 100644 index 0000000000..c94e491f7a --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.dns.https/output @@ -0,0 +1,3 @@ +[svc_priority=1, target_name=., svc_params={ + +}] \ No newline at end of file diff --git a/testing/btest/Traces/dns-https.pcap b/testing/btest/Traces/dns-https.pcap new file mode 100644 index 0000000000000000000000000000000000000000..b2c397e62e6cd614e7efe3af28da6dfd200399ab GIT binary patch literal 329 zcmca|c+)~A1{MYcU}0bcayB)GB=#(3XGjFHL3ql<`oO~aCC$&pDw#MKTp1XG112yq zI0!BaJ+Oi?8HgFbF)^4j*e_B4G@nrcYy<-XBUf@xerZZtPGV6ib8>zz1A`+-MpFQ2 zAQ12{a4~T3Uh$okllJX7*g}2|h7CXo5QbOQyKx09!#%kdK9*DIJjST)ERg4T0Oc} Date: Wed, 29 Sep 2021 18:06:58 -0400 Subject: [PATCH 005/210] add svcb test case --- .../scripts.base.protocols.dns.svcb/output | 3 +++ testing/btest/Traces/dns-svcb.pcap | Bin 0 -> 243 bytes .../btest/scripts/base/protocols/dns/svcb.zeek | 9 +++++++++ 3 files changed, 12 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.protocols.dns.svcb/output create mode 100644 testing/btest/Traces/dns-svcb.pcap create mode 100644 testing/btest/scripts/base/protocols/dns/svcb.zeek diff --git a/testing/btest/Baseline/scripts.base.protocols.dns.svcb/output b/testing/btest/Baseline/scripts.base.protocols.dns.svcb/output new file mode 100644 index 0000000000..15683c930c --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.dns.svcb/output @@ -0,0 +1,3 @@ +[svc_priority=0, target_name=foo.example.com, svc_params={ + +}] \ No newline at end of file diff --git a/testing/btest/Traces/dns-svcb.pcap b/testing/btest/Traces/dns-svcb.pcap new file mode 100644 index 0000000000000000000000000000000000000000..3060bdaaecd9555329d8972b037ba15951486f73 GIT binary patch literal 243 zcmca|c+)~A1{MYcfUtL84oTF!#>fx?lmKBSAa-S72-uXyz~CUrz);V?zzD(*&N7%X z*!=T9nZl^Rzz9?X6k|`VNX#wBNo7vX&jqRm32O>~r5Jb^xEMIrIa%DascE$U*->+y yks%!@2f|Q0l4mrc+QDE7w8j5l(#aH{1S7;23_CCcfaXOo2s1D+r{(9PNCE(EhB9pc literal 0 HcmV?d00001 diff --git a/testing/btest/scripts/base/protocols/dns/svcb.zeek b/testing/btest/scripts/base/protocols/dns/svcb.zeek new file mode 100644 index 0000000000..7a036f886b --- /dev/null +++ b/testing/btest/scripts/base/protocols/dns/svcb.zeek @@ -0,0 +1,9 @@ +# @TEST-EXEC: zeek -C -r $TRACES/dns-svcb.pcap %INPUT > output +# @TEST-EXEC: btest-diff output + +@load policy/protocols/dns/auth-addl + +event dns_SVCB(c: connection, msg: dns_msg, ans: dns_answer, svcb: dns_svcb_rr) + { + print svcb; + } \ No newline at end of file From 33c7fd5fba7fb944743aa731ac2ea4330fd67cc1 Mon Sep 17 00:00:00 2001 From: FlyingWithJerome Date: Wed, 29 Sep 2021 18:17:55 -0400 Subject: [PATCH 006/210] use tabs in init-bare.zeek --- scripts/base/init-bare.zeek | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 2a9331fdc3..7454a82ce5 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -3885,9 +3885,9 @@ type dns_loc_rr: record { }; type dns_svcb_rr: record { - svc_priority: count; ##< Service priority. (AliasMode? ServiceMode?) - target_name: string; ##< Target name, the hostname of the service endpoint. - svc_params: table[count] of vector of string; ##< service parameters as key-value pairs + svc_priority: count; ##< Service priority. (AliasMode? ServiceMode?) + target_name: string; ##< Target name, the hostname of the service endpoint. + svc_params: table[count] of vector of string; ##< service parameters as key-value pairs }; # DNS answer types. From b238cf3dcad04ce2bc3cb169d2a336e1349c1932 Mon Sep 17 00:00:00 2001 From: FlyingWithJerome Date: Wed, 29 Sep 2021 18:53:37 -0400 Subject: [PATCH 007/210] lazy commit --- scripts/base/init-bare.zeek | 2 +- src/analyzer/protocol/dns/DNS.cc | 6 +++--- src/analyzer/protocol/dns/events.bif | 16 ++++++++-------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 7454a82ce5..89510933c1 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -3887,7 +3887,7 @@ type dns_loc_rr: record { type dns_svcb_rr: record { svc_priority: count; ##< Service priority. (AliasMode? ServiceMode?) target_name: string; ##< Target name, the hostname of the service endpoint. - svc_params: table[count] of vector of string; ##< service parameters as key-value pairs + svc_params: table[count] of vector of string; ##< service parameters as key-value pairs (not used at this point) }; # DNS answer types. diff --git a/src/analyzer/protocol/dns/DNS.cc b/src/analyzer/protocol/dns/DNS.cc index c43888cb18..cb6b8cb5e8 100644 --- a/src/analyzer/protocol/dns/DNS.cc +++ b/src/analyzer/protocol/dns/DNS.cc @@ -1727,11 +1727,11 @@ bool DNS_Interpreter::ParseRR_SVCB(detail::DNS_MsgInfo* msg, const u_char*& data { case detail::TYPE_SVCB: analyzer->EnqueueConnEvent(dns_SVCB, analyzer->ConnVal(), msg->BuildHdrVal(), - msg->BuildAnswerVal(), msg->BuildSVCB_Val(&svcb_data)); + msg->BuildAnswerVal(), msg->BuildSVCB_Val(&svcb_data)); break; case detail::TYPE_HTTPS: analyzer->EnqueueConnEvent(dns_HTTPS, analyzer->ConnVal(), msg->BuildHdrVal(), - msg->BuildAnswerVal(), msg->BuildSVCB_Val(&svcb_data)); + msg->BuildAnswerVal(), msg->BuildSVCB_Val(&svcb_data)); break; default: break; // unreachable. for suppressing compiler warnings. } @@ -2046,7 +2046,7 @@ RecordValPtr DNS_MsgInfo::BuildSVCB_Val(SVCB_DATA* svcb) r->Assign(0, svcb->svc_priority); r->Assign(1, make_intrusive(svcb->target_name)); - // TODO: assign svcparams + // TODO: assign values to svcparams return r; } diff --git a/src/analyzer/protocol/dns/events.bif b/src/analyzer/protocol/dns/events.bif index 7e241c4d01..eed11bc820 100644 --- a/src/analyzer/protocol/dns/events.bif +++ b/src/analyzer/protocol/dns/events.bif @@ -721,9 +721,9 @@ event dns_SSHFP%(c: connection, msg: dns_msg, ans: dns_answer, algo: count, fpty ## loc: The parsed RDATA of LOC type record. event dns_LOC%(c: connection, msg: dns_msg, ans: dns_answer, loc: dns_loc_rr%); -## Generated for DNS replies of type *SVCB*. -## See `RFC draft `__ -## for more information about the DNS SVCB/HTTPS resource records. +## Generated for DNS replies of type *SVCB* (General Purpose Service Endpoints). +## See `RFC draft for DNS SVCB/HTTPS `__ +## for more information about DNS SVCB/HTTPS resource records. ## For replies with multiple answers, an individual event of the corresponding type is raised for each. ## ## c: The connection, which may be UDP or TCP depending on the type of the @@ -736,10 +736,10 @@ event dns_LOC%(c: connection, msg: dns_msg, ans: dns_answer, loc: dns_loc_rr%); ## svcb: The parsed RDATA of SVCB type record. event dns_SVCB%(c: connection, msg: dns_msg, ans: dns_answer, svcb: dns_svcb_rr%); -## Generated for DNS replies of type *HTTPS*. -## See `RFC draft `__ -## for more information about the DNS SVCB/HTTPS resource records. -## Since SVCB and HTTPS record share the same wire format layout, the argument https is dns_svcb_rr. +## Generated for DNS replies of type *HTTPS* (HTTPS Specific Service Endpoints). +## See `RFC draft for DNS SVCB/HTTPS `__ +## for more information about DNS SVCB/HTTPS resource records. +## Since SVCB and HTTPS records share the same wire format layout, the argument https is dns_svcb_rr. ## For replies with multiple answers, an individual event of the corresponding type is raised for each. ## ## c: The connection, which may be UDP or TCP depending on the type of the @@ -749,7 +749,7 @@ event dns_SVCB%(c: connection, msg: dns_msg, ans: dns_answer, svcb: dns_svcb_rr% ## ## ans: The type-independent part of the parsed answer record. ## -## https: The parsed RDATA of SVCB type record. +## https: The parsed RDATA of HTTPS type record. event dns_HTTPS%(c: connection, msg: dns_msg, ans: dns_answer, https: dns_svcb_rr%); ## Generated at the end of processing a DNS packet. This event is the last From ac1ea204fec06dfb2a0796718aefc1ec98f94dc9 Mon Sep 17 00:00:00 2001 From: FlyingWithJerome Date: Wed, 29 Sep 2021 20:02:29 -0400 Subject: [PATCH 008/210] newlines at the end of test outputs --- testing/btest/Baseline/scripts.base.protocols.dns.https/output | 2 +- testing/btest/Baseline/scripts.base.protocols.dns.svcb/output | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.protocols.dns.https/output b/testing/btest/Baseline/scripts.base.protocols.dns.https/output index c94e491f7a..739d29b9e8 100644 --- a/testing/btest/Baseline/scripts.base.protocols.dns.https/output +++ b/testing/btest/Baseline/scripts.base.protocols.dns.https/output @@ -1,3 +1,3 @@ [svc_priority=1, target_name=., svc_params={ -}] \ No newline at end of file +}] diff --git a/testing/btest/Baseline/scripts.base.protocols.dns.svcb/output b/testing/btest/Baseline/scripts.base.protocols.dns.svcb/output index 15683c930c..d6fcb527e3 100644 --- a/testing/btest/Baseline/scripts.base.protocols.dns.svcb/output +++ b/testing/btest/Baseline/scripts.base.protocols.dns.svcb/output @@ -1,3 +1,3 @@ [svc_priority=0, target_name=foo.example.com, svc_params={ -}] \ No newline at end of file +}] From c957e3e91e4024d109d9f10f0e7b6731a76016b4 Mon Sep 17 00:00:00 2001 From: FlyingWithJerome Date: Tue, 12 Oct 2021 20:36:35 -0400 Subject: [PATCH 009/210] address code reviews (formatting and type and intrusiveptr) --- .vscode/c_cpp_properties.json | 18 ---------------- .vscode/settings.json | 5 ----- auxil/package-manager | 2 +- scripts/base/init-bare.zeek | 2 +- scripts/base/protocols/dns/consts.zeek | 2 +- src/analyzer/protocol/dns/DNS.cc | 29 +++++++++++++++++--------- src/analyzer/protocol/dns/DNS.h | 21 +++++++++---------- 7 files changed, 32 insertions(+), 47 deletions(-) delete mode 100644 .vscode/c_cpp_properties.json delete mode 100644 .vscode/settings.json diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json deleted file mode 100644 index e48e2b73cd..0000000000 --- a/.vscode/c_cpp_properties.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "configurations": [ - { - "name": "Mac", - "includePath": [ - "${default}", - "${workspaceFolder}/**" - ], - "defines": [], - "macFrameworkPath": [], - "compilerPath": "/usr/local/bin/gcc-11", - "cStandard": "gnu17", - "cppStandard": "gnu++17", - "intelliSenseMode": "macos-gcc-x64" - } - ], - "version": 4 -} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index ed208baa59..0000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "files.associations": { - "thread": "cpp" - } -} \ No newline at end of file diff --git a/auxil/package-manager b/auxil/package-manager index feffa1d51e..9ccb796814 160000 --- a/auxil/package-manager +++ b/auxil/package-manager @@ -1 +1 @@ -Subproject commit feffa1d51e4e5494fef7daf2bd044138cb04f621 +Subproject commit 9ccb7968149ebf91a0c15ff04aca13e558a8b465 diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 89510933c1..aa98ecfe13 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -3885,7 +3885,7 @@ type dns_loc_rr: record { }; type dns_svcb_rr: record { - svc_priority: count; ##< Service priority. (AliasMode? ServiceMode?) + svc_priority: count; ##< Service priority for the current record, 0 indicates that this record is in AliasMode and cannot carry svc_params; otherwise this is in ServiceMode, and may include svc_params target_name: string; ##< Target name, the hostname of the service endpoint. svc_params: table[count] of vector of string; ##< service parameters as key-value pairs (not used at this point) }; diff --git a/scripts/base/protocols/dns/consts.zeek b/scripts/base/protocols/dns/consts.zeek index 283c2fc0b7..c780e6ffb4 100644 --- a/scripts/base/protocols/dns/consts.zeek +++ b/scripts/base/protocols/dns/consts.zeek @@ -182,5 +182,5 @@ export { [4] = "ipv4hint", [5] = "ech", [6] = "ipv6hint", - } &default = function(n: count): string { return fmt("key%d", n); }; + } &default = function(n: count): string { return fmt("key-%d", n); }; } diff --git a/src/analyzer/protocol/dns/DNS.cc b/src/analyzer/protocol/dns/DNS.cc index cb6b8cb5e8..789169261a 100644 --- a/src/analyzer/protocol/dns/DNS.cc +++ b/src/analyzer/protocol/dns/DNS.cc @@ -1698,7 +1698,12 @@ bool DNS_Interpreter::ParseRR_CAA(detail::DNS_MsgInfo* msg, const u_char*& data, bool DNS_Interpreter::ParseRR_SVCB(detail::DNS_MsgInfo* msg, const u_char*& data, int& len, int rdlength, const u_char* msg_start, const RR_Type& svcb_type) { - unsigned short svc_priority = ExtractShort(data, len); + // the smallest SVCB/HTTPS rr is 3 bytes: + // the first 2 bytes are for the svc priority, and the third byte is root (0x0) + if ( len < 3 ) + return false; + + uint16_t svc_priority = ExtractShort(data, len); u_char target_name[513]; int name_len = sizeof(target_name) - 1; @@ -1717,24 +1722,28 @@ bool DNS_Interpreter::ParseRR_SVCB(detail::DNS_MsgInfo* msg, const u_char*& data SVCB_DATA svcb_data = { .svc_priority = svc_priority, - .target_name = new String(target_name, name_end - target_name, true), - .svc_params = Dictionary(), + .target_name = make_intrusive(new String(target_name, name_end - target_name, true)), }; // TODO: parse svcparams + // we consume all the remaining raw data (svc params) but do nothing. + // this should be removed if the svc param parser is ready + String* unparsed_data = ExtractStream(data, len, rdlength); + delete unparsed_data; + switch( svcb_type ) - { + { case detail::TYPE_SVCB: analyzer->EnqueueConnEvent(dns_SVCB, analyzer->ConnVal(), msg->BuildHdrVal(), - msg->BuildAnswerVal(), msg->BuildSVCB_Val(&svcb_data)); + msg->BuildAnswerVal(), msg->BuildSVCB_Val(svcb_data)); break; case detail::TYPE_HTTPS: analyzer->EnqueueConnEvent(dns_HTTPS, analyzer->ConnVal(), msg->BuildHdrVal(), - msg->BuildAnswerVal(), msg->BuildSVCB_Val(&svcb_data)); + msg->BuildAnswerVal(), msg->BuildSVCB_Val(svcb_data)); break; default: break; // unreachable. for suppressing compiler warnings. - } + } return true; } @@ -2038,13 +2047,13 @@ RecordValPtr DNS_MsgInfo::BuildLOC_Val(LOC_DATA* loc) return r; } -RecordValPtr DNS_MsgInfo::BuildSVCB_Val(SVCB_DATA* svcb) +RecordValPtr DNS_MsgInfo::BuildSVCB_Val(const SVCB_DATA& svcb) { static auto dns_svcb_rr = id::find_type("dns_svcb_rr"); auto r = make_intrusive(dns_svcb_rr); - r->Assign(0, svcb->svc_priority); - r->Assign(1, make_intrusive(svcb->target_name)); + r->Assign(0, svcb.svc_priority); + r->Assign(1, svcb.target_name); // TODO: assign values to svcparams return r; diff --git a/src/analyzer/protocol/dns/DNS.h b/src/analyzer/protocol/dns/DNS.h index cfc7b089c9..f6f9a75f9a 100644 --- a/src/analyzer/protocol/dns/DNS.h +++ b/src/analyzer/protocol/dns/DNS.h @@ -153,13 +153,13 @@ enum DNSSEC_Digest ///< all keys are defined in RFC draft https://datatracker.ietf.org/doc/html/draft-ietf-dnsop-svcb-https-07#section-14.3.2 enum SVCPARAM_Key { - mandatory = 0, - alpn = 1, - no_default_alpn = 2, - port = 3, - ipv4hint = 4, - ech = 5, - ipv6hint = 6, + mandatory = 0, + alpn = 1, + no_default_alpn = 2, + port = 3, + ipv4hint = 4, + ech = 5, + ipv6hint = 6, }; struct DNS_RawMsgHdr @@ -285,9 +285,8 @@ struct LOC_DATA struct SVCB_DATA { - unsigned short svc_priority; // 2 - String* target_name; - Dictionary svc_params; + uint16_t svc_priority; // 2 + StringValPtr target_name; }; class DNS_MsgInfo @@ -309,7 +308,7 @@ public: RecordValPtr BuildDS_Val(struct DS_DATA*); RecordValPtr BuildBINDS_Val(struct BINDS_DATA*); RecordValPtr BuildLOC_Val(struct LOC_DATA*); - RecordValPtr BuildSVCB_Val(struct SVCB_DATA*); + RecordValPtr BuildSVCB_Val(const struct SVCB_DATA&); int id; int opcode; ///< query type, see DNS_Opcode From 605d4024e41e3b191ff2659a32f634f1a9083acd Mon Sep 17 00:00:00 2001 From: FlyingWithJerome Date: Tue, 12 Oct 2021 21:40:56 -0400 Subject: [PATCH 010/210] remove excussive fields in dns_svcb_rr --- scripts/base/init-bare.zeek | 1 - src/analyzer/protocol/dns/DNS.cc | 80 +++++++++---------- .../scripts.base.protocols.dns.https/output | 4 +- .../scripts.base.protocols.dns.svcb/output | 4 +- 4 files changed, 42 insertions(+), 47 deletions(-) diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index aa98ecfe13..ec13e3da90 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -3887,7 +3887,6 @@ type dns_loc_rr: record { type dns_svcb_rr: record { svc_priority: count; ##< Service priority for the current record, 0 indicates that this record is in AliasMode and cannot carry svc_params; otherwise this is in ServiceMode, and may include svc_params target_name: string; ##< Target name, the hostname of the service endpoint. - svc_params: table[count] of vector of string; ##< service parameters as key-value pairs (not used at this point) }; # DNS answer types. diff --git a/src/analyzer/protocol/dns/DNS.cc b/src/analyzer/protocol/dns/DNS.cc index 789169261a..868a26adde 100644 --- a/src/analyzer/protocol/dns/DNS.cc +++ b/src/analyzer/protocol/dns/DNS.cc @@ -1698,53 +1698,53 @@ bool DNS_Interpreter::ParseRR_CAA(detail::DNS_MsgInfo* msg, const u_char*& data, bool DNS_Interpreter::ParseRR_SVCB(detail::DNS_MsgInfo* msg, const u_char*& data, int& len, int rdlength, const u_char* msg_start, const RR_Type& svcb_type) { - // the smallest SVCB/HTTPS rr is 3 bytes: - // the first 2 bytes are for the svc priority, and the third byte is root (0x0) - if ( len < 3 ) - return false; + // the smallest SVCB/HTTPS rr is 3 bytes: + // the first 2 bytes are for the svc priority, and the third byte is root (0x0) + if ( len < 3 ) + return false; - uint16_t svc_priority = ExtractShort(data, len); + uint16_t svc_priority = ExtractShort(data, len); - u_char target_name[513]; - int name_len = sizeof(target_name) - 1; - u_char* name_end = ExtractName(data, len, target_name, name_len, msg_start, false); - if ( ! name_end ) - return false; + u_char target_name[513]; + int name_len = sizeof(target_name) - 1; + u_char* name_end = ExtractName(data, len, target_name, name_len, msg_start, false); + if ( ! name_end ) + return false; - // target name can be root - in this case the alternative endpoint is - // qname itself. make sure that we print "." instead of an empty string - if ( name_end - target_name == 0 ) - { - target_name[0] = '.'; - target_name[1] = '\0'; - name_end = target_name+1; - } + // target name can be root - in this case the alternative endpoint is + // qname itself. make sure that we print "." instead of an empty string + if ( name_end - target_name == 0 ) + { + target_name[0] = '.'; + target_name[1] = '\0'; + name_end = target_name+1; + } - SVCB_DATA svcb_data = { - .svc_priority = svc_priority, - .target_name = make_intrusive(new String(target_name, name_end - target_name, true)), - }; + SVCB_DATA svcb_data = { + .svc_priority = svc_priority, + .target_name = make_intrusive(new String(target_name, name_end - target_name, true)), + }; - // TODO: parse svcparams - // we consume all the remaining raw data (svc params) but do nothing. - // this should be removed if the svc param parser is ready - String* unparsed_data = ExtractStream(data, len, rdlength); - delete unparsed_data; + // TODO: parse svcparams + // we consume all the remaining raw data (svc params) but do nothing. + // this should be removed if the svc param parser is ready + String* unparsed_data = ExtractStream(data, len, rdlength); + delete unparsed_data; - switch( svcb_type ) - { - case detail::TYPE_SVCB: - analyzer->EnqueueConnEvent(dns_SVCB, analyzer->ConnVal(), msg->BuildHdrVal(), - msg->BuildAnswerVal(), msg->BuildSVCB_Val(svcb_data)); - break; - case detail::TYPE_HTTPS: - analyzer->EnqueueConnEvent(dns_HTTPS, analyzer->ConnVal(), msg->BuildHdrVal(), - msg->BuildAnswerVal(), msg->BuildSVCB_Val(svcb_data)); - break; - default: break; // unreachable. for suppressing compiler warnings. - } - return true; + switch( svcb_type ) + { + case detail::TYPE_SVCB: + analyzer->EnqueueConnEvent(dns_SVCB, analyzer->ConnVal(), msg->BuildHdrVal(), + msg->BuildAnswerVal(), msg->BuildSVCB_Val(svcb_data)); + break; + case detail::TYPE_HTTPS: + analyzer->EnqueueConnEvent(dns_HTTPS, analyzer->ConnVal(), msg->BuildHdrVal(), + msg->BuildAnswerVal(), msg->BuildSVCB_Val(svcb_data)); + break; + default: break; // unreachable. for suppressing compiler warnings. + } + return true; } void DNS_Interpreter::SendReplyOrRejectEvent(detail::DNS_MsgInfo* msg, EventHandlerPtr event, diff --git a/testing/btest/Baseline/scripts.base.protocols.dns.https/output b/testing/btest/Baseline/scripts.base.protocols.dns.https/output index 739d29b9e8..5f4d11d5dd 100644 --- a/testing/btest/Baseline/scripts.base.protocols.dns.https/output +++ b/testing/btest/Baseline/scripts.base.protocols.dns.https/output @@ -1,3 +1 @@ -[svc_priority=1, target_name=., svc_params={ - -}] +[svc_priority=1, target_name=.] diff --git a/testing/btest/Baseline/scripts.base.protocols.dns.svcb/output b/testing/btest/Baseline/scripts.base.protocols.dns.svcb/output index d6fcb527e3..233b838420 100644 --- a/testing/btest/Baseline/scripts.base.protocols.dns.svcb/output +++ b/testing/btest/Baseline/scripts.base.protocols.dns.svcb/output @@ -1,3 +1 @@ -[svc_priority=0, target_name=foo.example.com, svc_params={ - -}] +[svc_priority=0, target_name=foo.example.com] From dc6e21d6aec362790bceb8698a36f22247d24fff Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Thu, 14 Oct 2021 10:54:14 +0200 Subject: [PATCH 011/210] logging/writers/ascii: shadow files: Add fsync() before rename() We're using shadow files for log rotation on systems with ext4 running Linux 4.19. We've observed zero-length shadow files in the logger's working directory after a power-outage. This leads to a broken/stuck logger process due to empty shadow files being considered invalid and the process exiting: error: failed to process leftover log 'conn.log.gz': Found leftover log, 'conn.log.gz', but the associated shadow file, '.shadow.conn.log.gz', required to process it is invalid PR #1137 introduced atomic renaming of shadow files and was supposed to handle this. However, after more investigation, the rename() has to be preceded by an fsync() in order to avoid zero-length files in the presence of hard-crashes or power-failures. This is generally operating system and filesystem dependent, but should not hurt to add. The performance impact can likely be neglected due to the low frequency and limited number of log streams. This has happened to others, too. Some references around this issue: * https://stackoverflow.com/questions/7433057/is-rename-without-fsync-safe * https://unix.stackexchange.com/questions/464382/which-filesystems-require-fsync-for-crash-safety-when-replacing-an-existing-fi * https://bugzilla.kernel.org/show_bug.cgi?id=15910 Reproducer This issue was reproduced artificially on Linux using the sysrq-trigger functionality to hard-reset the system shortly after a .shadow file was renamed to it's final destination with the following script watching for .shadow.conn.log.gz: #!/bin/bash set -eu dir=/data/logger-01/ # Allow everything via /proc/sysrq-trigger echo "1" > /proc/sys/kernel/sysrq inotifywait -m -e MOVED_TO --format '%e %w%f' "${dir}" | while read -r line; do if echo "${line}" | grep -q '^MOVED_TO .*/.shadow.conn.log.gz$'; then echo "RESET: $line" sleep 4 # Trigger a hard-reset without sync/unmount echo "b" > /proc/sysrq-trigger fi done This quite reliably (4 out of 4 times) yielded a system with zero-length shadow files and a broken logger after it came back online: $ ls -lha /data/logger-01/.shadow.* -rw-r--r-- 1 bro bro 0 Oct 14 02:26 .shadow.conn.log.gz -rw-r--r-- 1 bro bro 0 Oct 14 02:26 .shadow.dns.log.gz -rw-r--r-- 1 bro bro 0 Oct 14 02:26 .shadow.files.log.gz After this change while running the reproducer, the shadow files always contained content after a hard-reset. Rework with util::safe_fsync helper --- src/logging/writers/ascii/Ascii.cc | 4 +++- src/util.cc | 18 ++++++++++++++++++ src/util.h | 4 ++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/logging/writers/ascii/Ascii.cc b/src/logging/writers/ascii/Ascii.cc index a53464f925..fe39b86c76 100644 --- a/src/logging/writers/ascii/Ascii.cc +++ b/src/logging/writers/ascii/Ascii.cc @@ -465,7 +465,7 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const threading::Fiel if ( sfd < 0 ) { - Error(Fmt("cannot open %s: %s", sfname.data(), Strerror(errno))); + Error(Fmt("cannot open %s: %s", tmp_sfname.data(), Strerror(errno))); return false; } @@ -479,6 +479,8 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const threading::Fiel util::safe_write(sfd, "\n", 1); + util::safe_fsync(sfd); + util::safe_close(sfd); if ( rename(tmp_sfname.data(), sfname.data()) == -1 ) diff --git a/src/util.cc b/src/util.cc index cfa612a864..51527303c0 100644 --- a/src/util.cc +++ b/src/util.cc @@ -2162,6 +2162,24 @@ bool safe_pwrite(int fd, const unsigned char* data, size_t len, size_t offset) return true; } +void safe_fsync(int fd) + { + int r; + + do + { + r = fsync(fd); + } while ( r < 0 && errno == EINTR ); + + if ( r < 0 ) + { + char buf[128]; + zeek_strerror_r(errno, buf, sizeof(buf)); + fprintf(stderr, "safe_fsync error %d: %s\n", errno, buf); + abort(); + } + } + void safe_close(int fd) { /* diff --git a/src/util.h b/src/util.h index fa07f585a7..851191246a 100644 --- a/src/util.h +++ b/src/util.h @@ -461,6 +461,10 @@ extern bool safe_write(int fd, const char* data, int len); // Same as safe_write(), but for pwrite(). extern bool safe_pwrite(int fd, const unsigned char* data, size_t len, size_t offset); +// Like fsync() but handles interrupted system calls by retrying and +// aborts on unrecoverable errors. +extern void safe_fsync(int fd); + // Wraps close(2) to emit error messages and abort on unrecoverable errors. extern void safe_close(int fd); From 5d3568d08d1f63ac1758c2243e161bdac2874648 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Thu, 14 Oct 2021 11:57:48 -0700 Subject: [PATCH 012/210] Bump zeek-aux submodule to pull in update-changes tweak [nomail] [skip ci] --- auxil/zeek-aux | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auxil/zeek-aux b/auxil/zeek-aux index 296383d577..00ec86d2da 160000 --- a/auxil/zeek-aux +++ b/auxil/zeek-aux @@ -1 +1 @@ -Subproject commit 296383d577a3f089c4f491061a985293cf6736e6 +Subproject commit 00ec86d2dad3c2d23438431a3ccb07e11dc59543 From 59013dedcf09b57d8b8eb38f2aff609ebeaec44e Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 14 Oct 2021 11:49:30 -0700 Subject: [PATCH 013/210] Fix typo in typedef changes that broke tests on 32-bit Debian 9 --- src/analyzer/Analyzer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/analyzer/Analyzer.h b/src/analyzer/Analyzer.h index 26bce61413..ec79400531 100644 --- a/src/analyzer/Analyzer.h +++ b/src/analyzer/Analyzer.h @@ -58,7 +58,7 @@ class OutputHandler; // causing a crash. using analyzer_list = std::list; using ID = uint32_t; -using analyzer_timer_func = void (Analyzer::*)(double_t); +using analyzer_timer_func = void (Analyzer::*)(double t); /** * Class to receive processed output from an anlyzer. From 237a21c3bf2f7bc4da147ca12e96ab771d384c0e Mon Sep 17 00:00:00 2001 From: zeek-bot Date: Wed, 20 Oct 2021 00:44:09 +0000 Subject: [PATCH 014/210] Update doc submodule [nomail] [skip ci] --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index fefd7e6ceb..e2f2af92c5 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit fefd7e6ceb67dd011c268c658171967f1281b970 +Subproject commit e2f2af92c55df3a27c31d8af79cb7a244af912b6 From a6378531dbc5c357926d98fe785bb719cc70e1b4 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 20 Oct 2021 09:55:11 -0700 Subject: [PATCH 015/210] Remove trailing whitespace from script files --- .../frameworks/cluster/nodes/manager.zeek | 2 +- scripts/base/frameworks/cluster/pools.zeek | 4 +- scripts/base/frameworks/dpd/main.zeek | 2 +- .../base/frameworks/files/magic/general.sig | 3 +- .../base/frameworks/files/magic/office.sig | 5 +- scripts/base/frameworks/files/main.zeek | 8 +-- scripts/base/frameworks/intel/main.zeek | 10 +-- .../base/frameworks/netcontrol/plugin.zeek | 2 +- .../frameworks/netcontrol/plugins/debug.zeek | 6 +- .../netcontrol/plugins/packetfilter.zeek | 3 +- scripts/base/frameworks/netcontrol/types.zeek | 2 +- .../notice/actions/add-geodata.zeek | 10 +-- .../notice/actions/email_admin.zeek | 4 +- scripts/base/frameworks/openflow/cluster.zeek | 4 +- scripts/base/frameworks/signatures/main.zeek | 25 ++++--- scripts/base/frameworks/software/main.zeek | 68 +++++++++---------- scripts/base/frameworks/sumstats/cluster.zeek | 2 +- scripts/base/frameworks/sumstats/main.zeek | 3 +- .../frameworks/sumstats/plugins/sample.zeek | 2 +- .../frameworks/sumstats/plugins/unique.zeek | 2 +- scripts/base/protocols/conn/contents.zeek | 8 +-- scripts/base/protocols/conn/inactivity.zeek | 6 +- scripts/base/protocols/dce-rpc/main.zeek | 2 +- scripts/base/protocols/dhcp/main.zeek | 2 +- scripts/base/protocols/dns/main.zeek | 8 +-- scripts/base/protocols/ftp/files.zeek | 6 +- .../base/protocols/ftp/utils-commands.zeek | 20 +++--- scripts/base/protocols/http/entities.zeek | 4 +- scripts/base/protocols/http/utils.zeek | 8 +-- scripts/base/protocols/irc/files.zeek | 2 +- scripts/base/protocols/irc/main.zeek | 16 ++--- scripts/base/protocols/krb/main.zeek | 10 +-- scripts/base/protocols/ntlm/main.zeek | 2 +- scripts/base/protocols/radius/main.zeek | 4 +- scripts/base/protocols/rdp/main.zeek | 22 +++--- scripts/base/protocols/smb/consts.zeek | 62 ++++++++--------- scripts/base/protocols/smb/files.zeek | 2 +- scripts/base/protocols/smb/main.zeek | 26 +++---- scripts/base/protocols/smb/smb1-main.zeek | 38 +++++------ scripts/base/protocols/smb/smb2-main.zeek | 10 +-- scripts/base/protocols/syslog/consts.zeek | 2 +- scripts/base/protocols/syslog/main.zeek | 4 +- scripts/base/utils/conn-ids.zeek | 8 +-- scripts/base/utils/directions-and-hosts.zeek | 2 +- scripts/base/utils/numbers.zeek | 5 +- scripts/base/utils/patterns.zeek | 2 +- scripts/base/utils/strings.zeek | 2 +- scripts/base/utils/thresholds.zeek | 8 +-- scripts/policy/files/unified2/main.zeek | 3 +- .../policy/frameworks/control/controller.zeek | 4 +- .../dpd/packet-segment-logging.zeek | 2 +- .../policy/frameworks/files/detect-MHR.zeek | 2 +- .../files/entropy-test-all-files.zeek | 5 +- .../frameworks/intel/seen/file-hashes.zeek | 2 +- .../policy/frameworks/intel/whitelist.zeek | 3 +- .../frameworks/software/version-changes.zeek | 4 +- .../policy/integration/barnyard2/main.zeek | 18 ++--- .../policy/integration/barnyard2/types.zeek | 2 +- scripts/policy/misc/trim-trace-file.zeek | 7 +- .../policy/protocols/conn/known-hosts.zeek | 10 +-- .../policy/protocols/conn/known-services.zeek | 3 +- .../protocols/dns/detect-external-names.zeek | 10 +-- scripts/policy/protocols/ftp/detect.zeek | 6 +- .../policy/protocols/http/detect-webapps.zeek | 2 +- .../policy/protocols/http/header-names.zeek | 6 +- .../protocols/http/var-extraction-uri.zeek | 2 +- .../policy/protocols/modbus/track-memmap.zeek | 6 +- scripts/policy/protocols/smb/log-cmds.zeek | 2 +- scripts/policy/protocols/smtp/blocklists.zeek | 16 ++--- .../smtp/detect-suspicious-orig.zeek | 2 +- scripts/policy/protocols/smtp/software.zeek | 23 +++---- .../protocols/ssh/interesting-hostnames.zeek | 11 ++- scripts/policy/protocols/ssh/software.zeek | 2 +- .../policy/protocols/ssl/expiring-certs.zeek | 2 +- scripts/policy/protocols/ssl/known-certs.zeek | 16 ++--- scripts/policy/tuning/__load__.zeek | 2 +- .../tuning/defaults/packet-fragments.zeek | 2 +- scripts/policy/tuning/defaults/warnings.zeek | 2 +- 78 files changed, 310 insertions(+), 325 deletions(-) diff --git a/scripts/base/frameworks/cluster/nodes/manager.zeek b/scripts/base/frameworks/cluster/nodes/manager.zeek index 53e3e52298..7504575dfc 100644 --- a/scripts/base/frameworks/cluster/nodes/manager.zeek +++ b/scripts/base/frameworks/cluster/nodes/manager.zeek @@ -2,7 +2,7 @@ ##! ##! The manager is passive (the workers connect to us), and once connected ##! the manager registers for the events on the workers that are needed -##! to get the desired data from the workers. This script will be +##! to get the desired data from the workers. This script will be ##! automatically loaded if necessary based on the type of node being started. ##! This is where the cluster manager sets it's specific settings for other diff --git a/scripts/base/frameworks/cluster/pools.zeek b/scripts/base/frameworks/cluster/pools.zeek index 1be7b336d8..8b2b9dee62 100644 --- a/scripts/base/frameworks/cluster/pools.zeek +++ b/scripts/base/frameworks/cluster/pools.zeek @@ -364,7 +364,7 @@ event zeek_init() &priority=-5 if ( manager_is_logger ) { local mgr = nodes_with_type(Cluster::MANAGER); - + if ( |mgr| > 0 ) { local eln = pool_eligibility[Cluster::LOGGER]$eligible_nodes; @@ -438,7 +438,7 @@ event zeek_init() &priority=-5 pet = pool_eligibility[pool$spec$node_type]; local nodes_to_init = |pet$eligible_nodes|; - + if ( pool$spec?$max_nodes && pool$spec$max_nodes < |pet$eligible_nodes| ) nodes_to_init = pool$spec$max_nodes; diff --git a/scripts/base/frameworks/dpd/main.zeek b/scripts/base/frameworks/dpd/main.zeek index 9424db3d5a..16e37dafdd 100644 --- a/scripts/base/frameworks/dpd/main.zeek +++ b/scripts/base/frameworks/dpd/main.zeek @@ -35,7 +35,7 @@ export { ## Number of protocol violations to tolerate before disabling an analyzer. option max_violations: table[Analyzer::Tag] of count = table() &default = 5; - ## Analyzers which you don't want to throw + ## Analyzers which you don't want to throw option ignore_violations: set[Analyzer::Tag] = set(); ## Ignore violations which go this many bytes into the connection. diff --git a/scripts/base/frameworks/files/magic/general.sig b/scripts/base/frameworks/files/magic/general.sig index d34c3d043d..f0e41018f6 100644 --- a/scripts/base/frameworks/files/magic/general.sig +++ b/scripts/base/frameworks/files/magic/general.sig @@ -252,7 +252,7 @@ signature file-mpqgame { file-magic /^MPQ\x1a/ } -# Blizzard CASC Format game file +# Blizzard CASC Format game file signature file-blizgame { file-mime "application/x-blizgame", 100 file-magic /^BLTE/ @@ -302,4 +302,3 @@ signature file-iso9660 { file-mime "application/x-iso9660-image", 99 file-magic /CD001/ } - diff --git a/scripts/base/frameworks/files/magic/office.sig b/scripts/base/frameworks/files/magic/office.sig index 3b3a264c24..b9563c0407 100644 --- a/scripts/base/frameworks/files/magic/office.sig +++ b/scripts/base/frameworks/files/magic/office.sig @@ -1,7 +1,6 @@ - # This signature is non-specific and terrible but after -# searching for a long time there doesn't seem to be a -# better option. +# searching for a long time there doesn't seem to be a +# better option. signature file-msword { file-magic /^\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1/ file-mime "application/msword", 50 diff --git a/scripts/base/frameworks/files/main.zeek b/scripts/base/frameworks/files/main.zeek index 10cc81ba56..5f799a26c4 100644 --- a/scripts/base/frameworks/files/main.zeek +++ b/scripts/base/frameworks/files/main.zeek @@ -104,7 +104,7 @@ export { missing_bytes: count &log &default=0; ## The number of bytes in the file stream that were not delivered to - ## stream file analyzers. This could be overlapping bytes or + ## stream file analyzers. This could be overlapping bytes or ## bytes that couldn't be reassembled. overflow_bytes: count &log &default=0; @@ -150,7 +150,7 @@ export { ## f: the file. global enable_reassembly: function(f: fa_file); - ## Disables the file reassembler on this file. If the file is not + ## Disables the file reassembler on this file. If the file is not ## transferred out of order this will have no effect. ## ## f: the file. @@ -266,7 +266,7 @@ export { }; ## Register callbacks for protocols that work with the Files framework. - ## The callbacks must uniquely identify a file and each protocol can + ## The callbacks must uniquely identify a file and each protocol can ## only have a single callback registered for it. ## ## tag: Tag for the protocol analyzer having a callback being registered. @@ -280,7 +280,7 @@ export { ## manipulation when they are being added to a file before the core code ## takes over. This is unlikely to be interesting for users and should ## only be called by file analyzer authors but is *not required*. - ## + ## ## tag: Tag for the file analyzer. ## ## callback: Function to execute when the given file analyzer is being added. diff --git a/scripts/base/frameworks/intel/main.zeek b/scripts/base/frameworks/intel/main.zeek index d88d0eb225..8464e7ebfe 100644 --- a/scripts/base/frameworks/intel/main.zeek +++ b/scripts/base/frameworks/intel/main.zeek @@ -49,7 +49,7 @@ export { ## A URL for more information about the data. url: string &optional; }; - + ## Represents a piece of intelligence. type Item: record { ## The intelligence indicator. @@ -57,12 +57,12 @@ export { ## The type of data that the indicator field represents. indicator_type: Type; - + ## Metadata for the item. Typically represents more deeply ## descriptive data for a piece of intelligence. meta: MetaData; }; - + ## Enum to represent where data came from when it was discovered. ## The convention is to prefix the name with ``IN_``. type Where: enum { @@ -158,8 +158,8 @@ export { global extend_match: hook(info: Info, s: Seen, items: set[Item]); ## The expiration timeout for intelligence items. Once an item expires, the - ## :zeek:id:`Intel::item_expired` hook is called. Reinsertion of an item - ## resets the timeout. A negative value disables expiration of intelligence + ## :zeek:id:`Intel::item_expired` hook is called. Reinsertion of an item + ## resets the timeout. A negative value disables expiration of intelligence ## items. const item_expiration = -1 min &redef; diff --git a/scripts/base/frameworks/netcontrol/plugin.zeek b/scripts/base/frameworks/netcontrol/plugin.zeek index 36d5a76173..8ecbef7911 100644 --- a/scripts/base/frameworks/netcontrol/plugin.zeek +++ b/scripts/base/frameworks/netcontrol/plugin.zeek @@ -41,7 +41,7 @@ export { name: function(state: PluginState) : string; ## If true, plugin can expire rules itself. If false, the NetControl - ## framework will manage rule expiration. + ## framework will manage rule expiration. can_expire: bool; ## One-time initialization function called when plugin gets registered, and diff --git a/scripts/base/frameworks/netcontrol/plugins/debug.zeek b/scripts/base/frameworks/netcontrol/plugins/debug.zeek index 479d934b6c..f159cda73f 100644 --- a/scripts/base/frameworks/netcontrol/plugins/debug.zeek +++ b/scripts/base/frameworks/netcontrol/plugins/debug.zeek @@ -46,7 +46,7 @@ function debug_add_rule(p: PluginState, r: Rule) : bool local s = fmt("add_rule: %s", r); debug_log(p, s); - if ( do_something(p) ) + if ( do_something(p) ) { event NetControl::rule_added(r, p); return T; @@ -76,12 +76,10 @@ global debug_plugin = Plugin( function create_debug(do_something: bool) : PluginState { local p: PluginState = [$plugin=debug_plugin]; - + # FIXME: Why's the default not working? p$config = table(); p$config["all"] = (do_something ? "1" : "0"); return p; } - - diff --git a/scripts/base/frameworks/netcontrol/plugins/packetfilter.zeek b/scripts/base/frameworks/netcontrol/plugins/packetfilter.zeek index 3648ed3955..ec3cc24247 100644 --- a/scripts/base/frameworks/netcontrol/plugins/packetfilter.zeek +++ b/scripts/base/frameworks/netcontrol/plugins/packetfilter.zeek @@ -1,7 +1,7 @@ ##! NetControl plugin for the process-level PacketFilter that comes with ##! Zeek. Since the PacketFilter in Zeek is quite limited in scope ##! and can only add/remove filters for addresses, this is quite -##! limited in scope at the moment. +##! limited in scope at the moment. @load ../plugin @@ -110,4 +110,3 @@ function create_packetfilter() : PluginState return p; } - diff --git a/scripts/base/frameworks/netcontrol/types.zeek b/scripts/base/frameworks/netcontrol/types.zeek index beac2302f6..731e8c38b0 100644 --- a/scripts/base/frameworks/netcontrol/types.zeek +++ b/scripts/base/frameworks/netcontrol/types.zeek @@ -1,7 +1,7 @@ ##! This file defines the types that are used by the NetControl framework. ##! ##! The most important type defined in this file is :zeek:see:`NetControl::Rule`, -##! which is used to describe all rules that can be expressed by the NetControl framework. +##! which is used to describe all rules that can be expressed by the NetControl framework. module NetControl; diff --git a/scripts/base/frameworks/notice/actions/add-geodata.zeek b/scripts/base/frameworks/notice/actions/add-geodata.zeek index 04cc10209d..6f07a9dd31 100644 --- a/scripts/base/frameworks/notice/actions/add-geodata.zeek +++ b/scripts/base/frameworks/notice/actions/add-geodata.zeek @@ -1,6 +1,6 @@ ##! This script adds geographic location data to notices for the "remote" -##! host in a connection. It does make the assumption that one of the -##! addresses in a connection is "local" and one is "remote" which is +##! host in a connection. It does make the assumption that one of the +##! addresses in a connection is "local" and one is "remote" which is ##! probably a safe assumption to make in most cases. If both addresses ##! are remote, it will use the $src address. @@ -17,13 +17,13 @@ export { ## in order for this to work. ACTION_ADD_GEODATA }; - + redef record Info += { ## If GeoIP support is built in, notices can have geographic ## information attached to them. remote_location: geo_location &log &optional; }; - + ## Notice types which should have the "remote" location looked up. ## If GeoIP support is not built in, this does nothing. option lookup_location_types: set[Notice::Type] = {}; @@ -35,7 +35,7 @@ hook policy(n: Notice::Info) &priority=10 add n$actions[ACTION_ADD_GEODATA]; } -# This is handled at a high priority in case other notice handlers +# This is handled at a high priority in case other notice handlers # want to use the data. hook notice(n: Notice::Info) &priority=10 { diff --git a/scripts/base/frameworks/notice/actions/email_admin.zeek b/scripts/base/frameworks/notice/actions/email_admin.zeek index 6ba5937bb7..71c26afef4 100644 --- a/scripts/base/frameworks/notice/actions/email_admin.zeek +++ b/scripts/base/frameworks/notice/actions/email_admin.zeek @@ -10,9 +10,9 @@ module Notice; export { redef enum Action += { - ## Indicate that the generated email should be addressed to the + ## Indicate that the generated email should be addressed to the ## appropriate email addresses as found by the - ## :zeek:id:`Site::get_emails` function based on the relevant + ## :zeek:id:`Site::get_emails` function based on the relevant ## address or addresses indicated in the notice. ACTION_EMAIL_ADMIN }; diff --git a/scripts/base/frameworks/openflow/cluster.zeek b/scripts/base/frameworks/openflow/cluster.zeek index 6ff005b877..0ab56ca293 100644 --- a/scripts/base/frameworks/openflow/cluster.zeek +++ b/scripts/base/frameworks/openflow/cluster.zeek @@ -112,12 +112,12 @@ function lookup_controller(name: string): vector of Controller if ( Cluster::local_node_type() != Cluster::MANAGER ) return vector(); - # I am not quite sure if we can actually get away with this - in the + # I am not quite sure if we can actually get away with this - in the # current state, this means that the individual nodes cannot lookup # a controller by name. # # This means that there can be no reactions to things on the actual - # worker nodes - because they cannot look up a name. On the other hand - + # worker nodes - because they cannot look up a name. On the other hand - # currently we also do not even send the events to the worker nodes (at least # not if we are using broker). Because of that I am not really feeling that # badly about it... diff --git a/scripts/base/frameworks/signatures/main.zeek b/scripts/base/frameworks/signatures/main.zeek index b235cba312..a610afccf5 100644 --- a/scripts/base/frameworks/signatures/main.zeek +++ b/scripts/base/frameworks/signatures/main.zeek @@ -60,7 +60,7 @@ export { SIG_ALARM_PER_ORIG, ## Alarm once and then never again. SIG_ALARM_ONCE, - ## Count signatures per responder host and alarm with the + ## Count signatures per responder host and alarm with the ## :zeek:enum:`Signatures::Count_Signature` notice if a threshold ## defined by :zeek:id:`Signatures::count_thresholds` is reached. SIG_COUNT_PER_RESP, @@ -100,15 +100,15 @@ export { ## Number of hosts, from a summary count. host_count: count &log &optional; }; - - ## Actions for a signature. + + ## Actions for a signature. const actions: table[string] of Action = { ["unspecified"] = SIG_IGNORE, # place-holder } &redef &default = SIG_ALARM; ## Signature IDs that should always be ignored. option ignored_ids = /NO_DEFAULT_MATCHES/; - + ## Generate a notice if, for a pair [orig, signature], the number of ## different responders has reached one of the thresholds. const horiz_scan_thresholds = { 5, 10, 50, 100, 500, 1000 } &redef; @@ -120,7 +120,7 @@ export { ## Generate a notice if a :zeek:enum:`Signatures::SIG_COUNT_PER_RESP` ## signature is triggered as often as given by one of these thresholds. const count_thresholds = { 5, 10, 50, 100, 500, 1000, 10000, 1000000, } &redef; - + ## The interval between when :zeek:enum:`Signatures::Signature_Summary` ## notices are generated. option summary_interval = 1 day; @@ -147,7 +147,7 @@ event zeek_init() &priority=5 { Log::create_stream(Signatures::LOG, [$columns=Info, $ev=log_signature, $path="signatures", $policy=log_policy]); } - + # Returns true if the given signature has already been triggered for the given # [orig, resp] pair. function has_signature_matched(id: string, orig: addr, resp: addr): bool @@ -173,7 +173,7 @@ event signature_match(state: signature_state, msg: string, data: string) # Trim the matched data down to something reasonable if ( |data| > 140 ) data = fmt("%s...", sub_bytes(data, 0, 140)); - + local src_addr: addr; local src_port: port; local dst_addr: addr; @@ -212,7 +212,7 @@ event signature_match(state: signature_state, msg: string, data: string) local notice = F; if ( action == SIG_ALARM ) notice = T; - + if ( action == SIG_COUNT_PER_RESP ) { local dst = state$conn$id$resp_h; @@ -252,7 +252,7 @@ event signature_match(state: signature_state, msg: string, data: string) $conn=state$conn, $src=src_addr, $dst=dst_addr, $msg=fmt("%s: %s", src_addr, msg), $sub=data]); - + if ( action == SIG_FILE_BUT_NO_SCAN || action == SIG_SUMMARY ) return; @@ -279,7 +279,7 @@ event signature_match(state: signature_state, msg: string, data: string) fmt("%s has triggered signature %s on %d hosts", orig, sig_id, hcount); - Log::write(Signatures::LOG, + Log::write(Signatures::LOG, [$ts=network_time(), $note=Multiple_Sig_Responders, $src_addr=orig, $sig_id=sig_id, $event_msg=msg, $host_count=hcount, $sub_msg=horz_scan_msg]); @@ -296,9 +296,9 @@ event signature_match(state: signature_state, msg: string, data: string) fmt("%s has triggered %d different signatures on host %s", orig, vcount, resp); - Log::write(Signatures::LOG, + Log::write(Signatures::LOG, [$ts=network_time(), - $note=Multiple_Signatures, + $note=Multiple_Signatures, $src_addr=orig, $dst_addr=resp, $sig_id=sig_id, $sig_count=vcount, $event_msg=fmt("%s different signatures triggered", vcount), @@ -311,4 +311,3 @@ event signature_match(state: signature_state, msg: string, data: string) last_vthresh[orig] = vcount; } } - diff --git a/scripts/base/frameworks/software/main.zeek b/scripts/base/frameworks/software/main.zeek index 5704ee98b9..9fed88668b 100644 --- a/scripts/base/frameworks/software/main.zeek +++ b/scripts/base/frameworks/software/main.zeek @@ -13,18 +13,18 @@ module Software; export { ## The software logging stream identifier. redef enum Log::ID += { LOG }; - + ## A default logging policy hook for the stream. global log_policy: Log::PolicyHook; ## Scripts detecting new types of software need to redef this enum to add - ## their own specific software types which would then be used when they + ## their own specific software types which would then be used when they ## create :zeek:type:`Software::Info` records. type Type: enum { ## A placeholder type for when the type of software is not known. UNKNOWN, }; - + ## A structure to represent the numeric version of software. type Version: record { ## Major version number. @@ -38,7 +38,7 @@ export { ## Additional version string (e.g. "beta42"). addl: string &optional; } &log; - + ## The record type that is used for representing and logging software. type Info: record { ## The time at which the software was detected. @@ -58,9 +58,9 @@ export { ## parsing doesn't always work reliably in all cases and this ## acts as a fallback in the logs. unparsed_version: string &log &optional; - + ## This can indicate that this software being detected should - ## definitely be sent onward to the logging framework. By + ## definitely be sent onward to the logging framework. By ## default, only software that is "interesting" due to a change ## in version or it being currently unknown is sent to the ## logging framework. This can be set to T to force the record @@ -68,7 +68,7 @@ export { ## tracking needs to happen in a specific way to the software. force_log: bool &default=F; }; - + ## Hosts whose software should be detected and tracked. ## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS. option asset_tracking = LOCAL_HOSTS; @@ -78,21 +78,21 @@ export { ## id: The connection id where the software was discovered. ## ## info: A record representing the software discovered. - ## + ## ## Returns: T if the software was logged, F otherwise. global found: function(id: conn_id, info: Info): bool; - + ## Compare two version records. - ## + ## ## Returns: -1 for v1 < v2, 0 for v1 == v2, 1 for v1 > v2. ## If the numerical version numbers match, the *addl* string ## is compared lexicographically. global cmp_versions: function(v1: Version, v2: Version): int; - - ## Sometimes software will expose itself on the network with - ## slight naming variations. This table provides a mechanism - ## for a piece of software to be renamed to a single name - ## even if it exposes itself with an alternate name. The + + ## Sometimes software will expose itself on the network with + ## slight naming variations. This table provides a mechanism + ## for a piece of software to be renamed to a single name + ## even if it exposes itself with an alternate name. The ## yielded string is the name that will be logged and generally ## used for everything. global alternate_names: table[string] of string { @@ -100,17 +100,17 @@ export { } &default=function(a: string): string { return a; }; ## Type to represent a collection of :zeek:type:`Software::Info` records. - ## It's indexed with the name of a piece of software such as "Firefox" + ## It's indexed with the name of a piece of software such as "Firefox" ## and it yields a :zeek:type:`Software::Info` record with more ## information about the software. type SoftwareSet: table[string] of Info; - + ## The set of software associated with an address. Data expires from - ## this table after one day by default so that a detected piece of + ## this table after one day by default so that a detected piece of ## software will be logged once each day. In a cluster, this table is ## uniformly distributed among proxy nodes. global tracked: table[addr] of SoftwareSet &create_expire=1day; - + ## This event can be handled to access the :zeek:type:`Software::Info` ## record as it is sent on to the logging framework. global log_software: event(rec: Info); @@ -128,7 +128,7 @@ event zeek_init() &priority=5 { Log::create_stream(Software::LOG, [$columns=Info, $ev=log_software, $path="software", $policy=log_policy]); } - + type Description: record { name: string; version: Version; @@ -138,13 +138,13 @@ type Description: record { # Defining this here because of a circular dependency between two functions. global parse_mozilla: function(unparsed_version: string): Description; -# Don't even try to understand this now, just make sure the tests are +# Don't even try to understand this now, just make sure the tests are # working. function parse(unparsed_version: string): Description { local software_name = ""; local v: Version; - + # Parse browser-alike versions separately if ( /^(Mozilla|Opera)\/[0-9]+\./ in unparsed_version ) { @@ -220,10 +220,10 @@ function parse(unparsed_version: string): Description { v$addl = strip(version_parts[2]); } - + } } - + if ( 3 in version_numbers && version_numbers[3] != "" ) v$minor3 = extract_count(version_numbers[3]); if ( 2 in version_numbers && version_numbers[2] != "" ) @@ -234,7 +234,7 @@ function parse(unparsed_version: string): Description v$major = extract_count(version_numbers[0]); } } - + return [$version=v, $unparsed_version=unparsed_version, $name=alternate_names[software_name]]; } @@ -245,7 +245,7 @@ function parse_with_cache(unparsed_version: string): Description { if (unparsed_version in parse_cache) return parse_cache[unparsed_version]; - + local res = parse(unparsed_version); parse_cache[unparsed_version] = res; return res; @@ -256,7 +256,7 @@ function parse_mozilla(unparsed_version: string): Description local software_name = ""; local v: Version; local parts: string_vec; - + if ( /Opera [0-9\.]*$/ in unparsed_version ) { software_name = "Opera"; @@ -349,7 +349,7 @@ function parse_mozilla(unparsed_version: string): Description if ( 2 in parts ) v = parse(parts[2])$version; } - + else if ( /AdobeAIR\/[0-9\.]*/ in unparsed_version ) { software_name = "AdobeAIR"; @@ -392,7 +392,7 @@ function cmp_versions(v1: Version, v2: Version): int else return v1?$major ? 1 : -1; } - + if ( v1?$minor && v2?$minor ) { if ( v1$minor < v2$minor ) @@ -407,7 +407,7 @@ function cmp_versions(v1: Version, v2: Version): int else return v1?$minor ? 1 : -1; } - + if ( v1?$minor2 && v2?$minor2 ) { if ( v1$minor2 < v2$minor2 ) @@ -462,7 +462,7 @@ function software_endpoint_name(id: conn_id, host: addr): string # Convert a version into a string "a.b.c-x". function software_fmt_version(v: Version): string { - return fmt("%s%s%s%s%s", + return fmt("%s%s%s%s%s", v?$major ? fmt("%d", v$major) : "0", v?$minor ? fmt(".%d", v$minor) : "", v?$minor2 ? fmt(".%d", v$minor2) : "", @@ -510,10 +510,10 @@ event Software::register(info: Info) local changed = cmp_versions(old$version, info$version) != 0; if ( changed ) - event Software::version_change(old, info); + event Software::version_change(old, info); else if ( ! info$force_log ) # If the version hasn't changed, then we're just redetecting the - # same thing, then we don't care. + # same thing, then we don't care. return; } @@ -526,7 +526,7 @@ function found(id: conn_id, info: Info): bool if ( ! info$force_log && ! addr_matches_host(info$host, asset_tracking) ) return F; - if ( ! info?$ts ) + if ( ! info?$ts ) info$ts = network_time(); if ( info?$version ) diff --git a/scripts/base/frameworks/sumstats/cluster.zeek b/scripts/base/frameworks/sumstats/cluster.zeek index 2296a4e38c..f055355170 100644 --- a/scripts/base/frameworks/sumstats/cluster.zeek +++ b/scripts/base/frameworks/sumstats/cluster.zeek @@ -220,7 +220,7 @@ event zeek_init() &priority=100 # This variable is maintained by manager nodes as they collect and aggregate # results. # Index on a uid. -global stats_keys: table[string] of set[Key] &read_expire=1min +global stats_keys: table[string] of set[Key] &read_expire=1min &expire_func=function(s: table[string] of set[Key], idx: string): interval { Reporter::warning(fmt("SumStat key request for the %s SumStat uid took longer than 1 minute and was automatically cancelled.", idx)); diff --git a/scripts/base/frameworks/sumstats/main.zeek b/scripts/base/frameworks/sumstats/main.zeek index 930125f6c8..8f8638b57b 100644 --- a/scripts/base/frameworks/sumstats/main.zeek +++ b/scripts/base/frameworks/sumstats/main.zeek @@ -510,7 +510,7 @@ function check_thresholds(ss: SumStat, key: Key, result: Result, modify_pct: dou return F; # Add in the extra ResultVals to make threshold_vals easier to write. - # This length comparison should work because we just need to make + # This length comparison should work because we just need to make # sure that we have the same number of reducers and results. if ( |ss$reducers| != |result| ) { @@ -568,4 +568,3 @@ function threshold_crossed(ss: SumStat, key: Key, result: Result) ss$threshold_crossed(key, result); } - diff --git a/scripts/base/frameworks/sumstats/plugins/sample.zeek b/scripts/base/frameworks/sumstats/plugins/sample.zeek index 2f96c5eb30..d5d236f43f 100644 --- a/scripts/base/frameworks/sumstats/plugins/sample.zeek +++ b/scripts/base/frameworks/sumstats/plugins/sample.zeek @@ -95,7 +95,7 @@ hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) { local other_vector: vector of Observation; local othercount: count; - + if ( rv1$sample_elements > rv2$sample_elements ) { result$samples = copy(rv1$samples); diff --git a/scripts/base/frameworks/sumstats/plugins/unique.zeek b/scripts/base/frameworks/sumstats/plugins/unique.zeek index 5fcaa1dc3c..069522effb 100644 --- a/scripts/base/frameworks/sumstats/plugins/unique.zeek +++ b/scripts/base/frameworks/sumstats/plugins/unique.zeek @@ -46,7 +46,7 @@ hook register_observe_plugins() if ( ! r?$unique_max || |rv$unique_vals| <= r$unique_max ) add rv$unique_vals[obs]; - + rv$unique = |rv$unique_vals|; }); } diff --git a/scripts/base/protocols/conn/contents.zeek b/scripts/base/protocols/conn/contents.zeek index ea689c6350..f3e64b00b1 100644 --- a/scripts/base/protocols/conn/contents.zeek +++ b/scripts/base/protocols/conn/contents.zeek @@ -1,5 +1,5 @@ -##! This script can be used to extract either the originator's data or the -##! responders data or both. By default nothing is extracted, and in order +##! This script can be used to extract either the originator's data or the +##! responders data or both. By default nothing is extracted, and in order ##! to actually extract data the ``c$extract_orig`` and/or the ##! ``c$extract_resp`` variable must be set to ``T``. One way to achieve this ##! would be to handle the :zeek:id:`connection_established` event elsewhere @@ -19,7 +19,7 @@ export { ## The prefix given to files containing extracted connections as they ## are opened on disk. option extraction_prefix = "contents"; - + ## If this variable is set to ``T``, then all contents of all ## connections will be extracted. option default_extract = F; @@ -38,7 +38,7 @@ event connection_established(c: connection) &priority=-5 local orig_f = open(orig_file); set_contents_file(c$id, CONTENTS_ORIG, orig_f); } - + if ( c$extract_resp ) { local resp_file = generate_extraction_filename(extraction_prefix, c, "resp.dat"); diff --git a/scripts/base/protocols/conn/inactivity.zeek b/scripts/base/protocols/conn/inactivity.zeek index b438a05b61..90c51c3909 100644 --- a/scripts/base/protocols/conn/inactivity.zeek +++ b/scripts/base/protocols/conn/inactivity.zeek @@ -10,14 +10,14 @@ export { # For interactive services, allow longer periods of inactivity. [[Analyzer::ANALYZER_SSH, Analyzer::ANALYZER_FTP]] = 1 hrs, }; - + ## Define inactivity timeouts based on common protocol ports. option port_inactivity_timeouts: table[port] of interval = { [[21/tcp, 22/tcp, 23/tcp, 513/tcp]] = 1 hrs, }; - + } - + event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) { if ( atype in analyzer_inactivity_timeouts ) diff --git a/scripts/base/protocols/dce-rpc/main.zeek b/scripts/base/protocols/dce-rpc/main.zeek index 2f69b33a7f..6092029e59 100644 --- a/scripts/base/protocols/dce-rpc/main.zeek +++ b/scripts/base/protocols/dce-rpc/main.zeek @@ -17,7 +17,7 @@ export { ## The connection's 4-tuple of endpoint addresses/ports. id : conn_id &log; ## Round trip time from the request to the response. - ## If either the request or response wasn't seen, + ## If either the request or response wasn't seen, ## this will be null. rtt : interval &log &optional; diff --git a/scripts/base/protocols/dhcp/main.zeek b/scripts/base/protocols/dhcp/main.zeek index b7a8f15649..d6a6807540 100644 --- a/scripts/base/protocols/dhcp/main.zeek +++ b/scripts/base/protocols/dhcp/main.zeek @@ -78,7 +78,7 @@ export { ## The DHCP message types seen by this DHCP transaction msg_types: vector of string &log &default=string_vec(); - ## Duration of the DHCP "session" representing the + ## Duration of the DHCP "session" representing the ## time from the first message to the last. duration: interval &log &default=0secs; diff --git a/scripts/base/protocols/dns/main.zeek b/scripts/base/protocols/dns/main.zeek index 85c90efadc..fa22cdba13 100644 --- a/scripts/base/protocols/dns/main.zeek +++ b/scripts/base/protocols/dns/main.zeek @@ -375,7 +375,7 @@ hook DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) if ( ! c$dns?$rtt ) { c$dns$rtt = network_time() - c$dns$ts; - # This could mean that only a reply was seen since + # This could mean that only a reply was seen since # we assume there must be some passage of time between # request and response. if ( c$dns$rtt == 0secs ) @@ -547,9 +547,9 @@ event dns_SRV_reply(c: connection, msg: dns_msg, ans: dns_answer, target: string # # } # event dns_EDNS_ecs(c: connection, msg: dns_msg, opt: dns_edns_ecs) -# { -# -# } +# { +# +# } # #event dns_TSIG_addl(c: connection, msg: dns_msg, ans: dns_tsig_additional) # { diff --git a/scripts/base/protocols/ftp/files.zeek b/scripts/base/protocols/ftp/files.zeek index f2c2625bdb..e7a200f927 100644 --- a/scripts/base/protocols/ftp/files.zeek +++ b/scripts/base/protocols/ftp/files.zeek @@ -18,14 +18,14 @@ export { ## Describe the file being transferred. global describe_file: function(f: fa_file): string; - redef record fa_file += { + redef record fa_file += { ftp: FTP::Info &optional; }; } function get_file_handle(c: connection, is_orig: bool): string { - if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) + if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) return ""; return cat(Analyzer::ANALYZER_FTP_DATA, c$start_time, c$id, is_orig); @@ -54,7 +54,7 @@ event zeek_init() &priority=5 event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5 { - if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) + if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) return; local ftp = ftp_data_expected[c$id$resp_h, c$id$resp_p]; diff --git a/scripts/base/protocols/ftp/utils-commands.zeek b/scripts/base/protocols/ftp/utils-commands.zeek index 43519b8422..5b75bd5032 100644 --- a/scripts/base/protocols/ftp/utils-commands.zeek +++ b/scripts/base/protocols/ftp/utils-commands.zeek @@ -11,12 +11,12 @@ export { ## Counter to track how many commands have been executed. seq: count &default=0; }; - + ## Structure for tracking pending commands in the event that the client - ## sends a large number of commands before the server has a chance to + ## sends a large number of commands before the server has a chance to ## reply. type PendingCmds: table[count] of CmdArg; - + ## Possible response codes for a wide variety of FTP commands. option cmd_reply_code: set[string, count] = { # According to RFC 959 @@ -65,7 +65,7 @@ export { ["MDTM", [213, 500, 501, 550]], # RFC3659 ["MLST", [150, 226, 250, 500, 501, 550]], # RFC3659 ["MLSD", [150, 226, 250, 500, 501, 550]], # RFC3659 - + ["CLNT", [200, 500]], # No RFC (indicate client software) ["MACB", [200, 500, 550]], # No RFC (test for MacBinary support) @@ -79,11 +79,11 @@ function add_pending_cmd(pc: PendingCmds, cmd: string, arg: string): CmdArg { local ca = [$cmd = cmd, $arg = arg, $seq=|pc|+1, $ts=network_time()]; pc[ca$seq] = ca; - + return ca; } -# Determine which is the best command to match with based on the +# Determine which is the best command to match with based on the # response code and message. function get_pending_cmd(pc: PendingCmds, reply_code: count, reply_msg: string): CmdArg { @@ -94,18 +94,18 @@ function get_pending_cmd(pc: PendingCmds, reply_code: count, reply_msg: string): for ( cmd_seq, cmd in pc ) { local score: int = 0; - + # if the command is compatible with the reply code # code 500 (syntax error) is compatible with all commands if ( reply_code == 500 || [cmd$cmd, reply_code] in cmd_reply_code ) score = score + 100; - + # if the command or the command arg appears in the reply message if ( strstr(reply_msg, cmd$cmd) > 0 ) score = score + 20; if ( strstr(reply_msg, cmd$arg) > 0 ) score = score + 10; - + if ( score > best_score || ( score == best_score && best_seq > cmd_seq ) ) # break tie with sequence number { @@ -132,7 +132,7 @@ function remove_pending_cmd(pc: PendingCmds, ca: CmdArg): bool else return F; } - + function pop_pending_cmd(pc: PendingCmds, reply_code: count, reply_msg: string): CmdArg { local ca = get_pending_cmd(pc, reply_code, reply_msg); diff --git a/scripts/base/protocols/http/entities.zeek b/scripts/base/protocols/http/entities.zeek index 0a72c6b76e..b0689c5478 100644 --- a/scripts/base/protocols/http/entities.zeek +++ b/scripts/base/protocols/http/entities.zeek @@ -97,7 +97,7 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5 { - if ( f$source == "HTTP" && c?$http ) + if ( f$source == "HTTP" && c?$http ) { f$http = c$http; @@ -199,6 +199,6 @@ event file_sniff(f: fa_file, meta: fa_metadata) &priority=5 event http_end_entity(c: connection, is_orig: bool) &priority=5 { - if ( c?$http && c$http?$current_entity ) + if ( c?$http && c$http?$current_entity ) delete c$http$current_entity; } diff --git a/scripts/base/protocols/http/utils.zeek b/scripts/base/protocols/http/utils.zeek index dcdbe4bc8e..5f4da1b77c 100644 --- a/scripts/base/protocols/http/utils.zeek +++ b/scripts/base/protocols/http/utils.zeek @@ -16,7 +16,7 @@ export { ## ## Returns: A vector of strings containing the keys. global extract_keys: function(data: string, kv_splitter: pattern): string_vec; - + ## Creates a URL from an :zeek:type:`HTTP::Info` record. This should ## handle edge cases such as proxied requests appropriately. ## @@ -24,7 +24,7 @@ export { ## ## Returns: A URL, not prefixed by ``"http://"``. global build_url: function(rec: Info): string; - + ## Creates a URL from an :zeek:type:`HTTP::Info` record. This should ## handle edge cases such as proxied requests appropriately. ## @@ -41,7 +41,7 @@ export { function extract_keys(data: string, kv_splitter: pattern): string_vec { local key_vec: vector of string = vector(); - + local parts = split_string(data, kv_splitter); for ( part_index in parts ) { @@ -64,7 +64,7 @@ function build_url(rec: Info): string host = fmt("%s:%d", host, resp_p); return fmt("%s%s", host, uri); } - + function build_url_http(rec: Info): string { return fmt("http://%s", build_url(rec)); diff --git a/scripts/base/protocols/irc/files.zeek b/scripts/base/protocols/irc/files.zeek index 59b178f4df..33128f57a6 100644 --- a/scripts/base/protocols/irc/files.zeek +++ b/scripts/base/protocols/irc/files.zeek @@ -31,7 +31,7 @@ event zeek_init() &priority=5 event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5 { - if ( [c$id$resp_h, c$id$resp_p] !in dcc_expected_transfers ) + if ( [c$id$resp_h, c$id$resp_p] !in dcc_expected_transfers ) return; local irc = dcc_expected_transfers[c$id$resp_h, c$id$resp_p]; diff --git a/scripts/base/protocols/irc/main.zeek b/scripts/base/protocols/irc/main.zeek index ae15098282..de4d2296ea 100644 --- a/scripts/base/protocols/irc/main.zeek +++ b/scripts/base/protocols/irc/main.zeek @@ -1,11 +1,11 @@ ##! Implements the core IRC analysis support. The logging model is to log -##! IRC commands along with the associated response and some additional +##! IRC commands along with the associated response and some additional ##! metadata about the connection if it's available. module IRC; export { - + redef enum Log::ID += { LOG }; global log_policy: Log::PolicyHook; @@ -21,7 +21,7 @@ export { nick: string &log &optional; ## Username given for the connection. user: string &log &optional; - + ## Command given by the client. command: string &log &optional; ## Value for the command given by the client. @@ -29,8 +29,8 @@ export { ## Any additional data for the command. addl: string &log &optional; }; - - ## Event that can be handled to access the IRC record as it is sent on + + ## Event that can be handled to access the IRC record as it is sent on ## to the logging framework. global irc_log: event(rec: Info); } @@ -48,7 +48,7 @@ event zeek_init() &priority=5 Log::create_stream(IRC::LOG, [$columns=Info, $ev=irc_log, $path="irc", $policy=log_policy]); Analyzer::register_for_ports(Analyzer::ANALYZER_IRC, ports); } - + function new_session(c: connection): Info { local info: Info; @@ -57,12 +57,12 @@ function new_session(c: connection): Info info$id = c$id; return info; } - + function set_session(c: connection) { if ( ! c?$irc ) c$irc = new_session(c); - + c$irc$ts=network_time(); } diff --git a/scripts/base/protocols/krb/main.zeek b/scripts/base/protocols/krb/main.zeek index d8fbbd12fc..16cdfac6f1 100644 --- a/scripts/base/protocols/krb/main.zeek +++ b/scripts/base/protocols/krb/main.zeek @@ -95,7 +95,7 @@ function set_session(c: connection): bool $id = c$id); Conn::register_removal_hook(c, finalize_krb); } - + return c$krb$logged; } @@ -115,7 +115,7 @@ event krb_error(c: connection, msg: Error_Msg) &priority=5 if ( msg?$error_text && msg$error_text in ignored_errors ) { - if ( c?$krb ) + if ( c?$krb ) delete c$krb; return; @@ -174,7 +174,7 @@ event krb_as_response(c: connection, msg: KDC_Response) &priority=5 if ( ! c$krb?$client && ( msg?$client_name || msg?$client_realm ) ) { - c$krb$client = fmt("%s/%s", msg?$client_name ? msg$client_name : "", + c$krb$client = fmt("%s/%s", msg?$client_name ? msg$client_name : "", msg?$client_realm ? msg$client_realm : ""); } @@ -202,7 +202,7 @@ event krb_tgs_request(c: connection, msg: KDC_Request) &priority=5 c$krb$request_type = "TGS"; if ( msg?$service_name ) c$krb$service = msg$service_name; - if ( msg?$from ) + if ( msg?$from ) c$krb$from = msg$from; if ( msg?$till ) c$krb$till = msg$till; @@ -221,7 +221,7 @@ event krb_tgs_response(c: connection, msg: KDC_Response) &priority=5 if ( ! c$krb?$client && ( msg?$client_name || msg?$client_realm ) ) { - c$krb$client = fmt("%s/%s", msg?$client_name ? msg$client_name : "", + c$krb$client = fmt("%s/%s", msg?$client_name ? msg$client_name : "", msg?$client_realm ? msg$client_realm : ""); } diff --git a/scripts/base/protocols/ntlm/main.zeek b/scripts/base/protocols/ntlm/main.zeek index 1aca98791e..17d8b85486 100644 --- a/scripts/base/protocols/ntlm/main.zeek +++ b/scripts/base/protocols/ntlm/main.zeek @@ -33,7 +33,7 @@ export { ## Indicate whether or not the authentication was successful. success : bool &log &optional; - ## Internally used field to indicate if the login attempt + ## Internally used field to indicate if the login attempt ## has already been logged. done: bool &default=F; }; diff --git a/scripts/base/protocols/radius/main.zeek b/scripts/base/protocols/radius/main.zeek index f5c1f2cbc3..dc692cac2f 100644 --- a/scripts/base/protocols/radius/main.zeek +++ b/scripts/base/protocols/radius/main.zeek @@ -24,7 +24,7 @@ export { mac : string &log &optional; ## The address given to the network access server, if ## present. This is only a hint from the RADIUS server - ## and the network access server is not required to honor + ## and the network access server is not required to honor ## the address. framed_addr : addr &log &optional; ## Address (IPv4, IPv6, or FQDN) of the initiator end of the tunnel, @@ -33,7 +33,7 @@ export { tunnel_client: string &log &optional; ## Connect info, if present. connect_info : string &log &optional; - ## Reply message from the server challenge. This is + ## Reply message from the server challenge. This is ## frequently shown to the user authenticating. reply_msg : string &log &optional; ## Successful or failed authentication. diff --git a/scripts/base/protocols/rdp/main.zeek b/scripts/base/protocols/rdp/main.zeek index 51c94d326a..53cc6ea812 100644 --- a/scripts/base/protocols/rdp/main.zeek +++ b/scripts/base/protocols/rdp/main.zeek @@ -41,15 +41,15 @@ export { desktop_width: count &log &optional; ## Desktop height of the client machine. desktop_height: count &log &optional; - ## The color depth requested by the client in + ## The color depth requested by the client in ## the high_color_depth field. requested_color_depth: string &log &optional; ## If the connection is being encrypted with native - ## RDP encryption, this is the type of cert + ## RDP encryption, this is the type of cert ## being used. cert_type: string &log &optional; - ## The number of certs seen. X.509 can transfer an + ## The number of certs seen. X.509 can transfer an ## entire certificate chain. cert_count: count &log &default=0; ## Indicates if the provided certificate or certificate @@ -57,7 +57,7 @@ export { cert_permanent: bool &log &optional; ## Encryption level of the connection. encryption_level: string &log &optional; - ## Encryption method of the connection. + ## Encryption method of the connection. encryption_method: string &log &optional; }; @@ -65,7 +65,7 @@ export { ## continuing to process encrypted traffic. option disable_analyzer_after_detection = F; - ## The amount of time to monitor an RDP session from when it is first + ## The amount of time to monitor an RDP session from when it is first ## identified. When this interval is reached, the session is logged. option rdp_check_interval = 10secs; @@ -113,7 +113,7 @@ function write_log(c: connection) info$done = T; # Verify that the RDP session contains - # RDP data before writing it to the log. + # RDP data before writing it to the log. if ( info?$cookie || info?$keyboard_layout || info?$result ) Log::write(RDP::LOG, info); } @@ -124,16 +124,16 @@ event check_record(c: connection) if ( c$rdp$done ) return; - # If the value rdp_check_interval has passed since the - # RDP session was started, then log the record. + # If the value rdp_check_interval has passed since the + # RDP session was started, then log the record. local diff = network_time() - c$rdp$ts; if ( diff > rdp_check_interval ) { write_log(c); # Remove the analyzer if it is still attached. - if ( disable_analyzer_after_detection && - connection_exists(c$id) && + if ( disable_analyzer_after_detection && + connection_exists(c$id) && c$rdp?$analyzer_id ) { disable_analyzer(c$id, c$rdp$analyzer_id); @@ -240,7 +240,7 @@ event rdp_server_certificate(c: connection, cert_type: count, permanently_issued # now so we manually count this one. if ( c$rdp$cert_type == "RSA" ) ++c$rdp$cert_count; - + c$rdp$cert_permanent = permanently_issued; } diff --git a/scripts/base/protocols/smb/consts.zeek b/scripts/base/protocols/smb/consts.zeek index 32a03dd17d..9b68419baa 100644 --- a/scripts/base/protocols/smb/consts.zeek +++ b/scripts/base/protocols/smb/consts.zeek @@ -107,13 +107,13 @@ export { } &redef &default=function(i: count):string { return fmt("unknown-wksta-command-%d", i); }; type rpc_cmd_table: table[count] of string; - + ## The subcommands for RPC endpoints. const rpc_sub_cmds: table[string] of rpc_cmd_table = { ["4b324fc8-1670-01d3-1278-5a47bf6ee188"] = srv_cmds, - ["6bffd098-a112-3610-9833-46c3f87e345a"] = wksta_cmds, + ["6bffd098-a112-3610-9833-46c3f87e345a"] = wksta_cmds, } &redef &default=function(i: string):rpc_cmd_table { return table() &default=function(j: string):string { return fmt("unknown-uuid-%s", j); }; }; - + } module SMB1; @@ -195,37 +195,37 @@ export { } &default=function(i: count):string { return fmt("unknown-%d", i); }; const trans2_sub_commands: table[count] of string = { - [0x00] = "OPEN2", - [0x01] = "FIND_FIRST2", - [0x02] = "FIND_NEXT2", - [0x03] = "QUERY_FS_INFORMATION", - [0x04] = "SET_FS_INFORMATION", - [0x05] = "QUERY_PATH_INFORMATION", - [0x06] = "SET_PATH_INFORMATION", - [0x07] = "QUERY_FILE_INFORMATION", - [0x08] = "SET_FILE_INFORMATION", - [0x09] = "FSCTL", - [0x0A] = "IOCTL", - [0x0B] = "FIND_NOTIFY_FIRST", - [0x0C] = "FIND_NOTIFY_NEXT", - [0x0D] = "CREATE_DIRECTORY", - [0x0E] = "SESSION_SETUP", - [0x10] = "GET_DFS_REFERRAL", - [0x11] = "REPORT_DFS_INCONSISTENCY", + [0x00] = "OPEN2", + [0x01] = "FIND_FIRST2", + [0x02] = "FIND_NEXT2", + [0x03] = "QUERY_FS_INFORMATION", + [0x04] = "SET_FS_INFORMATION", + [0x05] = "QUERY_PATH_INFORMATION", + [0x06] = "SET_PATH_INFORMATION", + [0x07] = "QUERY_FILE_INFORMATION", + [0x08] = "SET_FILE_INFORMATION", + [0x09] = "FSCTL", + [0x0A] = "IOCTL", + [0x0B] = "FIND_NOTIFY_FIRST", + [0x0C] = "FIND_NOTIFY_NEXT", + [0x0D] = "CREATE_DIRECTORY", + [0x0E] = "SESSION_SETUP", + [0x10] = "GET_DFS_REFERRAL", + [0x11] = "REPORT_DFS_INCONSISTENCY", } &default=function(i: count):string { return fmt("unknown-trans2-sub-cmd-%d", i); }; const trans_sub_commands: table[count] of string = { - [0x01] = "SET_NMPIPE_STATE", - [0x11] = "RAW_READ_NMPIPE", - [0x21] = "QUERY_NMPIPE_STATE", - [0x22] = "QUERY_NMPIPE_INFO", - [0x23] = "PEEK_NMPIPE", - [0x26] = "TRANSACT_NMPIPE", - [0x31] = "RAW_WRITE_NMPIPE", - [0x36] = "READ_NMPIPE", - [0x37] = "WRITE_NMPIPE", - [0x53] = "WAIT_NMPIPE", - [0x54] = "CALL_NMPIPE", + [0x01] = "SET_NMPIPE_STATE", + [0x11] = "RAW_READ_NMPIPE", + [0x21] = "QUERY_NMPIPE_STATE", + [0x22] = "QUERY_NMPIPE_INFO", + [0x23] = "PEEK_NMPIPE", + [0x26] = "TRANSACT_NMPIPE", + [0x31] = "RAW_WRITE_NMPIPE", + [0x36] = "READ_NMPIPE", + [0x37] = "WRITE_NMPIPE", + [0x53] = "WAIT_NMPIPE", + [0x54] = "CALL_NMPIPE", } &default=function(i: count):string { return fmt("unknown-trans-sub-cmd-%d", i); }; } diff --git a/scripts/base/protocols/smb/files.zeek b/scripts/base/protocols/smb/files.zeek index e3b387b771..a47874d480 100644 --- a/scripts/base/protocols/smb/files.zeek +++ b/scripts/base/protocols/smb/files.zeek @@ -14,7 +14,7 @@ export { function get_file_handle(c: connection, is_orig: bool): string { if ( ! (c$smb_state?$current_file && - (c$smb_state$current_file?$name || + (c$smb_state$current_file?$name || c$smb_state$current_file?$path)) ) { # TODO - figure out what are the cases where this happens. diff --git a/scripts/base/protocols/smb/main.zeek b/scripts/base/protocols/smb/main.zeek index cfccde16ac..703a76903d 100644 --- a/scripts/base/protocols/smb/main.zeek +++ b/scripts/base/protocols/smb/main.zeek @@ -5,7 +5,7 @@ module SMB; export { - redef enum Log::ID += { + redef enum Log::ID += { AUTH_LOG, MAPPING_LOG, FILES_LOG @@ -13,7 +13,7 @@ export { global log_policy_files: Log::PolicyHook; global log_policy_mapping: Log::PolicyHook; - + ## Abstracted actions for SMB file actions. type Action: enum { FILE_READ, @@ -55,7 +55,7 @@ export { id : conn_id &log; ## Unique ID of the file. fuid : string &log &optional; - + ## Action this log record represents. action : Action &log &optional; ## Path pulled from the tree this file was transferred to or from. @@ -99,14 +99,14 @@ export { uid : string &log; ## ID of the connection the request was sent over. id : conn_id &log; - + ## The command sent by the client. command : string &log; ## The subcommand sent by the client, if present. sub_command : string &log &optional; ## Command argument sent by the client, if any. argument : string &log &optional; - + ## Server reply to the client's command. status : string &log &optional; ## Round trip time from the request to the response. @@ -116,13 +116,13 @@ export { ## Authenticated username, if available. username : string &log &optional; - + ## If this is related to a tree, this is the tree ## that was used for the current command. tree : string &log &optional; ## The type of tree (disk share, printer share, named pipe, etc.). tree_service : string &log &optional; - + ## If the command referenced a file, store it here. referenced_file : FileInfo &log &optional; ## If the command referenced a tree, store it here. @@ -138,7 +138,7 @@ export { current_file : FileInfo &optional; ## A reference to the current tree. current_tree : TreeInfo &optional; - + ## Indexed on MID to map responses to requests. pending_cmds : table[count] of CmdInfo &optional; ## File map to retrieve file information based on the file ID. @@ -161,7 +161,7 @@ export { redef record connection += { smb_state : State &optional; }; - + ## This is an internally used function. const set_current_file: function(smb_state: State, file_id: count) &redef; @@ -195,7 +195,7 @@ function set_current_file(smb_state: State, file_id: count) smb_state$fid_map[file_id] = smb_state$current_cmd$referenced_file; smb_state$fid_map[file_id]$fid = file_id; } - + smb_state$current_cmd$referenced_file = smb_state$fid_map[file_id]; smb_state$current_file = smb_state$current_cmd$referenced_file; } @@ -203,7 +203,7 @@ function set_current_file(smb_state: State, file_id: count) function write_file_log(state: State) { local f = state$current_file; - if ( f?$name && + if ( f?$name && f$action in logged_file_actions ) { # Everything in this if statement is to avoid overlogging @@ -225,7 +225,7 @@ function write_file_log(state: State) else add state$recent_files[file_ident]; } - + Log::write(FILES_LOG, f); } } @@ -240,7 +240,7 @@ event file_state_remove(f: fa_file) &priority=-5 { if ( f$source != "SMB" ) return; - + for ( id, c in f$conns ) { if ( c?$smb_state && c$smb_state?$current_file) diff --git a/scripts/base/protocols/smb/smb1-main.zeek b/scripts/base/protocols/smb/smb1-main.zeek index 9dabdd1c36..a6bfcc8035 100644 --- a/scripts/base/protocols/smb/smb1-main.zeek +++ b/scripts/base/protocols/smb/smb1-main.zeek @@ -39,12 +39,12 @@ event smb1_message(c: connection, hdr: SMB1::Header, is_orig: bool) &priority=5 { smb_state$current_cmd$tree = smb_state$current_tree$path; } - + if ( smb_state$current_tree?$service ) { smb_state$current_cmd$tree_service = smb_state$current_tree$service; } - + if ( mid !in smb_state$pending_cmds ) { local tmp_cmd = SMB::CmdInfo($uid=c$uid, $id=c$id, $version="SMB1", $command = SMB1::commands[hdr$command]); @@ -52,10 +52,10 @@ event smb1_message(c: connection, hdr: SMB1::Header, is_orig: bool) &priority=5 local tmp_file = SMB::FileInfo($uid=c$uid, $id=c$id); tmp_cmd$referenced_file = tmp_file; tmp_cmd$referenced_tree = smb_state$current_tree; - + smb_state$pending_cmds[mid] = tmp_cmd; } - + smb_state$current_cmd = smb_state$pending_cmds[mid]; if ( !is_orig ) @@ -97,11 +97,11 @@ event smb1_negotiate_response(c: connection, hdr: SMB1::Header, response: SMB1:: delete c$smb_state$current_cmd$smb1_offered_dialects; } } - + event smb1_negotiate_response(c: connection, hdr: SMB1::Header, response: SMB1::NegotiateResponse) &priority=-5 { } - + event smb1_tree_connect_andx_request(c: connection, hdr: SMB1::Header, path: string, service: string) &priority=5 { local tmp_tree = SMB::TreeInfo($uid=c$uid, $id=c$id, $path=path, $service=service); @@ -117,7 +117,7 @@ event smb1_tree_connect_andx_response(c: connection, hdr: SMB1::Header, service: c$smb_state$current_cmd$referenced_tree$share_type = "PIPE"; c$smb_state$current_cmd$tree_service = service; - + if ( native_file_system != "" ) c$smb_state$current_cmd$referenced_tree$native_file_system = native_file_system; @@ -150,13 +150,13 @@ event smb1_nt_create_andx_response(c: connection, hdr: SMB1::Header, file_id: co # I'm seeing negative data from IPC tree transfers if ( time_to_double(times$modified) > 0.0 ) c$smb_state$current_cmd$referenced_file$times = times; - - # We can identify the file by its file id now so let's stick it + + # We can identify the file by its file id now so let's stick it # in the file map. c$smb_state$fid_map[file_id] = c$smb_state$current_cmd$referenced_file; - + c$smb_state$current_file = c$smb_state$fid_map[file_id]; - + SMB::write_file_log(c$smb_state); } @@ -167,7 +167,7 @@ event smb1_read_andx_request(c: connection, hdr: SMB1::Header, file_id: count, o if ( c$smb_state$current_file?$name ) c$smb_state$current_cmd$argument = c$smb_state$current_file$name; } - + event smb1_read_andx_request(c: connection, hdr: SMB1::Header, file_id: count, offset: count, length: count) &priority=-5 { if ( c$smb_state$current_tree?$path && !c$smb_state$current_file?$path ) @@ -180,12 +180,12 @@ event smb1_write_andx_request(c: connection, hdr: SMB1::Header, file_id: count, { SMB::set_current_file(c$smb_state, file_id); c$smb_state$current_file$action = SMB::FILE_WRITE; - if ( !c$smb_state$current_cmd?$argument && + if ( !c$smb_state$current_cmd?$argument && # TODO: figure out why name isn't getting set sometimes. c$smb_state$current_file?$name ) c$smb_state$current_cmd$argument = c$smb_state$current_file$name; } - + event smb1_write_andx_request(c: connection, hdr: SMB1::Header, file_id: count, offset: count, data_len: count) &priority=-5 { if ( c$smb_state$current_tree?$path && !c$smb_state$current_file?$path ) @@ -217,7 +217,7 @@ event smb1_close_request(c: connection, hdr: SMB1::Header, file_id: count) &prio if ( fl?$name ) c$smb_state$current_cmd$argument = fl$name; - + delete c$smb_state$fid_map[file_id]; SMB::write_file_log(c$smb_state); @@ -254,7 +254,7 @@ event smb1_session_setup_andx_response(c: connection, hdr: SMB1::Header, respons { # No behavior yet. } - + event smb1_transaction_request(c: connection, hdr: SMB1::Header, name: string, sub_cmd: count, parameters: string, data: string) { c$smb_state$current_cmd$sub_command = SMB1::trans_sub_commands[sub_cmd]; @@ -267,7 +267,7 @@ event smb1_write_andx_request(c: connection, hdr: SMB1::Header, file_id: count, # TODO: figure out why the uuid isn't getting set sometimes. return; } - + c$smb_state$pipe_map[file_id] = c$smb_state$current_file$uuid; } @@ -278,11 +278,11 @@ event smb_pipe_bind_ack_response(c: connection, hdr: SMB1::Header) # TODO: figure out why the uuid isn't getting set sometimes. return; } - + c$smb_state$current_cmd$sub_command = "RPC_BIND_ACK"; c$smb_state$current_cmd$argument = SMB::rpc_uuids[c$smb_state$current_file$uuid]; } - + event smb_pipe_bind_request(c: connection, hdr: SMB1::Header, uuid: string, version: string) { if ( ! c$smb_state?$current_file || ! c$smb_state$current_file?$uuid ) diff --git a/scripts/base/protocols/smb/smb2-main.zeek b/scripts/base/protocols/smb/smb2-main.zeek index 59436a2c8c..c45a56a799 100644 --- a/scripts/base/protocols/smb/smb2-main.zeek +++ b/scripts/base/protocols/smb/smb2-main.zeek @@ -19,7 +19,7 @@ event smb2_message(c: connection, hdr: SMB2::Header, is_orig: bool) &priority=5 state$pipe_map = table(); c$smb_state = state; } - + local smb_state = c$smb_state; local tid = hdr$tree_id; local mid = hdr$message_id; @@ -159,10 +159,10 @@ event smb2_create_response(c: connection, hdr: SMB2::Header, response: SMB2::Cre if ( time_to_double(response$times$modified) > 0.0 ) c$smb_state$current_file$times = response$times; - # We can identify the file by its file id now so let's stick it + # We can identify the file by its file id now so let's stick it # in the file map. c$smb_state$fid_map[response$file_id$persistent+response$file_id$volatile] = c$smb_state$current_file; - + c$smb_state$current_file = c$smb_state$fid_map[response$file_id$persistent+response$file_id$volatile]; } @@ -193,7 +193,7 @@ event smb2_read_request(c: connection, hdr: SMB2::Header, file_id: SMB2::GUID, o } event smb2_read_request(c: connection, hdr: SMB2::Header, file_id: SMB2::GUID, offset: count, length: count) &priority=-5 - { + { SMB::write_file_log(c$smb_state); } @@ -249,7 +249,7 @@ event smb2_file_rename(c: connection, hdr: SMB2::Header, file_id: SMB2::GUID, ds if ( c$smb_state$current_file?$name ) c$smb_state$current_file$prev_name = c$smb_state$current_file$name; - + c$smb_state$current_file$name = dst_filename; switch ( c$smb_state$current_tree$share_type ) diff --git a/scripts/base/protocols/syslog/consts.zeek b/scripts/base/protocols/syslog/consts.zeek index c68cbda658..9ec7c4ce12 100644 --- a/scripts/base/protocols/syslog/consts.zeek +++ b/scripts/base/protocols/syslog/consts.zeek @@ -31,7 +31,7 @@ export { [23] = "LOCAL7", [999] = "UNSPECIFIED", } &default=function(c: count): string { return fmt("?-%d", c); }; - + ## Mapping between the constants and string values for syslog severities. const severity_codes: table[count] of string = { [0] = "EMERG", diff --git a/scripts/base/protocols/syslog/main.zeek b/scripts/base/protocols/syslog/main.zeek index 612507c9c1..7a789a9400 100644 --- a/scripts/base/protocols/syslog/main.zeek +++ b/scripts/base/protocols/syslog/main.zeek @@ -1,4 +1,4 @@ -##! Core script support for logging syslog messages. This script represents +##! Core script support for logging syslog messages. This script represents ##! one syslog message as one logged record. @load ./consts @@ -52,7 +52,7 @@ event syslog_message(c: connection, facility: count, severity: count, msg: strin info$facility=facility_codes[facility]; info$severity=severity_codes[severity]; info$message=msg; - + c$syslog = info; } diff --git a/scripts/base/utils/conn-ids.zeek b/scripts/base/utils/conn-ids.zeek index b5d7fffd77..5222d1ce82 100644 --- a/scripts/base/utils/conn-ids.zeek +++ b/scripts/base/utils/conn-ids.zeek @@ -3,16 +3,16 @@ module GLOBAL; export { - ## Takes a conn_id record and returns a string representation with the + ## Takes a conn_id record and returns a string representation with the ## general data flow appearing to be from the connection originator ## on the left to the responder on the right. global id_string: function(id: conn_id): string; - - ## Takes a conn_id record and returns a string representation with the + + ## Takes a conn_id record and returns a string representation with the ## general data flow appearing to be from the connection responder ## on the right to the originator on the left. global reverse_id_string: function(id: conn_id): string; - + ## Calls :zeek:id:`id_string` or :zeek:id:`reverse_id_string` if the ## second argument is T or F, respectively. global directed_id_string: function(id: conn_id, is_orig: bool): string; diff --git a/scripts/base/utils/directions-and-hosts.zeek b/scripts/base/utils/directions-and-hosts.zeek index 442b4d8454..70f1e9aa2e 100644 --- a/scripts/base/utils/directions-and-hosts.zeek +++ b/scripts/base/utils/directions-and-hosts.zeek @@ -58,7 +58,7 @@ type Host: enum { function addr_matches_host(ip: addr, h: Host): bool { if ( h == NO_HOSTS ) return F; - + return ( h == ALL_HOSTS || (h == LOCAL_HOSTS && Site::is_local_addr(ip)) || (h == REMOTE_HOSTS && !Site::is_local_addr(ip)) ); diff --git a/scripts/base/utils/numbers.zeek b/scripts/base/utils/numbers.zeek index d2adb49ea2..41b8e601bb 100644 --- a/scripts/base/utils/numbers.zeek +++ b/scripts/base/utils/numbers.zeek @@ -1,8 +1,7 @@ - ## Extract an integer from a string. -## +## ## s: The string to search for a number. -## +## ## get_first: Provide `F` if you would like the last number found. ## ## Returns: The request integer from the given string or 0 if diff --git a/scripts/base/utils/patterns.zeek b/scripts/base/utils/patterns.zeek index c52556e86a..0fb7e0b72a 100644 --- a/scripts/base/utils/patterns.zeek +++ b/scripts/base/utils/patterns.zeek @@ -27,7 +27,7 @@ function set_to_regex(ss: set[string], pat: string): pattern for ( s in ss ) { local tmp_pattern = convert_for_pattern(s); - return_pat = ( i == 0 ) ? + return_pat = ( i == 0 ) ? tmp_pattern : cat(tmp_pattern, "|", return_pat); ++i; } diff --git a/scripts/base/utils/strings.zeek b/scripts/base/utils/strings.zeek index 4fa002acd6..e50954309f 100644 --- a/scripts/base/utils/strings.zeek +++ b/scripts/base/utils/strings.zeek @@ -25,7 +25,7 @@ function join_string_set(ss: set[string], j: string): string { if ( i > 0 ) output = cat(output, j); - + output = cat(output, s); ++i; } diff --git a/scripts/base/utils/thresholds.zeek b/scripts/base/utils/thresholds.zeek index d30e9f2b0a..d095edf007 100644 --- a/scripts/base/utils/thresholds.zeek +++ b/scripts/base/utils/thresholds.zeek @@ -16,13 +16,13 @@ export { ## for. index: count &default=0; }; - - ## The thresholds you would like to use as defaults with the + + ## The thresholds you would like to use as defaults with the ## :zeek:id:`default_check_threshold` function. const default_notice_thresholds: vector of count = { 30, 100, 1000, 10000, 100000, 1000000, 10000000, } &redef; - + ## This will check if a :zeek:type:`TrackCount` variable has crossed any ## thresholds in a given set. ## @@ -33,7 +33,7 @@ export { ## ## Returns: T if a threshold has been crossed, else F. global check_threshold: function(v: vector of count, tracker: TrackCount): bool; - + ## This will use the :zeek:id:`default_notice_thresholds` variable to ## check a :zeek:type:`TrackCount` variable to see if it has crossed ## another threshold. diff --git a/scripts/policy/files/unified2/main.zeek b/scripts/policy/files/unified2/main.zeek index c1ce27baf7..2930f483a0 100644 --- a/scripts/policy/files/unified2/main.zeek +++ b/scripts/policy/files/unified2/main.zeek @@ -1,4 +1,3 @@ - @load base/utils/dir @load base/utils/paths @@ -255,7 +254,7 @@ event file_new(f: fa_file) if ( |parts| == 3 ) file_dir = parts[0]; - if ( (watch_file != "" && f$source == watch_file) || + if ( (watch_file != "" && f$source == watch_file) || (watch_dir != "" && compress_path(watch_dir) == file_dir) ) { Files::add_analyzer(f, Files::ANALYZER_UNIFIED2); diff --git a/scripts/policy/frameworks/control/controller.zeek b/scripts/policy/frameworks/control/controller.zeek index b68f89b345..91820b7828 100644 --- a/scripts/policy/frameworks/control/controller.zeek +++ b/scripts/policy/frameworks/control/controller.zeek @@ -41,7 +41,7 @@ event Control::net_stats_response(s: string) &priority=-10 { event terminate_event(); } - + event Control::configuration_update_response() &priority=-10 { event terminate_event(); @@ -68,7 +68,7 @@ function configurable_ids(): id_table # We don't want to update non-const globals because that's usually # where state is stored and those values will frequently be declared # with &redef so that attributes can be redefined. - # + # # NOTE: functions are currently not fully supported for serialization and hence # aren't sent. if ( t$constant && t$redefinable && t$type_name != "func" ) diff --git a/scripts/policy/frameworks/dpd/packet-segment-logging.zeek b/scripts/policy/frameworks/dpd/packet-segment-logging.zeek index 7dff2b07f8..7386097551 100644 --- a/scripts/policy/frameworks/dpd/packet-segment-logging.zeek +++ b/scripts/policy/frameworks/dpd/packet-segment-logging.zeek @@ -24,6 +24,6 @@ event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count, reason: string) &priority=4 { if ( ! c?$dpd ) return; - + c$dpd$packet_segment=fmt("%s", sub_bytes(get_current_packet()$data, 0, packet_segment_size)); } diff --git a/scripts/policy/frameworks/files/detect-MHR.zeek b/scripts/policy/frameworks/files/detect-MHR.zeek index 0c95dadec4..52f8dd7355 100644 --- a/scripts/policy/frameworks/files/detect-MHR.zeek +++ b/scripts/policy/frameworks/files/detect-MHR.zeek @@ -66,7 +66,7 @@ function do_mhr_lookup(hash: string, fi: Notice::FileInfo) event file_hash(f: fa_file, kind: string, hash: string) { - if ( kind == "sha1" && f?$info && f$info?$mime_type && + if ( kind == "sha1" && f?$info && f$info?$mime_type && match_file_types in f$info$mime_type ) do_mhr_lookup(hash, Notice::create_file_info(f)); } diff --git a/scripts/policy/frameworks/files/entropy-test-all-files.zeek b/scripts/policy/frameworks/files/entropy-test-all-files.zeek index 9c704211f8..38c89e8c5d 100644 --- a/scripts/policy/frameworks/files/entropy-test-all-files.zeek +++ b/scripts/policy/frameworks/files/entropy-test-all-files.zeek @@ -1,10 +1,9 @@ - module Files; export { redef record Files::Info += { - ## The information density of the contents of the file, - ## expressed as a number of bits per character. + ## The information density of the contents of the file, + ## expressed as a number of bits per character. entropy: double &log &optional; }; } diff --git a/scripts/policy/frameworks/intel/seen/file-hashes.zeek b/scripts/policy/frameworks/intel/seen/file-hashes.zeek index 2e56ad3c48..e3295c5609 100644 --- a/scripts/policy/frameworks/intel/seen/file-hashes.zeek +++ b/scripts/policy/frameworks/intel/seen/file-hashes.zeek @@ -7,6 +7,6 @@ event file_hash(f: fa_file, kind: string, hash: string) $indicator_type=Intel::FILE_HASH, $f=f, $where=Files::IN_HASH); - + Intel::seen(seen); } \ No newline at end of file diff --git a/scripts/policy/frameworks/intel/whitelist.zeek b/scripts/policy/frameworks/intel/whitelist.zeek index 527d828881..e1aa150c7f 100644 --- a/scripts/policy/frameworks/intel/whitelist.zeek +++ b/scripts/policy/frameworks/intel/whitelist.zeek @@ -22,9 +22,8 @@ hook Intel::extend_match(info: Info, s: Seen, items: set[Item]) &priority=9 break; } } - + if ( whitelisted ) # Prevent logging break; } - diff --git a/scripts/policy/frameworks/software/version-changes.zeek b/scripts/policy/frameworks/software/version-changes.zeek index 865cc20447..060778584b 100644 --- a/scripts/policy/frameworks/software/version-changes.zeek +++ b/scripts/policy/frameworks/software/version-changes.zeek @@ -8,14 +8,14 @@ module Software; export { - redef enum Notice::Type += { + redef enum Notice::Type += { ## For certain software, a version changing may matter. In that ## case, this notice will be generated. Software that matters ## if the version changes can be configured with the ## :zeek:id:`Software::interesting_version_changes` variable. Software_Version_Change, }; - + ## Some software is more interesting when the version changes and this ## is a set of all software that should raise a notice when a different ## version is seen on a host. diff --git a/scripts/policy/integration/barnyard2/main.zeek b/scripts/policy/integration/barnyard2/main.zeek index 6e85db48d1..35f5a281ba 100644 --- a/scripts/policy/integration/barnyard2/main.zeek +++ b/scripts/policy/integration/barnyard2/main.zeek @@ -8,7 +8,7 @@ module Barnyard2; export { redef enum Log::ID += { LOG }; - + global log_policy: Log::PolicyHook; type Info: record { @@ -19,9 +19,9 @@ export { ## Associated alert data. alert: AlertData &log; }; - + ## This can convert a Barnyard :zeek:type:`Barnyard2::PacketID` value to - ## a :zeek:type:`conn_id` value in the case that you might need to index + ## a :zeek:type:`conn_id` value in the case that you might need to index ## into an existing data structure elsewhere within Zeek. global pid2cid: function(p: PacketID): conn_id; } @@ -40,22 +40,22 @@ function pid2cid(p: PacketID): conn_id event barnyard_alert(id: PacketID, alert: AlertData, msg: string, data: string) { Log::write(Barnyard2::LOG, [$ts=network_time(), $pid=id, $alert=alert]); - + #local proto_connection_string: string; #if ( id$src_p == 0/tcp ) # proto_connection_string = fmt("{PROTO:255} %s -> %s", id$src_ip, id$dst_ip); #else - # proto_connection_string = fmt("{%s} %s:%d -> %s:%d", + # proto_connection_string = fmt("{%s} %s:%d -> %s:%d", # to_upper(fmt("%s", get_port_transport_proto(id$dst_p))), # id$src_ip, id$src_p, id$dst_ip, id$dst_p); # - #local snort_alike_msg = fmt("%.6f [**] [%d:%d:%d] %s [**] [Classification: %s] [Priority: %d] %s", + #local snort_alike_msg = fmt("%.6f [**] [%d:%d:%d] %s [**] [Classification: %s] [Priority: %d] %s", # sad$ts, # sad$generator_id, # sad$signature_id, # sad$signature_revision, - # msg, - # sad$classification, - # sad$priority_id, + # msg, + # sad$classification, + # sad$priority_id, # proto_connection_string); } diff --git a/scripts/policy/integration/barnyard2/types.zeek b/scripts/policy/integration/barnyard2/types.zeek index da7015b302..ed8b35cf58 100644 --- a/scripts/policy/integration/barnyard2/types.zeek +++ b/scripts/policy/integration/barnyard2/types.zeek @@ -23,7 +23,7 @@ export { dst_p: port; } &log; - ## This is the event that Barnyard2 instances will send if they're + ## This is the event that Barnyard2 instances will send if they're ## configured with the bro_alert output plugin. global barnyard_alert: event(id: Barnyard2::PacketID, alert: Barnyard2::AlertData, diff --git a/scripts/policy/misc/trim-trace-file.zeek b/scripts/policy/misc/trim-trace-file.zeek index f702e9027c..4e31d5fc3e 100644 --- a/scripts/policy/misc/trim-trace-file.zeek +++ b/scripts/policy/misc/trim-trace-file.zeek @@ -6,7 +6,7 @@ module TrimTraceFile; export { ## The interval between times that the output tracefile is rotated. const trim_interval = 10 mins &redef; - + ## This event can be generated externally to this script if on-demand ## tracefile rotation is required with the caveat that the script ## doesn't currently attempt to get back on schedule automatically and @@ -19,14 +19,14 @@ event TrimTraceFile::go(first_trim: bool) { if ( zeek_is_terminating() || trace_output_file == "" ) return; - + if ( ! first_trim ) { local info = rotate_file_by_name(trace_output_file); if ( info$old_name != "" ) system(fmt("/bin/rm %s", safe_shell_quote(info$new_name))); } - + schedule trim_interval { TrimTraceFile::go(F) }; } @@ -35,4 +35,3 @@ event zeek_init() if ( trim_interval > 0 secs ) schedule trim_interval { TrimTraceFile::go(T) }; } - diff --git a/scripts/policy/protocols/conn/known-hosts.zeek b/scripts/policy/protocols/conn/known-hosts.zeek index b95c1176b2..279fa11917 100644 --- a/scripts/policy/protocols/conn/known-hosts.zeek +++ b/scripts/policy/protocols/conn/known-hosts.zeek @@ -1,5 +1,5 @@ ##! This script logs hosts that Zeek determines have performed complete TCP -##! handshakes and logs the address once per day (by default). The log that +##! handshakes and logs the address once per day (by default). The log that ##! is output provides an easy way to determine a count of the IP addresses in ##! use on a network per day. @@ -29,11 +29,11 @@ export { ## with keys uniformly distributed over proxy nodes in cluster ## operation. const use_host_store = T &redef; - + ## The hosts whose existence should be logged and tracked. ## See :zeek:type:`Host` for possible choices. option host_tracking = LOCAL_HOSTS; - + ## Holds the set of all known hosts. Keys in the store are addresses ## and their associated value will always be the "true" boolean. global host_store: Cluster::StoreInfo; @@ -49,8 +49,8 @@ export { ## :zeek:see:`Known::host_store`. option host_store_timeout = 15sec; - ## The set of all known addresses to store for preventing duplicate - ## logging of addresses. It can also be used from other scripts to + ## The set of all known addresses to store for preventing duplicate + ## logging of addresses. It can also be used from other scripts to ## inspect if an address has been seen in use. ## Maintain the list of known hosts for 24 hours so that the existence ## of each individual address is logged each day. diff --git a/scripts/policy/protocols/conn/known-services.zeek b/scripts/policy/protocols/conn/known-services.zeek index 7143c63547..336504fcf6 100644 --- a/scripts/policy/protocols/conn/known-services.zeek +++ b/scripts/policy/protocols/conn/known-services.zeek @@ -84,7 +84,7 @@ export { } redef record connection += { - # This field is to indicate whether or not the processing for detecting + # This field is to indicate whether or not the processing for detecting # and logging the service for this connection is complete. known_services_done: bool &default=F; }; @@ -314,4 +314,3 @@ event zeek_init() &priority=5 $path="known_services", $policy=log_policy_services]); } - diff --git a/scripts/policy/protocols/dns/detect-external-names.zeek b/scripts/policy/protocols/dns/detect-external-names.zeek index 9533f396a2..8798df6361 100644 --- a/scripts/policy/protocols/dns/detect-external-names.zeek +++ b/scripts/policy/protocols/dns/detect-external-names.zeek @@ -1,6 +1,6 @@ ##! This script detects names which are not within zones considered to be -##! local but resolving to addresses considered local. -##! The :zeek:id:`Site::local_zones` variable **must** be set appropriately for +##! local but resolving to addresses considered local. +##! The :zeek:id:`Site::local_zones` variable **must** be set appropriately for ##! this detection. @load base/frameworks/notice @@ -9,7 +9,7 @@ module DNS; export { - redef enum Notice::Type += { + redef enum Notice::Type += { ## Raised when a non-local name is found to be pointing at a ## local host. The :zeek:id:`Site::local_zones` variable ## **must** be set appropriately for this detection. @@ -21,7 +21,7 @@ event dns_A_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priori { if ( |Site::local_zones| == 0 ) return; - + # Check for responses from remote hosts that point at local hosts # but the name is not considered to be within a "local" zone. if ( Site::is_local_addr(a) && # referring to a local host @@ -29,7 +29,7 @@ event dns_A_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priori { NOTICE([$note=External_Name, $msg=fmt("%s is pointing to a local host - %s.", ans$query, a), - $conn=c, + $conn=c, $identifier=cat(a,ans$query)]); } } diff --git a/scripts/policy/protocols/ftp/detect.zeek b/scripts/policy/protocols/ftp/detect.zeek index e1bd627921..1b3128065a 100644 --- a/scripts/policy/protocols/ftp/detect.zeek +++ b/scripts/policy/protocols/ftp/detect.zeek @@ -7,7 +7,7 @@ module FTP; export { redef enum Notice::Type += { - ## Indicates that a successful response to a "SITE EXEC" + ## Indicates that a successful response to a "SITE EXEC" ## command/arg pair was seen. Site_Exec_Success, }; @@ -16,10 +16,10 @@ export { event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &priority=3 { local response_xyz = parse_ftp_reply_code(code); - + # If a successful SITE EXEC command is executed, raise a notice. if ( response_xyz$x == 2 && - c$ftp$cmdarg$cmd == "SITE" && + c$ftp$cmdarg$cmd == "SITE" && /[Ee][Xx][Ee][Cc]/ in c$ftp$cmdarg$arg ) { NOTICE([$note=Site_Exec_Success, $conn=c, diff --git a/scripts/policy/protocols/http/detect-webapps.zeek b/scripts/policy/protocols/http/detect-webapps.zeek index 29adbc6580..8b405eae9f 100644 --- a/scripts/policy/protocols/http/detect-webapps.zeek +++ b/scripts/policy/protocols/http/detect-webapps.zeek @@ -26,7 +26,7 @@ export { event signature_match(state: signature_state, msg: string, data: string) &priority=5 { if ( /^webapp-/ !in state$sig_id ) return; - + local c = state$conn; local si: Software::Info; si = [$name=msg, $unparsed_version=msg, $host=c$id$resp_h, $host_p=c$id$resp_p, $software_type=WEB_APPLICATION]; diff --git a/scripts/policy/protocols/http/header-names.zeek b/scripts/policy/protocols/http/header-names.zeek index 9f4e83638c..8838b41a0a 100644 --- a/scripts/policy/protocols/http/header-names.zeek +++ b/scripts/policy/protocols/http/header-names.zeek @@ -11,15 +11,15 @@ export { ## The vector of HTTP header names sent by the client. No ## header values are included here, just the header names. client_header_names: vector of string &log &optional; - + ## The vector of HTTP header names sent by the server. No ## header values are included here, just the header names. server_header_names: vector of string &log &optional; }; - + ## A boolean value to determine if client header names are to be logged. option log_client_header_names = T; - + ## A boolean value to determine if server header names are to be logged. option log_server_header_names = F; } diff --git a/scripts/policy/protocols/http/var-extraction-uri.zeek b/scripts/policy/protocols/http/var-extraction-uri.zeek index 98eba48fed..776c659530 100644 --- a/scripts/policy/protocols/http/var-extraction-uri.zeek +++ b/scripts/policy/protocols/http/var-extraction-uri.zeek @@ -1,4 +1,4 @@ -##! Extracts and logs variables from the requested URI in the default HTTP +##! Extracts and logs variables from the requested URI in the default HTTP ##! logging stream. @load base/protocols/http diff --git a/scripts/policy/protocols/modbus/track-memmap.zeek b/scripts/policy/protocols/modbus/track-memmap.zeek index c725a27241..b92e90b891 100644 --- a/scripts/policy/protocols/modbus/track-memmap.zeek +++ b/scripts/policy/protocols/modbus/track-memmap.zeek @@ -82,10 +82,10 @@ event modbus_read_holding_registers_response(c: connection, headers: ModbusHeade if ( slave_regs[c$modbus$track_address]$value != registers[i] ) { local delta = network_time() - slave_regs[c$modbus$track_address]$last_set; - event Modbus::changed_register(c, c$modbus$track_address, + event Modbus::changed_register(c, c$modbus$track_address, slave_regs[c$modbus$track_address]$value, registers[i], delta); - + slave_regs[c$modbus$track_address]$last_set = network_time(); slave_regs[c$modbus$track_address]$value = registers[i]; } @@ -102,7 +102,7 @@ event modbus_read_holding_registers_response(c: connection, headers: ModbusHeade event Modbus::changed_register(c: connection, register: count, old_val: count, new_val: count, delta: interval) { - local rec: MemmapInfo = [$ts=network_time(), $uid=c$uid, $id=c$id, + local rec: MemmapInfo = [$ts=network_time(), $uid=c$uid, $id=c$id, $register=register, $old_val=old_val, $new_val=new_val, $delta=delta]; Log::write(REGISTER_CHANGE_LOG, rec); } diff --git a/scripts/policy/protocols/smb/log-cmds.zeek b/scripts/policy/protocols/smb/log-cmds.zeek index 569314e980..0d5e4acde3 100644 --- a/scripts/policy/protocols/smb/log-cmds.zeek +++ b/scripts/policy/protocols/smb/log-cmds.zeek @@ -39,7 +39,7 @@ event smb1_message(c: connection, hdr: SMB1::Header, is_orig: bool) &priority=-5 if ( c$smb_state$current_cmd$status in SMB::ignored_command_statuses ) return; - + if ( c$smb_state$current_cmd$command in SMB::deferred_logging_cmds ) return; diff --git a/scripts/policy/protocols/smtp/blocklists.zeek b/scripts/policy/protocols/smtp/blocklists.zeek index 4524a6dabb..16292c4390 100644 --- a/scripts/policy/protocols/smtp/blocklists.zeek +++ b/scripts/policy/protocols/smtp/blocklists.zeek @@ -6,7 +6,7 @@ module SMTP; export { - redef enum Notice::Type += { + redef enum Notice::Type += { ## An SMTP server sent a reply mentioning an SMTP block list. Blocklist_Error_Message, ## The originator's address is seen in the block list error message. @@ -21,19 +21,19 @@ export { /spamhaus\.org\// | /sophos\.com\/security\// | /spamcop\.net\/bl/ - | /cbl\.abuseat\.org\// - | /sorbs\.net\// + | /cbl\.abuseat\.org\// + | /sorbs\.net\// | /bsn\.borderware\.com\// | /mail-abuse\.com\// | /b\.barracudacentral\.com\// - | /psbl\.surriel\.com\// - | /antispam\.imp\.ch\// + | /psbl\.surriel\.com\// + | /antispam\.imp\.ch\// | /dyndns\.com\/.*spam/ | /rbl\.knology\.net\// | /intercept\.datapacket\.net\// | /uceprotect\.net\// | /hostkarma\.junkemailfilter\.com\//; - + } event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string, @@ -55,8 +55,8 @@ event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string, note = Blocklist_Blocked_Host; message = fmt("%s is on an SMTP block list", c$id$orig_h); } - - NOTICE([$note=note, $conn=c, $msg=message, $sub=msg, + + NOTICE([$note=note, $conn=c, $msg=message, $sub=msg, $identifier=cat(c$id$orig_h)]); } } diff --git a/scripts/policy/protocols/smtp/detect-suspicious-orig.zeek b/scripts/policy/protocols/smtp/detect-suspicious-orig.zeek index 12a9a0c312..94edd62f27 100644 --- a/scripts/policy/protocols/smtp/detect-suspicious-orig.zeek +++ b/scripts/policy/protocols/smtp/detect-suspicious-orig.zeek @@ -24,7 +24,7 @@ event log_smtp(rec: Info) { ip = rec$x_originating_ip; loc = lookup_location(ip); - + if ( (loc?$country_code && loc$country_code in suspicious_origination_countries) || ip in suspicious_origination_networks ) diff --git a/scripts/policy/protocols/smtp/software.zeek b/scripts/policy/protocols/smtp/software.zeek index 342beedae0..06b4ca6c27 100644 --- a/scripts/policy/protocols/smtp/software.zeek +++ b/scripts/policy/protocols/smtp/software.zeek @@ -1,10 +1,10 @@ ##! This script feeds software detected through email into the software -##! framework. Mail clients and webmail interfaces are the only thing +##! framework. Mail clients and webmail interfaces are the only thing ##! currently detected. -##! +##! ##! TODO: ##! -##! * Find some heuristic to determine if email was sent through +##! * Find some heuristic to determine if email was sent through ##! a MS Exchange webmail interface as opposed to a desktop client. @load base/frameworks/software/main @@ -18,13 +18,13 @@ export { MAIL_SERVER, WEBMAIL_SERVER }; - + redef record Info += { ## Boolean indicator of if the message was sent through a ## webmail interface. is_webmail: bool &log &default=F; }; - + ## Assuming that local mail servers are more trustworthy with the ## headers they insert into message envelopes, this default makes Zeek ## not attempt to detect software in inbound message bodies. If mail @@ -34,15 +34,15 @@ export { ## incoming messages (network traffic originating from a non-local ## address), set this variable to EXTERNAL_HOSTS or ALL_HOSTS. option detect_clients_in_messages_from = LOCAL_HOSTS; - - ## A regular expression to match USER-AGENT-like headers to find if a + + ## A regular expression to match USER-AGENT-like headers to find if a ## message was sent with a webmail interface. option webmail_user_agents = - /^iPlanet Messenger/ + /^iPlanet Messenger/ | /^Sun Java\(tm\) System Messenger Express/ | /\(IMP\)/ # Horde Internet Messaging Program | /^SquirrelMail/ - | /^NeoMail/ + | /^NeoMail/ | /ZimbraWebClient/; } @@ -66,12 +66,12 @@ event log_smtp(rec: Info) { s_type = WEBMAIL_SERVER; # If the earliest received header indicates that the connection - # was via HTTP, then that likely means the actual mail software + # was via HTTP, then that likely means the actual mail software # is installed on the second address in the path. if ( rec?$first_received && /via HTTP/ in rec$first_received ) client_ip = rec$path[|rec$path|-2]; } - + if ( addr_matches_host(rec$id$orig_h, detect_clients_in_messages_from) ) { @@ -79,4 +79,3 @@ event log_smtp(rec: Info) } } } - diff --git a/scripts/policy/protocols/ssh/interesting-hostnames.zeek b/scripts/policy/protocols/ssh/interesting-hostnames.zeek index 92f7bfc1dd..db80d7c6ac 100644 --- a/scripts/policy/protocols/ssh/interesting-hostnames.zeek +++ b/scripts/policy/protocols/ssh/interesting-hostnames.zeek @@ -1,7 +1,7 @@ -##! This script will generate a notice if an apparent SSH login originates -##! or heads to a host with a reverse hostname that looks suspicious. By -##! default, the regular expression to match "interesting" hostnames includes -##! names that are typically used for infrastructure hosts like nameservers, +##! This script will generate a notice if an apparent SSH login originates +##! or heads to a host with a reverse hostname that looks suspicious. By +##! default, the regular expression to match "interesting" hostnames includes +##! names that are typically used for infrastructure hosts like nameservers, ##! mail servers, web servers and ftp servers. @load base/frameworks/notice @@ -15,7 +15,7 @@ export { ## :zeek:id:`SSH::interesting_hostnames` regular expression. Interesting_Hostname_Login, }; - + ## Strange/bad host names to see successful SSH logins from or to. option interesting_hostnames = /^d?ns[0-9]*\./ | @@ -49,4 +49,3 @@ event ssh_auth_successful(c: connection, auth_method_none: bool) check_ssh_hostname(c$id, c$uid, host); } } - diff --git a/scripts/policy/protocols/ssh/software.zeek b/scripts/policy/protocols/ssh/software.zeek index ba03bed284..4c44636914 100644 --- a/scripts/policy/protocols/ssh/software.zeek +++ b/scripts/policy/protocols/ssh/software.zeek @@ -1,4 +1,4 @@ -##! Extracts SSH client and server information from SSH +##! Extracts SSH client and server information from SSH ##! connections and forwards it to the software framework. @load base/frameworks/software diff --git a/scripts/policy/protocols/ssl/expiring-certs.zeek b/scripts/policy/protocols/ssl/expiring-certs.zeek index 066ff0d690..a217c03db4 100644 --- a/scripts/policy/protocols/ssl/expiring-certs.zeek +++ b/scripts/policy/protocols/ssl/expiring-certs.zeek @@ -1,4 +1,4 @@ -##! Generate notices when X.509 certificates over SSL/TLS are expired or +##! Generate notices when X.509 certificates over SSL/TLS are expired or ##! going to expire soon based on the date and time values stored within the ##! certificate. diff --git a/scripts/policy/protocols/ssl/known-certs.zeek b/scripts/policy/protocols/ssl/known-certs.zeek index 03b583cc9d..35fbcf0f7b 100644 --- a/scripts/policy/protocols/ssl/known-certs.zeek +++ b/scripts/policy/protocols/ssl/known-certs.zeek @@ -12,13 +12,13 @@ export { redef enum Log::ID += { CERTS_LOG }; global log_policy_certs: Log::PolicyHook; - + type CertsInfo: record { ## The timestamp when the certificate was detected. ts: time &log; ## The address that offered the certificate. host: addr &log; - ## If the certificate was handed out by a server, this is the + ## If the certificate was handed out by a server, this is the ## port that the server was listening on. port_num: port &log &optional; ## Certificate subject. @@ -28,7 +28,7 @@ export { ## Serial number for the certificate. serial: string &log &optional; }; - + ## The certificates whose existence should be logged and tracked. ## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS. option cert_tracking = LOCAL_HOSTS; @@ -38,7 +38,7 @@ export { ## with keys uniformly distributed over proxy nodes in cluster ## operation. const use_cert_store = T &redef; - + type AddrCertHashPair: record { host: addr; hash: string; @@ -60,15 +60,15 @@ export { ## :zeek:see:`Known::cert_store`. option cert_store_timeout = 15sec; - ## The set of all known certificates to store for preventing duplicate - ## logging. It can also be used from other scripts to - ## inspect if a certificate has been seen in use. The string value + ## The set of all known certificates to store for preventing duplicate + ## logging. It can also be used from other scripts to + ## inspect if a certificate has been seen in use. The string value ## in the set is for storing the DER formatted certificate' SHA1 hash. ## ## In cluster operation, this set is uniformly distributed across ## proxy nodes. global certs: set[addr, string] &create_expire=1day &redef; - + ## Event that can be handled to access the loggable record as it is sent ## on to the logging framework. global log_known_certs: event(rec: CertsInfo); diff --git a/scripts/policy/tuning/__load__.zeek b/scripts/policy/tuning/__load__.zeek index 03449882f8..db9fe9a572 100644 --- a/scripts/policy/tuning/__load__.zeek +++ b/scripts/policy/tuning/__load__.zeek @@ -1,2 +1,2 @@ -##! This loads the default tuning +##! This loads the default tuning @load ./defaults \ No newline at end of file diff --git a/scripts/policy/tuning/defaults/packet-fragments.zeek b/scripts/policy/tuning/defaults/packet-fragments.zeek index f95c826547..7ae0e4363c 100644 --- a/scripts/policy/tuning/defaults/packet-fragments.zeek +++ b/scripts/policy/tuning/defaults/packet-fragments.zeek @@ -1,7 +1,7 @@ # Capture TCP fragments, but not UDP (or ICMP), since those are a lot more # common due to high-volume, fragmenting protocols such as NFS :-(. -# This normally isn't used because of the default open packet filter +# This normally isn't used because of the default open packet filter # but we set it anyway in case the user is using a packet filter. # Note: This was removed because the default model now is to have a wide # open packet filter. diff --git a/scripts/policy/tuning/defaults/warnings.zeek b/scripts/policy/tuning/defaults/warnings.zeek index 6c31e82d4e..0220fc78de 100644 --- a/scripts/policy/tuning/defaults/warnings.zeek +++ b/scripts/policy/tuning/defaults/warnings.zeek @@ -1,5 +1,5 @@ ##! This file is meant to print messages on stdout for settings that would be -##! good to set in most cases or other things that could be done to achieve +##! good to set in most cases or other things that could be done to achieve ##! better detection. @load base/utils/site From 34694e67ea86b3cf053873f84276e0248130a1d3 Mon Sep 17 00:00:00 2001 From: zeek-bot Date: Thu, 21 Oct 2021 00:34:50 +0000 Subject: [PATCH 016/210] Update doc submodule [nomail] [skip ci] --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index e2f2af92c5..9aaee4e6a3 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit e2f2af92c55df3a27c31d8af79cb7a244af912b6 +Subproject commit 9aaee4e6a30826ed8e298e0b2bcd663b1f76f569 From e63ea95664cbd6988c14e9cee07ef29465b01e30 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 21 Oct 2021 16:38:57 -0700 Subject: [PATCH 017/210] Apply some missing clang-format changes --- src/Expr.cc | 3 +-- src/analyzer/protocol/dns/DNS.cc | 20 +++++++++++--------- src/analyzer/protocol/dns/DNS.h | 9 ++++++--- src/net_util.h | 5 +++++ 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/Expr.cc b/src/Expr.cc index 0c9c866b6c..41ae5bca3c 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -4328,8 +4328,7 @@ InExpr::InExpr(ExprPtr arg_op1, ExprPtr arg_op2) return; } - if ( op2->GetType()->Tag() == TYPE_TABLE && - op2->GetType()->AsTableType()->IsSubNetIndex() ) + if ( op2->GetType()->Tag() == TYPE_TABLE && op2->GetType()->AsTableType()->IsSubNetIndex() ) { SetType(base_type(TYPE_BOOL)); return; diff --git a/src/analyzer/protocol/dns/DNS.cc b/src/analyzer/protocol/dns/DNS.cc index e7f0d9be09..b87a2a8598 100644 --- a/src/analyzer/protocol/dns/DNS.cc +++ b/src/analyzer/protocol/dns/DNS.cc @@ -1696,7 +1696,7 @@ bool DNS_Interpreter::ParseRR_CAA(detail::DNS_MsgInfo* msg, const u_char*& data, } bool DNS_Interpreter::ParseRR_SVCB(detail::DNS_MsgInfo* msg, const u_char*& data, int& len, - int rdlength, const u_char* msg_start, const RR_Type& svcb_type) + int rdlength, const u_char* msg_start, const RR_Type& svcb_type) { const u_char* data_start = data; // the smallest SVCB/HTTPS rr is 3 bytes: @@ -1721,12 +1721,13 @@ bool DNS_Interpreter::ParseRR_SVCB(detail::DNS_MsgInfo* msg, const u_char*& data { target_name[0] = '.'; target_name[1] = '\0'; - name_end = target_name+1; + name_end = target_name + 1; } SVCB_DATA svcb_data = { .svc_priority = svc_priority, - .target_name = make_intrusive(new String(target_name, name_end - target_name, true)), + .target_name = make_intrusive( + new String(target_name, name_end - target_name, true)), }; // TODO: parse svcparams @@ -1735,21 +1736,22 @@ bool DNS_Interpreter::ParseRR_SVCB(detail::DNS_MsgInfo* msg, const u_char*& data std::ptrdiff_t parsed_bytes = data - data_start; if ( parsed_bytes < rdlength ) { - len -= ( rdlength - parsed_bytes ); - data += ( rdlength - parsed_bytes ); + len -= (rdlength - parsed_bytes); + data += (rdlength - parsed_bytes); } - switch( svcb_type ) + switch ( svcb_type ) { case detail::TYPE_SVCB: analyzer->EnqueueConnEvent(dns_SVCB, analyzer->ConnVal(), msg->BuildHdrVal(), - msg->BuildAnswerVal(), msg->BuildSVCB_Val(svcb_data)); + msg->BuildAnswerVal(), msg->BuildSVCB_Val(svcb_data)); break; case detail::TYPE_HTTPS: analyzer->EnqueueConnEvent(dns_HTTPS, analyzer->ConnVal(), msg->BuildHdrVal(), - msg->BuildAnswerVal(), msg->BuildSVCB_Val(svcb_data)); + msg->BuildAnswerVal(), msg->BuildSVCB_Val(svcb_data)); break; - default: break; // unreachable. for suppressing compiler warnings. + default: + break; // unreachable. for suppressing compiler warnings. } return true; } diff --git a/src/analyzer/protocol/dns/DNS.h b/src/analyzer/protocol/dns/DNS.h index f6f9a75f9a..edfeee031f 100644 --- a/src/analyzer/protocol/dns/DNS.h +++ b/src/analyzer/protocol/dns/DNS.h @@ -71,7 +71,9 @@ enum RR_Type TYPE_DS = 43, ///< Delegation signer (RFC 4034) TYPE_NSEC3 = 50, TYPE_NSEC3PARAM = 51, ///< Contains the NSEC3 parameters (RFC 5155) - TYPE_SVCB = 64, ///< SerViCe Binding (RFC draft: https://datatracker.ietf.org/doc/html/draft-ietf-dnsop-svcb-https-07#section-1.1) + TYPE_SVCB = + 64, ///< SerViCe Binding (RFC draft: + ///< https://datatracker.ietf.org/doc/html/draft-ietf-dnsop-svcb-https-07#section-1.1) TYPE_HTTPS = 65, ///< HTTPS record (HTTPS specific SVCB resource record) // Obsoleted TYPE_SPF = 99, ///< Alternative: storing SPF data in TXT records, using the same format (RFC @@ -150,7 +152,8 @@ enum DNSSEC_Digest SHA384 = 4, }; -///< all keys are defined in RFC draft https://datatracker.ietf.org/doc/html/draft-ietf-dnsop-svcb-https-07#section-14.3.2 +///< all keys are defined in RFC draft +///< https://datatracker.ietf.org/doc/html/draft-ietf-dnsop-svcb-https-07#section-14.3.2 enum SVCPARAM_Key { mandatory = 0, @@ -415,7 +418,7 @@ protected: bool ParseRR_LOC(detail::DNS_MsgInfo* msg, const u_char*& data, int& len, int rdlength, const u_char* msg_start); bool ParseRR_SVCB(detail::DNS_MsgInfo* msg, const u_char*& data, int& len, int rdlength, - const u_char* msg_start, const RR_Type& svcb_type); + const u_char* msg_start, const RR_Type& svcb_type); void SendReplyOrRejectEvent(detail::DNS_MsgInfo* msg, EventHandlerPtr event, const u_char*& data, int& len, String* question_name, String* original_name); diff --git a/src/net_util.h b/src/net_util.h index 3a6f697388..90a22f0d50 100644 --- a/src/net_util.h +++ b/src/net_util.h @@ -21,12 +21,17 @@ enum IPFamily IPv6 }; +// Force these files to stay in this order. Normally, clang-format +// wants to move sys/types.h to the end of this block, but that +// breaks FreeBSD builds. +// clang-format off #include #include #include #include #include #include +// clang-format on #ifdef HAVE_LINUX #define __FAVOR_BSD #endif From e14b695497f858efc3c10d532e8e166838e10c80 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Thu, 28 Oct 2021 16:55:02 +0200 Subject: [PATCH 018/210] Accept packets that use tcp segment offloading. When checksum offloading is enabled, we now forward packets that have 0 header lengths set - and assume that they have TSO enabled. If checksum offloading is not enabled, we drop the packets. Addresses GH-1829 --- src/IP.h | 6 +++++- src/packet_analysis/protocol/ip/IP.cc | 11 ++++++++--- src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc | 4 +++- src/packet_analysis/protocol/tcp/TCP.cc | 4 ++++ .../core.ip-broken-header/ip-bogus-header-weird.log | 1 - 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/IP.h b/src/IP.h index 3c2fbef813..2b670cf72e 100644 --- a/src/IP.h +++ b/src/IP.h @@ -415,7 +415,11 @@ public: uint16_t PayloadLen() const { if ( ip4 ) - return ntohs(ip4->ip_len) - ip4->ip_hl * 4; + { + // prevent overflow in case of segment offloading/zeroed header length. + auto total_len = ntohs(ip4->ip_len); + return total_len ? total_len - ip4->ip_hl * 4 : 0; + } return ntohs(ip6->ip6_plen) + 40 - ip6_hdrs->TotalLength(); } diff --git a/src/packet_analysis/protocol/ip/IP.cc b/src/packet_analysis/protocol/ip/IP.cc index e7ea5e75da..cb45f02e7e 100644 --- a/src/packet_analysis/protocol/ip/IP.cc +++ b/src/packet_analysis/protocol/ip/IP.cc @@ -81,8 +81,13 @@ bool IPAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* packet) // TCP segmentation offloading can zero out the ip_len field. Weird("ip_hdr_len_zero", packet); - // Cope with the zero'd out ip_len field by using the caplen. - total_len = packet->cap_len - hdr_size; + if ( detail::ignore_checksums ) + // Cope with the zero'd out ip_len field by using the caplen. + total_len = packet->cap_len - hdr_size; + else + // If this is caused by segmentation offloading, the checksum will + // also be incorrect. If checksum validation is enabled - jus tbail here. + return false; } if ( packet->len < total_len + hdr_size ) @@ -236,7 +241,7 @@ bool IPAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* packet) packet->proto = proto; // Double check the lengths one more time before forwarding this on. - if ( packet->ip_hdr->TotalLen() < packet->ip_hdr->HdrLen() ) + if ( total_len < packet->ip_hdr->HdrLen() ) { Weird("bogus_IP_header_lengths", packet); return false; diff --git a/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc b/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc index 785d914498..e18e46b2c1 100644 --- a/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc +++ b/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc @@ -115,7 +115,9 @@ bool IPBasedAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* pkt bool IPBasedAnalyzer::CheckHeaderTrunc(size_t min_hdr_len, size_t remaining, Packet* packet) { - if ( packet->ip_hdr->PayloadLen() < min_hdr_len ) + // If segment offloading or similar is enabled, the payload len will return 0. + // Thus, let's ignore that case. + if ( packet->ip_hdr->PayloadLen() && packet->ip_hdr->PayloadLen() < min_hdr_len ) { Weird("truncated_header", packet); return false; diff --git a/src/packet_analysis/protocol/tcp/TCP.cc b/src/packet_analysis/protocol/tcp/TCP.cc index 5a45e28438..cdd8a4c80f 100644 --- a/src/packet_analysis/protocol/tcp/TCP.cc +++ b/src/packet_analysis/protocol/tcp/TCP.cc @@ -96,6 +96,10 @@ void TCPAnalyzer::DeliverPacket(Connection* c, double t, bool is_orig, int remai { const u_char* data = pkt->ip_hdr->Payload(); int len = pkt->ip_hdr->PayloadLen(); + // If the header length is zero, tcp checksum offloading is probably enabled + // In this case, let's fix up the length. + if ( pkt->ip_hdr->TotalLen() == 0 ) + len = remaining; auto* adapter = static_cast(c->GetSessionAdapter()); const struct tcphdr* tp = ExtractTCP_Header(data, len, remaining, adapter); diff --git a/testing/btest/Baseline/core.ip-broken-header/ip-bogus-header-weird.log b/testing/btest/Baseline/core.ip-broken-header/ip-bogus-header-weird.log index d5adac806f..179c4fd1a7 100644 --- a/testing/btest/Baseline/core.ip-broken-header/ip-bogus-header-weird.log +++ b/testing/btest/Baseline/core.ip-broken-header/ip-bogus-header-weird.log @@ -8,5 +8,4 @@ #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source #types time string addr port addr port string string bool string string XXXXXXXXXX.XXXXXX - 118.181.144.194 0 136.255.115.116 0 ip_hdr_len_zero - F zeek IP -XXXXXXXXXX.XXXXXX - 118.181.144.194 0 136.255.115.116 0 bogus_IP_header_lengths - F zeek IP #close XXXX-XX-XX-XX-XX-XX From f1c81e3ab9cd6a70edc3d51c596e782069ae152d Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Thu, 28 Oct 2021 13:18:46 -0700 Subject: [PATCH 019/210] Update btest submodule to 0.70 [nomail] [skip ci] --- auxil/btest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auxil/btest b/auxil/btest index 0a37819d48..882e5eb3dc 160000 --- a/auxil/btest +++ b/auxil/btest @@ -1 +1 @@ -Subproject commit 0a37819d484358999a47e76ac473da74799ab08d +Subproject commit 882e5eb3dcec92e7e65423717ff59628ee9684a0 From b6444dce0c9c830ee3be5398b62e9a68efe56902 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 27 Sep 2021 12:11:19 -0700 Subject: [PATCH 020/210] Fix issue with broken libpcaps that return repeat packets This is apparently a problem with the Myricom version of libpcap, where instead of returning a null or a zero if no packets are available, it returns the previous packet. This causes Zeek to improperly parse the packet and crash. We thought we had fixed this previously with a check for a null packet but that fix was not enough. --- src/iosource/pcap/Source.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/iosource/pcap/Source.cc b/src/iosource/pcap/Source.cc index 164bd06a2c..8afe55638a 100644 --- a/src/iosource/pcap/Source.cc +++ b/src/iosource/pcap/Source.cc @@ -247,6 +247,14 @@ bool PcapSource::ExtractNextPacket(Packet* pkt) ++stats.received; stats.bytes_received += header->len; + // Some versions of libpcap (myricom) are somewhat broken and will return a duplicate + // packet if there are no more packets available. Namely, it returns the exact same + // packet structure (including the header) out of the library without reinitializing + // any of the values. If we set the header lengths to zero here, we can keep from + // processing it a second time. + header->len = 0; + header->caplen = 0; + return true; } From c25d5cecb260920af5a0403fdd73985b070dfef9 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Mon, 1 Nov 2021 12:10:39 -0700 Subject: [PATCH 021/210] Update btest submodule to 0.71 [nomail] [skip ci] --- auxil/btest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auxil/btest b/auxil/btest index 882e5eb3dc..5f954ec65c 160000 --- a/auxil/btest +++ b/auxil/btest @@ -1 +1 @@ -Subproject commit 882e5eb3dcec92e7e65423717ff59628ee9684a0 +Subproject commit 5f954ec65cb78b17f7156455c8c3c905a816ae96 From c2372ca9d85163952b534578f6826375bf0479f6 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 2 Nov 2021 16:05:59 -0700 Subject: [PATCH 022/210] GH-1781: Add .git-blame-ignore-revs file --- .git-blame-ignore-revs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000000..4a41859c7b --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,26 @@ +# Reformat the world (initial clang-format formatting) +b2f171ec69eae3a833a9db1b16e5234bd3eaf0b6 + +# clang-format: Force zeek-config.h to be earlier in the config ordering +9cb54f5d449b63006cc9a1f451a47732c92fef2d + +# clang-format: A few minor comment-spacing fixes +07e276ab2e351ce71b709139f1933b9ead40d094 + +# clang-format: Enforce ordering of includes in ZBody +cb99ae2b7c9988656b097ad2789dffd2c0c37939 + +# clang-format: Other include ordering changes +e97c14add5b04aedc7f3f9dba59f665cbad793af + +# clang-format: Other minor formatting changes +02206f3215f977ba7752476ba89ca06abe93375c + +# clang-format: Set IndentCaseBlocks to false +4423574d265749da8e707ab0fbcffcbfaed26614 + +# clang-format: Set penalty for breaking after assignment operator +9af6b2f48d11b4e287d0f18034a486f76f9f2d61 + +# Remove trailing whitespace from script files +a6378531dbc5c357926d98fe785bb719cc70e1b4 From ceaec0902468c4dcf253c1e84c7237c09ad74aae Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 2 Nov 2021 12:59:54 -0700 Subject: [PATCH 023/210] GH-693: use pcap_dump_open_append where supported --- cmake | 2 +- src/iosource/pcap/Dumper.cc | 6 +++++- testing/btest/bifs/dump_current_packet.zeek | 4 ++++ zeek-config.h.in | 3 +++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cmake b/cmake index 4d1990f0e4..ed6c3e3414 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 4d1990f0e4c273cf51ec52278add6ff256f9c889 +Subproject commit ed6c3e341408c2469093c40969354c4fb1ebce99 diff --git a/src/iosource/pcap/Dumper.cc b/src/iosource/pcap/Dumper.cc index 25aff8269e..3b46c5c15f 100644 --- a/src/iosource/pcap/Dumper.cc +++ b/src/iosource/pcap/Dumper.cc @@ -68,11 +68,15 @@ void PcapDumper::Open() else { +#ifdef HAVE_PCAP_DUMP_OPEN_APPEND + dumper = pcap_dump_open_append(pd, props.path.c_str()); +#else // Old file and we need to append, which, unfortunately, // is not supported by libpcap. So, we have to hack a - // little bit, knowing that pcap_dumpter_t is, in fact, + // little bit, knowing that pcap_dumper_t is, in fact, // a FILE ... :-( dumper = (pcap_dumper_t*)fopen(props.path.c_str(), "a"); +#endif if ( ! dumper ) { Error(util::fmt("can't open dump %s: %s", props.path.c_str(), strerror(errno))); diff --git a/testing/btest/bifs/dump_current_packet.zeek b/testing/btest/bifs/dump_current_packet.zeek index ce177a1daf..712ffb0e4f 100644 --- a/testing/btest/bifs/dump_current_packet.zeek +++ b/testing/btest/bifs/dump_current_packet.zeek @@ -5,6 +5,10 @@ # @TEST-EXEC: btest-diff 1.hex # @TEST-EXEC: btest-diff 2.hex +# Run the same test a second time, which will try to write to an +# existing file and shouldn't crash a sanitizer build. +# @TEST-EXEC: zeek -b -r $TRACES/wikipedia.trace %INPUT + # Note that the hex output will contain global pcap header information, # including Zeek's snaplen setting (so maybe check that out in the case # you are reading this message due to this test failing in the future). diff --git a/zeek-config.h.in b/zeek-config.h.in index 3c90bd5ceb..ff5edc8c8b 100644 --- a/zeek-config.h.in +++ b/zeek-config.h.in @@ -49,6 +49,9 @@ /* Define if you have the header file. */ #cmakedefine HAVE_PCAP_INT_H +/* Define if libpcap supports pcap_dump_open_append(). */ +#cmakedefine HAVE_PCAP_DUMP_OPEN_APPEND + /* line editing & history powers */ #cmakedefine HAVE_READLINE From 4c7aac079f9d59c354e3c7c728c1eaf754afb13c Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Tue, 2 Nov 2021 21:44:06 -0700 Subject: [PATCH 024/210] Update doc submodule [nomail] [skip ci] --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index 9aaee4e6a3..931cc50ce8 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 9aaee4e6a30826ed8e298e0b2bcd663b1f76f569 +Subproject commit 931cc50ce8fa21dbe8b2816136069b2d18716f06 From 75e584cb73561842f13c56ab3acaa18aff4cffb9 Mon Sep 17 00:00:00 2001 From: Yacin Nadji Date: Wed, 3 Nov 2021 14:21:02 -0400 Subject: [PATCH 025/210] Change set intersection test to be correct --- testing/btest/Baseline/language.set/out | 2 +- testing/btest/language/set.zeek | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/btest/Baseline/language.set/out b/testing/btest/Baseline/language.set/out index 57bb3c068f..119293ede6 100644 --- a/testing/btest/Baseline/language.set/out +++ b/testing/btest/Baseline/language.set/out @@ -44,7 +44,7 @@ remove element (PASS) remove element (PASS) !in operator (PASS) union (PASS) -intersection (FAIL) +intersection (PASS) difference (PASS) difference (PASS) union/inter. (PASS) diff --git a/testing/btest/language/set.zeek b/testing/btest/language/set.zeek index 627474b2d3..60ef7b796a 100644 --- a/testing/btest/language/set.zeek +++ b/testing/btest/language/set.zeek @@ -150,7 +150,7 @@ function basic_functionality() local a_and_b = a & b; test_case( "union", a_or_b == a_plus_b ); - test_case( "intersection", a_and_b == a_plus_b ); + test_case( "intersection", a_and_b == a_also_b ); test_case( "difference", a - b == a_sans_b ); test_case( "difference", b - a == b_sans_a ); From aacb812b9ad08c50e3a6562aabfdb331f7cc5b6b Mon Sep 17 00:00:00 2001 From: Yacin Nadji Date: Wed, 3 Nov 2021 14:26:31 -0400 Subject: [PATCH 026/210] Fix C++ Intersection code --- src/Val.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Val.cc b/src/Val.cc index 305a638d83..b4bf2d2be0 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -1716,7 +1716,7 @@ TableValPtr TableVal::Intersection(const TableVal& tv) const } const PDict* tbl = AsTable(); - for ( const auto& tble : *tbl ) + for ( const auto& tble : *t1 ) { auto k = tble.GetHashKey(); From d426e285ebfa0b5e0830c6512b31737e2dc9f039 Mon Sep 17 00:00:00 2001 From: Yacin Nadji Date: Wed, 3 Nov 2021 14:52:45 -0400 Subject: [PATCH 027/210] Add reverse order intersection check --- testing/btest/Baseline/language.set/out | 1 + testing/btest/language/set.zeek | 2 ++ 2 files changed, 3 insertions(+) diff --git a/testing/btest/Baseline/language.set/out b/testing/btest/Baseline/language.set/out index 119293ede6..51d2a72711 100644 --- a/testing/btest/Baseline/language.set/out +++ b/testing/btest/Baseline/language.set/out @@ -45,6 +45,7 @@ remove element (PASS) !in operator (PASS) union (PASS) intersection (PASS) +intersection (PASS) difference (PASS) difference (PASS) union/inter. (PASS) diff --git a/testing/btest/language/set.zeek b/testing/btest/language/set.zeek index 60ef7b796a..4b81ecd0c8 100644 --- a/testing/btest/language/set.zeek +++ b/testing/btest/language/set.zeek @@ -148,9 +148,11 @@ function basic_functionality() local a_or_b = a | b; local a_and_b = a & b; + local b_and_a = b & a; test_case( "union", a_or_b == a_plus_b ); test_case( "intersection", a_and_b == a_also_b ); + test_case( "intersection", b_and_a == a_also_b ); test_case( "difference", a - b == a_sans_b ); test_case( "difference", b - a == b_sans_a ); From 96c45a6c93737b44d73541fee8e6a872b8be6fb4 Mon Sep 17 00:00:00 2001 From: Yacin Nadji Date: Wed, 3 Nov 2021 15:44:55 -0400 Subject: [PATCH 028/210] Remove unused tbl --- src/Val.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Val.cc b/src/Val.cc index b4bf2d2be0..09beec75c5 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -1715,7 +1715,6 @@ TableValPtr TableVal::Intersection(const TableVal& tv) const t0 = tmp; } - const PDict* tbl = AsTable(); for ( const auto& tble : *t1 ) { auto k = tble.GetHashKey(); From cdb52e64602b3905d18dc5b688bcc160fac76cfa Mon Sep 17 00:00:00 2001 From: Yacin Nadji Date: Wed, 3 Nov 2021 15:46:17 -0400 Subject: [PATCH 029/210] Curse you tabs! --- testing/btest/language/set.zeek | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/btest/language/set.zeek b/testing/btest/language/set.zeek index 4b81ecd0c8..b396a0c572 100644 --- a/testing/btest/language/set.zeek +++ b/testing/btest/language/set.zeek @@ -148,11 +148,11 @@ function basic_functionality() local a_or_b = a | b; local a_and_b = a & b; - local b_and_a = b & a; + local b_and_a = b & a; test_case( "union", a_or_b == a_plus_b ); test_case( "intersection", a_and_b == a_also_b ); - test_case( "intersection", b_and_a == a_also_b ); + test_case( "intersection", b_and_a == a_also_b ); test_case( "difference", a - b == a_sans_b ); test_case( "difference", b - a == b_sans_a ); From f7d18eeb9ec6ce0cfae9574940f4c0a8ba8f9eee Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Wed, 3 Nov 2021 13:40:22 -0700 Subject: [PATCH 030/210] Add Fedora 35 to CI --- .cirrus.yml | 7 +++++++ ci/fedora-35/Dockerfile | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 ci/fedora-35/Dockerfile diff --git a/.cirrus.yml b/.cirrus.yml index 6157b8fb5d..04420c306d 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -93,6 +93,13 @@ env: # Linux EOL timelines: https://linuxlifecycle.com/ # Fedora (~13 months): https://fedoraproject.org/wiki/Fedora_Release_Life_Cycle +fedora35_task: + container: + # Fedora 35 EOL: Around Dec 2022 + dockerfile: ci/fedora-35/Dockerfile + << : *RESOURCES_TEMPLATE + << : *CI_TEMPLATE + fedora34_task: container: # Fedora 34 EOL: Around May 2022 diff --git a/ci/fedora-35/Dockerfile b/ci/fedora-35/Dockerfile new file mode 100644 index 0000000000..a168642ff3 --- /dev/null +++ b/ci/fedora-35/Dockerfile @@ -0,0 +1,23 @@ +FROM fedora:35 + +RUN dnf -y install \ + bison \ + cmake \ + diffutils \ + findutils \ + flex \ + git \ + gcc \ + gcc-c++ \ + libpcap-devel \ + make \ + openssl-devel \ + python3-devel \ + python3-pip\ + sqlite \ + swig \ + which \ + zlib-devel \ + && dnf clean all && rm -rf /var/cache/dnf + +RUN pip3 install junit2html From bac833a2f9facd0034214b5ab1b30e3ef469159d Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Wed, 3 Nov 2021 13:52:40 -0700 Subject: [PATCH 031/210] Add macOS Monterey and drop Catalina in CI --- .cirrus.yml | 8 ++++---- ci/build.sh | 5 +++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 04420c306d..3b0b6bc8d9 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -219,16 +219,16 @@ alpine_task: # Apple doesn't publish official long-term support timelines. # We aim to support both the current and previous macOS release. -macos_big_sur_task: +macos_monterey_task: macos_instance: - image: big-sur-xcode-12.5 + image: monterey-xcode-13.1 prepare_script: ./ci/macos/prepare.sh << : *CI_TEMPLATE << : *MACOS_RESOURCES_TEMPLATE -macos_catalina_task: +macos_big_sur_task: macos_instance: - image: catalina-xcode + image: big-sur-xcode-12.5 prepare_script: ./ci/macos/prepare.sh << : *CI_TEMPLATE << : *MACOS_RESOURCES_TEMPLATE diff --git a/ci/build.sh b/ci/build.sh index 20ca3237cc..051d119f62 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -10,6 +10,11 @@ set -x # some problems with Catalina specifically, but it doesn't break anything on Big Sur either. if [[ "${CIRRUS_OS}" == "darwin" ]]; then export ZEEK_CI_CONFIGURE_FLAGS="${ZEEK_CI_CONFIGURE_FLAGS} --osx-sysroot=$(xcrun --show-sdk-path)" + + # Starting with Monterey & Xcode 13.1 we need to help it find OpenSSL + if [ -d /usr/local/opt/openssl@1.1/lib/pkgconfig ]; then + export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/opt/openssl@1.1/lib/pkgconfig + fi fi if [[ "${ZEEK_CI_CREATE_ARTIFACT}" != "1" ]]; then From c1ebb279cad11d25f0d0255c73b697d1c44c0a01 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Wed, 3 Nov 2021 15:22:20 -0700 Subject: [PATCH 032/210] Only push CI's Docker images when we're on the main repo --- .github/workflows/docker.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8253d3d4e8..459e81e417 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -27,6 +27,7 @@ jobs: # Create and boot a loader. This will e.g., provide caching # so we avoid rebuilds of the same image after this step. - uses: docker/setup-buildx-action@v1 + - name: Build uses: docker/build-push-action@v2 with: @@ -43,6 +44,7 @@ jobs: - name: Get version id: version run: echo "::set-output name=RELEASE_VERSION::$(cat VERSION)" + - name: Compute target tag id: target env: @@ -65,15 +67,16 @@ jobs: - name: Login to DockerHub uses: docker/login-action@v1 - # Secrets for the login are not available for pull requests. - if: github.event_name == 'push' + # Don't publish on forks. Also note that secrets for the login are not + # available for pull requests, so trigger on pushes only. + if: github.repository == 'zeek/zeek' && github.event_name == 'push' with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Push # Only publish if we did compute a tag. - if: github.event_name == 'push' && steps.target.outputs.tag != '' + if: github.repository == 'zeek/zeek' && github.event_name == 'push' && steps.target.outputs.tag != '' uses: docker/build-push-action@v2 with: context: ./ From 1efaf8d7a4321ba06e967142d71dd398ccc7e01a Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 7 Oct 2021 09:59:15 +0200 Subject: [PATCH 033/210] Move logic to execute `HookLoadFile` for signatures into rule matcher code. This (1) fixes an issue where signature files supplied on the command line wouldn't pass through the hooks, and (2) prepares for allowing hooks to supply the content of a signature file directly. --- src/RuleMatcher.cc | 47 +++++++++++++++++-- src/RuleMatcher.h | 3 +- src/ScannedFile.cc | 12 ++++- src/ScannedFile.h | 13 ++++- src/scan.l | 28 +---------- src/zeek-setup.cc | 10 ++-- testing/btest/Baseline/plugins.hooks/output | 6 +++ .../Baseline/signatures.udp-state/reject | 2 +- testing/btest/plugins/hooks.zeek | 13 ++++- 9 files changed, 94 insertions(+), 40 deletions(-) diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc index 4cf0c15214..cd2455c074 100644 --- a/src/RuleMatcher.cc +++ b/src/RuleMatcher.cc @@ -24,6 +24,7 @@ #include "zeek/ZeekString.h" #include "zeek/analyzer/Analyzer.h" #include "zeek/module_util.h" +#include "zeek/plugin/Manager.h" using namespace std; @@ -248,7 +249,7 @@ void RuleMatcher::Delete(RuleHdrTest* node) delete node; } -bool RuleMatcher::ReadFiles(const std::vector& files) +bool RuleMatcher::ReadFiles(const std::vector& files) { #ifdef USE_PERFTOOLS_DEBUG HeapLeakChecker::Disabler disabler; @@ -256,18 +257,54 @@ bool RuleMatcher::ReadFiles(const std::vector& files) parse_error = false; - for ( const auto& f : files ) + for ( auto f : files ) { - rules_in = util::open_file(util::find_file(f, util::zeek_path(), ".sig")); + if ( ! f.full_path ) + f.full_path = util::find_file(f.file, util::zeek_path(), ".sig"); + + int rc = PLUGIN_HOOK_WITH_RESULT( + HOOK_LOAD_FILE, HookLoadFile(zeek::plugin::Plugin::SIGNATURES, f.file, *f.full_path), + -1); + + switch ( rc ) + { + case -1: + // No plugin in charge of this file. + if ( f.full_path->empty() ) + { + zeek::reporter->Error("failed to find file associated with @load-sigs %s", + f.file.c_str()); + continue; + } + break; + + case 0: + if ( ! zeek::reporter->Errors() ) + zeek::reporter->Error("Plugin reported error loading signatures %s", + f.file.c_str()); + + exit(1); + break; + + case 1: + // A plugin took care of it, just skip. + continue; + + default: + assert(false); + break; + } + + rules_in = util::open_file(*f.full_path); if ( ! rules_in ) { - reporter->Error("Can't open signature file %s", f.data()); + reporter->Error("Can't open signature file %s", f.file.c_str()); return false; } rules_line_number = 0; - current_rule_file = f.data(); + current_rule_file = f.full_path->c_str(); rules_parse(); fclose(rules_in); } diff --git a/src/RuleMatcher.h b/src/RuleMatcher.h index 9a89d39efd..f64a7bc97e 100644 --- a/src/RuleMatcher.h +++ b/src/RuleMatcher.h @@ -11,6 +11,7 @@ #include "zeek/CCL.h" #include "zeek/RE.h" #include "zeek/Rule.h" +#include "zeek/ScannedFile.h" //#define MATCHER_PRINT_STATS @@ -259,7 +260,7 @@ public: ~RuleMatcher(); // Parse the given files and built up data structures. - bool ReadFiles(const std::vector& files); + bool ReadFiles(const std::vector& files); /** * Inititialize a state object for matching file magic signatures. diff --git a/src/ScannedFile.cc b/src/ScannedFile.cc index c5367748e4..8050f0fc12 100644 --- a/src/ScannedFile.cc +++ b/src/ScannedFile.cc @@ -10,7 +10,7 @@ namespace zeek::detail { std::list files_scanned; -std::vector sig_files; +std::vector sig_files; ScannedFile::ScannedFile(int arg_include_level, std::string arg_name, bool arg_skipped, bool arg_prefixes_checked) @@ -47,4 +47,14 @@ bool ScannedFile::AlreadyScanned() const return rval; } +SignatureFile::SignatureFile(std::string file) + : file(std::move(file)) + { + } + +SignatureFile::SignatureFile(std::string file, std::string full_path) + : file(std::move(file)), full_path(std::move(full_path)) + { + } + } // namespace zeek::detail diff --git a/src/ScannedFile.h b/src/ScannedFile.h index 902d627f7d..9829b2c1d2 100644 --- a/src/ScannedFile.h +++ b/src/ScannedFile.h @@ -3,6 +3,7 @@ #pragma once #include +#include #include #include @@ -34,6 +35,16 @@ public: }; extern std::list files_scanned; -extern std::vector sig_files; + +struct SignatureFile + { + std::string file; + std::optional full_path; + + SignatureFile(std::string file); + SignatureFile(std::string file, std::string full_path); + }; + +extern std::vector sig_files; } // namespace zeek::detail diff --git a/src/scan.l b/src/scan.l index e6af14cd89..dbcb3fcfd3 100644 --- a/src/scan.l +++ b/src/scan.l @@ -348,33 +348,7 @@ when return TOK_WHEN; @load-sigs{WS}{FILE} { const char* file = zeek::util::skip_whitespace(yytext + 10); std::string path = find_relative_file(file, ".sig"); - int rc = PLUGIN_HOOK_WITH_RESULT(HOOK_LOAD_FILE, HookLoadFile(zeek::plugin::Plugin::SIGNATURES, file, path), -1); - - switch ( rc ) { - case -1: - // No plugin in charge of this file. - if ( path.empty() ) - zeek::reporter->Error("failed to find file associated with @load-sigs %s", - file); - else - zeek::detail::sig_files.push_back(std::move(path)); - break; - - case 0: - if ( ! zeek::reporter->Errors() ) - zeek::reporter->Error("Plugin reported error loading signatures %s", file); - - exit(1); - break; - - case 1: - // A plugin took care of it, just skip. - break; - - default: - assert(false); - break; - } + sig_files.emplace_back(file, path); } @load-plugin{WS}{ID} { diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index dad543c282..b2e77df628 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -737,15 +737,19 @@ SetupResult setup(int argc, char** argv, Options* zopts) id->SetVal(make_intrusive(*options.pcap_filter)); } - auto all_signature_files = options.signature_files; + std::vector all_signature_files; + + // Append signature files given on the command line + for ( const auto& sf : options.signature_files ) + all_signature_files.push_back(sf); // Append signature files defined in "signature_files" script option for ( auto&& sf : get_script_signature_files() ) - all_signature_files.emplace_back(std::move(sf)); + all_signature_files.push_back(std::move(sf)); // Append signature files defined in @load-sigs for ( const auto& sf : zeek::detail::sig_files ) - all_signature_files.emplace_back(sf); + all_signature_files.push_back(sf); if ( ! all_signature_files.empty() ) { diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index e16e7c63b6..e340ed914b 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -1034,6 +1034,7 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/zeek.bif, <...>/zeek.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, builtin-plugins/__load__.zeek, <...>/__load__.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, builtin-plugins/__preload__.zeek, <...>/__preload__.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, s1.sig, ./s1.sig) -> -1 0.000000 MetaHookPost LoadFile(1, ./archive, <...>/archive.sig) -> -1 0.000000 MetaHookPost LoadFile(1, ./audio, <...>/audio.sig) -> -1 0.000000 MetaHookPost LoadFile(1, ./dpd.sig, <...>/dpd.sig) -> -1 @@ -1046,6 +1047,7 @@ 0.000000 MetaHookPost LoadFile(1, ./office, <...>/office.sig) -> -1 0.000000 MetaHookPost LoadFile(1, ./programming, <...>/programming.sig) -> -1 0.000000 MetaHookPost LoadFile(1, ./video, <...>/video.sig) -> -1 +0.000000 MetaHookPost LoadFile(1, s2, ./s2.sig) -> -1 0.000000 MetaHookPost LogInit(Log::WRITER_ASCII, default, true, true, packet_filter(0.0,0.0,0.0), 5, {ts (time), node (string), filter (string), init (bool), success (bool)}) -> 0.000000 MetaHookPost LogWrite(Log::WRITER_ASCII, default, packet_filter(0.0,0.0,0.0), 5, {ts (time), node (string), filter (string), init (bool), success (bool)}, ) -> true 0.000000 MetaHookPost QueueEvent(NetControl::init()) -> false @@ -2086,6 +2088,7 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/zeek.bif, <...>/zeek.bif.zeek) 0.000000 MetaHookPre LoadFile(0, builtin-plugins/__load__.zeek, <...>/__load__.zeek) 0.000000 MetaHookPre LoadFile(0, builtin-plugins/__preload__.zeek, <...>/__preload__.zeek) +0.000000 MetaHookPre LoadFile(0, s1.sig, ./s1.sig) 0.000000 MetaHookPre LoadFile(1, ./archive, <...>/archive.sig) 0.000000 MetaHookPre LoadFile(1, ./audio, <...>/audio.sig) 0.000000 MetaHookPre LoadFile(1, ./dpd.sig, <...>/dpd.sig) @@ -2098,6 +2101,7 @@ 0.000000 MetaHookPre LoadFile(1, ./office, <...>/office.sig) 0.000000 MetaHookPre LoadFile(1, ./programming, <...>/programming.sig) 0.000000 MetaHookPre LoadFile(1, ./video, <...>/video.sig) +0.000000 MetaHookPre LoadFile(1, s2, ./s2.sig) 0.000000 MetaHookPre LogInit(Log::WRITER_ASCII, default, true, true, packet_filter(0.0,0.0,0.0), 5, {ts (time), node (string), filter (string), init (bool), success (bool)}) 0.000000 MetaHookPre LogWrite(Log::WRITER_ASCII, default, packet_filter(0.0,0.0,0.0), 5, {ts (time), node (string), filter (string), init (bool), success (bool)}, ) 0.000000 MetaHookPre QueueEvent(NetControl::init()) @@ -3149,6 +3153,8 @@ 0.000000 | HookLoadFile base<...>/zeek.bif <...>/zeek.bif.zeek 0.000000 | HookLoadFile builtin-plugins/__load__.zeek <...>/__load__.zeek 0.000000 | HookLoadFile builtin-plugins/__preload__.zeek <...>/__preload__.zeek +0.000000 | HookLoadFile s1.sig ./s1.sig +0.000000 | HookLoadFile s2 ./s2.sig 0.000000 | HookLogInit packet_filter 1/1 {ts (time), node (string), filter (string), init (bool), success (bool)} 0.000000 | HookLogWrite packet_filter [ts=XXXXXXXXXX.XXXXXX, node=zeek, filter=ip or not ip, init=T, success=T] 0.000000 | HookQueueEvent NetControl::init() diff --git a/testing/btest/Baseline/signatures.udp-state/reject b/testing/btest/Baseline/signatures.udp-state/reject index aaeba4be58..9d97a9ac45 100644 --- a/testing/btest/Baseline/signatures.udp-state/reject +++ b/testing/btest/Baseline/signatures.udp-state/reject @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error: Error in signature (udp-established.sig:5): 'established' is not a valid 'udp-state' +error: Error in signature (./udp-established.sig:5): 'established' is not a valid 'udp-state' diff --git a/testing/btest/plugins/hooks.zeek b/testing/btest/plugins/hooks.zeek index cfff751eca..992d305f14 100644 --- a/testing/btest/plugins/hooks.zeek +++ b/testing/btest/plugins/hooks.zeek @@ -1,8 +1,19 @@ # @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Demo Hooks # @TEST-EXEC: cp -r %DIR/hooks-plugin/* . # @TEST-EXEC: ./configure --zeek-dist=${DIST} && make -# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::Hooks" ZEEK_PLUGIN_PATH=`pwd` zeek -b -r $TRACES/http/get.trace %INPUT 2>&1 | $SCRIPTS/diff-remove-abspath | sort | uniq >output +# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::Hooks" ZEEK_PLUGIN_PATH=`pwd` zeek -b -r $TRACES/http/get.trace %INPUT s1.sig 2>&1 | $SCRIPTS/diff-remove-abspath | sort | uniq >output # @TEST-EXEC: btest-diff output @unload base/misc/version @load base/init-default + +@load-sigs s2 + +@TEST-START-FILE s1.sig +# Just empty. +@TEST-END-FILE + +@TEST-START-FILE s2.sig +# Just empty. +@TEST-END-FILE + From 34eaf42b926c64424a6bd5060ce59e16c5baa37d Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 24 Sep 2021 12:50:27 +0200 Subject: [PATCH 034/210] Add new hook `HookLoadFileExtended` that allows plugins to supply Zeek script code to parse. The new hooks works similar to the existing `HookLoadFile` but, additionally, allows the plugin to return a string that contains the code to be used for the file being loaded. If the plugin does so, the content of any actual file on disk will be ignored (in fact, there doesn't even need to be a file on disk in that case). This works for both Zeek scripts and signatures. There's a new test that covers the new functionality, testing loading both scripts and signatures from memory. I also manually tested that the debugger integration works, but I don't see much of a way to add a regression test for that part. We keep the existing hook as well for backwards compatibility. We could decide to deprecate it, but not sure that buys us much, so left that out. Closes #1757. --- src/PolicyFile.cc | 65 +- src/PolicyFile.h | 6 +- src/RuleMatcher.cc | 52 +- src/RuleMatcher.h | 2 +- src/ScannedFile.cc | 5 +- src/plugin/Manager.cc | 35 + src/plugin/Manager.h | 26 + src/plugin/Plugin.cc | 23 + src/plugin/Plugin.h | 52 +- src/rule-scan.l | 20 + src/scan.l | 104 +- src/zeek-setup.cc | 6 +- .../missing-sig-file | 2 +- testing/btest/Baseline/plugins.hooks/output | 1098 +++++++++++++++++ .../plugins.plugin-load-file-extended/output | 7 + .../btest/plugins/hooks-plugin/src/Plugin.cc | 8 + .../btest/plugins/hooks-plugin/src/Plugin.h | 1 + .../plugins/plugin-load-file-extended.zeek | 17 + .../plugin-load-file-extended/.btest-ignore | 0 .../plugin-load-file-extended/src/Plugin.cc | 70 ++ .../plugin-load-file-extended/src/Plugin.h | 18 + 21 files changed, 1525 insertions(+), 92 deletions(-) create mode 100644 testing/btest/Baseline/plugins.plugin-load-file-extended/output create mode 100644 testing/btest/plugins/plugin-load-file-extended.zeek create mode 100644 testing/btest/plugins/plugin-load-file-extended/.btest-ignore create mode 100644 testing/btest/plugins/plugin-load-file-extended/src/Plugin.cc create mode 100644 testing/btest/plugins/plugin-load-file-extended/src/Plugin.h diff --git a/src/PolicyFile.cc b/src/PolicyFile.cc index 3d7592f52b..eaee5a9916 100644 --- a/src/PolicyFile.cc +++ b/src/PolicyFile.cc @@ -72,49 +72,60 @@ int how_many_lines_in(const char* policy_filename) return pf->lines.size(); } -bool LoadPolicyFileText(const char* policy_filename) +bool LoadPolicyFileText(const char* policy_filename, + const std::optional& preloaded_content) { if ( ! policy_filename ) return true; - FILE* f = fopen(policy_filename, "r"); - - if ( ! f ) - { - debug_msg("No such policy file: %s.\n", policy_filename); - return false; - } - - PolicyFile* pf = new PolicyFile; - if ( policy_files.find(policy_filename) != policy_files.end() ) debug_msg("Policy file %s already loaded\n", policy_filename); + PolicyFile* pf = new PolicyFile; policy_files.insert(PolicyFileMap::value_type(policy_filename, pf)); - struct stat st; - if ( fstat(fileno(f), &st) != 0 ) + if ( preloaded_content ) { - char buf[256]; - util::zeek_strerror_r(errno, buf, sizeof(buf)); - reporter->Error("fstat failed on %s: %s", policy_filename, buf); - fclose(f); - return false; + auto size = preloaded_content->size(); + pf->filedata = new char[size + 1]; + memcpy(pf->filedata, preloaded_content->data(), size); + pf->filedata[size] = '\0'; } + else + { + FILE* f = fopen(policy_filename, "r"); - pf->lmtime = st.st_mtime; - off_t size = st.st_size; + if ( ! f ) + { + debug_msg("No such policy file: %s.\n", policy_filename); + return false; + } - // ### This code is not necessarily Unicode safe! - // (probably fine with UTF-8) - pf->filedata = new char[size + 1]; - if ( fread(pf->filedata, size, 1, f) != 1 ) - reporter->InternalError("Failed to fread() file data"); - pf->filedata[size] = 0; - fclose(f); + struct stat st; + if ( fstat(fileno(f), &st) != 0 ) + { + char buf[256]; + util::zeek_strerror_r(errno, buf, sizeof(buf)); + reporter->Error("fstat failed on %s: %s", policy_filename, buf); + fclose(f); + return false; + } + + pf->lmtime = st.st_mtime; + off_t size = st.st_size; + + // ### This code is not necessarily Unicode safe! + // (probably fine with UTF-8) + pf->filedata = new char[size + 1]; + if ( fread(pf->filedata, size, 1, f) != 1 ) + reporter->InternalError("Failed to fread() file data"); + pf->filedata[size] = 0; + fclose(f); + } // Separate the string by newlines. pf->lines.push_back(pf->filedata); + for ( char* iter = pf->filedata; *iter; ++iter ) { if ( *iter == '\n' ) diff --git a/src/PolicyFile.h b/src/PolicyFile.h index bda675638e..d85883cb0b 100644 --- a/src/PolicyFile.h +++ b/src/PolicyFile.h @@ -14,12 +14,16 @@ // policy_filename arguments should be absolute or relative paths; // no expansion is done. +#include +#include + namespace zeek::detail { int how_many_lines_in(const char* policy_filename); -bool LoadPolicyFileText(const char* policy_filename); +bool LoadPolicyFileText(const char* policy_filename, + const std::optional& preloaded_content = {}); // start_line is 1-based (the intuitive way) bool PrintLines(const char* policy_filename, unsigned int start_line, unsigned int how_many_lines, diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc index cd2455c074..42c3c568b0 100644 --- a/src/RuleMatcher.cc +++ b/src/RuleMatcher.cc @@ -24,10 +24,14 @@ #include "zeek/ZeekString.h" #include "zeek/analyzer/Analyzer.h" #include "zeek/module_util.h" -#include "zeek/plugin/Manager.h" using namespace std; +// Functions exposed by rule-scan.l +extern void rules_set_input_from_buffer(const char* data, size_t size); +extern void rules_set_input_from_file(FILE* f); +extern void rules_parse_input(); + namespace zeek::detail { @@ -262,11 +266,18 @@ bool RuleMatcher::ReadFiles(const std::vector& files) if ( ! f.full_path ) f.full_path = util::find_file(f.file, util::zeek_path(), ".sig"); - int rc = PLUGIN_HOOK_WITH_RESULT( + std::pair> rc = {-1, std::nullopt}; + rc.first = PLUGIN_HOOK_WITH_RESULT( HOOK_LOAD_FILE, HookLoadFile(zeek::plugin::Plugin::SIGNATURES, f.file, *f.full_path), -1); - switch ( rc ) + if ( rc.first < 0 ) + rc = PLUGIN_HOOK_WITH_RESULT( + HOOK_LOAD_FILE_EXT, + HookLoadFileExtended(zeek::plugin::Plugin::SIGNATURES, f.file, *f.full_path), + std::make_pair(-1, std::nullopt)); + + switch ( rc.first ) { case -1: // No plugin in charge of this file. @@ -287,26 +298,45 @@ bool RuleMatcher::ReadFiles(const std::vector& files) break; case 1: - // A plugin took care of it, just skip. - continue; + if ( ! rc.second ) + // A plugin took care of it, just skip. + continue; + + break; default: assert(false); break; } - rules_in = util::open_file(*f.full_path); + FILE* rules_in = nullptr; - if ( ! rules_in ) + if ( rc.first == 1 ) { - reporter->Error("Can't open signature file %s", f.file.c_str()); - return false; + // Parse code provided by plugin. + assert(rc.second); + rules_set_input_from_buffer(rc.second->data(), rc.second->size()); + } + else + { + // Parse from file. + rules_in = util::open_file(*f.full_path); + + if ( ! rules_in ) + { + reporter->Error("Can't open signature file %s", f.file.c_str()); + return false; + } + + rules_set_input_from_file(rules_in); } rules_line_number = 0; current_rule_file = f.full_path->c_str(); - rules_parse(); - fclose(rules_in); + rules_parse_input(); + + if ( rules_in ) + fclose(rules_in); } if ( parse_error ) diff --git a/src/RuleMatcher.h b/src/RuleMatcher.h index f64a7bc97e..76599c117a 100644 --- a/src/RuleMatcher.h +++ b/src/RuleMatcher.h @@ -12,6 +12,7 @@ #include "zeek/RE.h" #include "zeek/Rule.h" #include "zeek/ScannedFile.h" +#include "zeek/plugin/Manager.h" //#define MATCHER_PRINT_STATS @@ -23,7 +24,6 @@ extern void rules_error(zeek::detail::Rule* id, const char* msg); extern int rules_lex(void); extern int rules_parse(void); extern "C" int rules_wrap(void); -extern FILE* rules_in; extern int rules_line_number; extern const char* current_rule_file; diff --git a/src/ScannedFile.cc b/src/ScannedFile.cc index 8050f0fc12..590511050e 100644 --- a/src/ScannedFile.cc +++ b/src/ScannedFile.cc @@ -47,10 +47,7 @@ bool ScannedFile::AlreadyScanned() const return rval; } -SignatureFile::SignatureFile(std::string file) - : file(std::move(file)) - { - } +SignatureFile::SignatureFile(std::string file) : file(std::move(file)) { } SignatureFile::SignatureFile(std::string file, std::string full_path) : file(std::move(file)), full_path(std::move(full_path)) diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index f8f9783894..8215c5c56c 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -692,6 +692,41 @@ int Manager::HookLoadFile(const Plugin::LoadType type, const string& file, const return rc; } +std::pair> +Manager::HookLoadFileExtended(const Plugin::LoadType type, const string& file, + const string& resolved) + { + HookArgumentList args; + + if ( HavePluginForHook(META_HOOK_PRE) ) + { + args.push_back(HookArgument(type)); + args.push_back(HookArgument(file)); + args.push_back(HookArgument(resolved)); + MetaHookPre(HOOK_LOAD_FILE_EXT, args); + } + + hook_list* l = hooks[HOOK_LOAD_FILE_EXT]; + + std::pair> rc = {-1, std::nullopt}; + + if ( l ) + for ( hook_list::iterator i = l->begin(); i != l->end(); ++i ) + { + Plugin* p = (*i).second; + + rc = p->HookLoadFileExtended(type, file, resolved); + + if ( rc.first >= 0 ) + break; + } + + if ( HavePluginForHook(META_HOOK_POST) ) + MetaHookPost(HOOK_LOAD_FILE_EXT, args, HookArgument(rc)); + + return rc; + } + std::pair Manager::HookCallFunction(const Func* func, zeek::detail::Frame* parent, Args* vecargs) const { diff --git a/src/plugin/Manager.h b/src/plugin/Manager.h index 92b28d7b62..4873a271ba 100644 --- a/src/plugin/Manager.h +++ b/src/plugin/Manager.h @@ -245,6 +245,32 @@ public: virtual int HookLoadFile(const Plugin::LoadType type, const std::string& file, const std::string& resolved); + /** + * Hook that gives plugins a chance to take over loading an input file, + * including replacing the file's content. This method must be called + * between InitPreScript() and InitPostScript() for each input file Bro is + * about to load, either given on the command line or via @load script + * directives. The hook can take over the file, in which case Bro must not + * further process it otherwise; or provide its content, in which case Bro + * must use that and ignore the original file. + * + * @return tuple where the first element is 1 if a plugin took over the + * file; 0 if a plugin took over the file but had trouble loading it; and + * -1 if no plugin was interested in the file at all. + * + * If the plugins takes over by returning 1, there are two cases: if the + * second tuple element remains unset, the plugin handled the loading + * completely internally; the caller must not process it any further. + * Alternatively, the plugin may optionally return the acutal content to + * use for the file as a string through the tuple's second element. If so, + * the caller must ignore the file on disk and use that provided content + * instead (including when there's actually no physical file in place on + * disk at all). + */ + virtual std::pair> + HookLoadFileExtended(const Plugin::LoadType type, const std::string& file, + const std::string& resolved); + /** * Hook that filters calls to a script function/event/hook. * diff --git a/src/plugin/Plugin.cc b/src/plugin/Plugin.cc index c569b1afeb..315a844a3b 100644 --- a/src/plugin/Plugin.cc +++ b/src/plugin/Plugin.cc @@ -22,6 +22,7 @@ const char* hook_name(HookType h) static constexpr const char* hook_names[int(NUM_HOOKS) + 1] = { // Order must match that of HookType. "LoadFile", + "LoadFileExtended", "CallFunction", "QueueEvent", "DrainEvents", @@ -230,6 +231,21 @@ void HookArgument::Describe(ODesc* d) const { d->Add(""); } + break; + + case INPUT_FILE: + { + d->Add("("); + d->Add(input_file.first); + d->Add(", "); + if ( input_file.second ) + d->Add(input_file.second->substr(0, 20)); // cut content off + else + d->Add(""); + + d->Add(")"); + } + break; } } @@ -368,6 +384,13 @@ int Plugin::HookLoadFile(const LoadType type, const std::string& file, const std return -1; } +std::pair> Plugin::HookLoadFileExtended(const LoadType type, + const std::string& file, + const std::string& resolved) + { + return std::make_pair(-1, std::nullopt); + } + std::pair Plugin::HookFunctionCall(const Func* func, zeek::detail::Frame* parent, Args* args) { diff --git a/src/plugin/Plugin.h b/src/plugin/Plugin.h index b9dc8a6a22..f625fc1a49 100644 --- a/src/plugin/Plugin.h +++ b/src/plugin/Plugin.h @@ -5,6 +5,7 @@ #include "zeek/zeek-config.h" #include +#include #include #include @@ -57,6 +58,7 @@ enum HookType { // Note: when changing this table, update hook_name() in Plugin.cc. HOOK_LOAD_FILE, //< Activates Plugin::HookLoadFile(). + HOOK_LOAD_FILE_EXT, //< Activates Plugin::HookLoadFileExtended(). HOOK_CALL_FUNCTION, //< Activates Plugin::HookCallFunction(). HOOK_QUEUE_EVENT, //< Activates Plugin::HookQueueEvent(). HOOK_DRAIN_EVENTS, //< Activates Plugin::HookDrainEvents() @@ -205,7 +207,8 @@ public: CONN, THREAD_FIELDS, LOCATION, - ARG_LIST + ARG_LIST, + INPUT_FILE }; /** @@ -357,6 +360,15 @@ public: arg.args = args; } + /** + * Constructor with HookLoadFileExtended result describing an input file. + */ + explicit HookArgument(std::pair> file) + { + type = INPUT_FILE; + input_file = std::move(file); + } + /** * Returns the value for a boolen argument. The argument's type must * match accordingly. @@ -540,6 +552,7 @@ private: std::pair func_result; std::pair tfields; std::string arg_string; + std::pair> input_file; }; using HookArgumentList = std::list; @@ -815,6 +828,43 @@ protected: virtual int HookLoadFile(const LoadType type, const std::string& file, const std::string& resolved); + /** + * Hook into loading input files, with extended capabilities. This method + * will be called between InitPreScript() and InitPostScript(), but with no + * further order or timing guaranteed. It will be called once for each + * input file Bro is about to load, either given on the command line or via + * @load script directives. The hook can take over the file, in which case + * Bro will not further process it otherwise. It can, alternatively, also + * provide the file content as a string, which Bro will then process just + * as if it had read it from a file. + * + * @param type The type of load encountered: script load, signatures load, + * or plugin load. + * + * @param file The filename that was passed to @load. Only includes + * an extension if it was given in @load. + * + * @param resolved The file or directory name Bro resolved from + * the given path and is going to load. Empty string + * if Bro was not able to resolve a path. + * + * @return tuple of an integer and an optional string, where: the integer + * must be 1 if the plugin takes over loading the file (see below); 0 if + * the plugin wanted to take over the file but had trouble loading it + * (processing will abort in this case, and the plugin should have printed + * an error message); and -1 if the plugin wants Bro to proceeed processing + * the file normally. If the plugins takes over by returning 1, there are + * two cases: if the second tuple element remains unset, the plugin handled + * the loading completely internally; Bro will not do anything further with + * it. Alternatively, the plugin may optionally return the acutal content + * to use for the file as a string through the tuple's second element. If + * so, Bro will ignore the file on disk and use that provided content + * instead (including when there's actually no physical file in place on + * disk at all, and loading would have hence failed otherwise). + */ + virtual std::pair> + HookLoadFileExtended(const LoadType type, const std::string& file, const std::string& resolved); + /** * Hook into executing a script-level function/event/hook. Whenever * the script interpreter is about to execution a function, it first diff --git a/src/rule-scan.l b/src/rule-scan.l index 2b424d2ee5..3d9bc6da84 100644 --- a/src/rule-scan.l +++ b/src/rule-scan.l @@ -228,3 +228,23 @@ void end_PS() { BEGIN(INITIAL); } + +static YY_BUFFER_STATE rules_buffer; + +void rules_set_input_from_buffer(const char* data, size_t size) + { + rules_buffer = yy_scan_bytes(data, size); // this copies the data + } + +void rules_set_input_from_file(FILE* f) + { + rules_buffer = yy_create_buffer(f, YY_BUF_SIZE); + } + +void rules_parse_input() + { + yy_switch_to_buffer(rules_buffer); + rules_parse(); + yy_delete_buffer(rules_buffer); + } + diff --git a/src/scan.l b/src/scan.l index dbcb3fcfd3..0301494e2f 100644 --- a/src/scan.l +++ b/src/scan.l @@ -353,11 +353,14 @@ when return TOK_WHEN; @load-plugin{WS}{ID} { const char* plugin = zeek::util::skip_whitespace(yytext + 12); - int rc = PLUGIN_HOOK_WITH_RESULT(HOOK_LOAD_FILE, HookLoadFile(zeek::plugin::Plugin::PLUGIN, plugin, ""), -1); + std::pair> rc; + rc.first = PLUGIN_HOOK_WITH_RESULT(HOOK_LOAD_FILE, HookLoadFile(zeek::plugin::Plugin::PLUGIN, plugin, ""), -1); + if ( rc.first < 0 ) + rc = PLUGIN_HOOK_WITH_RESULT(HOOK_LOAD_FILE_EXT, HookLoadFileExtended(zeek::plugin::Plugin::PLUGIN, plugin, ""), std::make_pair(-1, std::nullopt)); - switch ( rc ) { + switch ( rc.first ) { case -1: - // No plugin in charge of this file. + // No plugin in charge of this file. (We ignore any returned content.) zeek::plugin_mgr->ActivateDynamicPlugin(plugin); break; @@ -560,12 +563,13 @@ YYLTYPE zeek::detail::GetCurrentLocation() static int load_files(const char* orig_file) { std::string file_path = find_relative_script_file(orig_file); - int rc = PLUGIN_HOOK_WITH_RESULT(HOOK_LOAD_FILE, HookLoadFile(zeek::plugin::Plugin::SCRIPT, orig_file, file_path), -1); - if ( rc == 1 ) - return 0; // A plugin took care of it, just skip. + std::pair> rc = {-1, std::nullopt}; + rc.first = PLUGIN_HOOK_WITH_RESULT(HOOK_LOAD_FILE, HookLoadFile(zeek::plugin::Plugin::SCRIPT, orig_file, file_path), -1); + if ( rc.first < 0 ) + rc = PLUGIN_HOOK_WITH_RESULT(HOOK_LOAD_FILE_EXT, HookLoadFileExtended(zeek::plugin::Plugin::SCRIPT, orig_file, file_path), std::make_pair(-1, std::nullopt)); - if ( rc == 0 ) + if ( rc.first == 0 ) { if ( ! zeek::reporter->Errors() ) // This is just in case the plugin failed to report @@ -576,55 +580,57 @@ static int load_files(const char* orig_file) exit(1); } - assert(rc == -1); // No plugin in charge of this file. + if ( rc.first == 1 && ! rc.second ) + return 0; // A plugin took care of it, just skip. FILE* f = nullptr; - if ( zeek::util::streq(orig_file, "-") ) + if ( rc.first == -1 ) { - f = stdin; - file_path = zeek::detail::ScannedFile::canonical_stdin_path; - - if ( zeek::detail::g_policy_debug ) + if ( zeek::util::streq(orig_file, "-") ) { - zeek::detail::debug_msg("Warning: can't use debugger while reading policy from stdin; turning off debugging.\n"); - zeek::detail::g_policy_debug = false; + f = stdin; + file_path = zeek::detail::ScannedFile::canonical_stdin_path; + + if ( zeek::detail::g_policy_debug ) + { + zeek::detail::debug_msg("Warning: can't use debugger while reading policy from stdin; turning off debugging.\n"); + zeek::detail::g_policy_debug = false; + } } - } - else - { - if ( file_path.empty() ) - zeek::reporter->FatalError("can't find %s", orig_file); - - if ( zeek::util::is_dir(file_path.c_str()) ) - f = zeek::util::detail::open_package(file_path); else - f = zeek::util::open_file(file_path); + { + if ( file_path.empty() ) + zeek::reporter->FatalError("can't find %s", orig_file); - if ( ! f ) - zeek::reporter->FatalError("can't open %s", file_path.c_str()); + if ( zeek::util::is_dir(file_path.c_str()) ) + f = zeek::util::detail::open_package(file_path); + else + f = zeek::util::open_file(file_path); + + if ( ! f ) + zeek::reporter->FatalError("can't open %s", file_path.c_str()); + } + + zeek::detail::ScannedFile sf(file_stack.length(), file_path); + if ( sf.AlreadyScanned() ) + { + if ( rc.first == -1 && f != stdin ) + fclose(f); + + return 0; + } + + zeek::detail::files_scanned.push_back(std::move(sf)); } - zeek::detail::ScannedFile sf(file_stack.length(), file_path); - - if ( sf.AlreadyScanned() ) - { - if ( f != stdin ) - fclose(f); - - return 0; - } - - zeek::detail::files_scanned.push_back(std::move(sf)); - if ( zeek::detail::g_policy_debug && ! file_path.empty() ) { // Add the filename to the file mapping table (Debug.h). zeek::detail::Filemap* map = new zeek::detail::Filemap; - zeek::detail::HashKey* key = new zeek::detail::HashKey(file_path.c_str()); zeek::detail::g_dbgfilemaps.emplace(file_path, map); - LoadPolicyFileText(file_path.c_str()); + LoadPolicyFileText(file_path.c_str(), rc.second); } // Remember where we were to restore the module scope in which @@ -633,12 +639,24 @@ static int load_files(const char* orig_file) zeek::detail::zeekygen_mgr->Script(file_path); - DBG_LOG(zeek::DBG_SCRIPTS, "Loading %s", file_path.c_str()); - // "orig_file", could be an alias for yytext, which is ephemeral // and will be zapped after the yy_switch_to_buffer() below. - yy_switch_to_buffer(yy_create_buffer(f, YY_BUF_SIZE)); + YY_BUFFER_STATE buffer; + if ( rc.first == 1 ) { + // Parse code provided by plugin. + assert(rc.second); + DBG_LOG(zeek::DBG_SCRIPTS, "Loading %s from code supplied by plugin ", file_path.c_str()); + buffer = yy_scan_bytes(rc.second->data(), rc.second->size()); // this copies the data + } + else { + // Parse from file. + assert(f); + DBG_LOG(zeek::DBG_SCRIPTS, "Loading %s", file_path.c_str()); + buffer = yy_create_buffer(f, YY_BUF_SIZE); + } + + yy_switch_to_buffer(buffer); yylloc.first_line = yylloc.last_line = line_number = 1; // Don't delete the old filename - it's pointed to by diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index b2e77df628..2e38a48d45 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -741,15 +741,15 @@ SetupResult setup(int argc, char** argv, Options* zopts) // Append signature files given on the command line for ( const auto& sf : options.signature_files ) - all_signature_files.push_back(sf); + all_signature_files.emplace_back(sf); // Append signature files defined in "signature_files" script option for ( auto&& sf : get_script_signature_files() ) - all_signature_files.push_back(std::move(sf)); + all_signature_files.emplace_back(std::move(sf)); // Append signature files defined in @load-sigs for ( const auto& sf : zeek::detail::sig_files ) - all_signature_files.push_back(sf); + all_signature_files.emplace_back(sf); if ( ! all_signature_files.empty() ) { diff --git a/testing/btest/Baseline/core.parse-only-signature-file-issues/missing-sig-file b/testing/btest/Baseline/core.parse-only-signature-file-issues/missing-sig-file index 3fc89824f4..aaaa346b61 100644 --- a/testing/btest/Baseline/core.parse-only-signature-file-issues/missing-sig-file +++ b/testing/btest/Baseline/core.parse-only-signature-file-issues/missing-sig-file @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error: Can't open signature file nope +error: failed to find file associated with @load-sigs nope diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index e340ed914b..898f73cb3b 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -1048,6 +1048,372 @@ 0.000000 MetaHookPost LoadFile(1, ./programming, <...>/programming.sig) -> -1 0.000000 MetaHookPost LoadFile(1, ./video, <...>/video.sig) -> -1 0.000000 MetaHookPost LoadFile(1, s2, ./s2.sig) -> -1 +0.000000 MetaHookPost LoadFileExtended(0, ../main, <...>/main.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ../plugin, <...>/plugin.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./CPP-load.bif.zeek, <...>/CPP-load.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_ARP.events.bif.zeek, <...>/Zeek_ARP.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_AsciiReader.ascii.bif.zeek, <...>/Zeek_AsciiReader.ascii.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_AsciiWriter.ascii.bif.zeek, <...>/Zeek_AsciiWriter.ascii.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_BenchmarkReader.benchmark.bif.zeek, <...>/Zeek_BenchmarkReader.benchmark.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_BinaryReader.binary.bif.zeek, <...>/Zeek_BinaryReader.binary.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_BitTorrent.events.bif.zeek, <...>/Zeek_BitTorrent.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_ConfigReader.config.bif.zeek, <...>/Zeek_ConfigReader.config.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_ConnSize.events.bif.zeek, <...>/Zeek_ConnSize.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_ConnSize.functions.bif.zeek, <...>/Zeek_ConnSize.functions.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_DCE_RPC.consts.bif.zeek, <...>/Zeek_DCE_RPC.consts.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_DCE_RPC.events.bif.zeek, <...>/Zeek_DCE_RPC.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_DCE_RPC.types.bif.zeek, <...>/Zeek_DCE_RPC.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_DHCP.events.bif.zeek, <...>/Zeek_DHCP.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_DHCP.types.bif.zeek, <...>/Zeek_DHCP.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_DNP3.events.bif.zeek, <...>/Zeek_DNP3.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_DNS.events.bif.zeek, <...>/Zeek_DNS.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_FTP.events.bif.zeek, <...>/Zeek_FTP.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_FTP.functions.bif.zeek, <...>/Zeek_FTP.functions.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_File.events.bif.zeek, <...>/Zeek_File.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_FileEntropy.events.bif.zeek, <...>/Zeek_FileEntropy.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_FileExtract.events.bif.zeek, <...>/Zeek_FileExtract.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_FileExtract.functions.bif.zeek, <...>/Zeek_FileExtract.functions.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_FileHash.events.bif.zeek, <...>/Zeek_FileHash.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Finger.events.bif.zeek, <...>/Zeek_Finger.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_GSSAPI.events.bif.zeek, <...>/Zeek_GSSAPI.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_GTPv1.events.bif.zeek, <...>/Zeek_GTPv1.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Geneve.events.bif.zeek, <...>/Zeek_Geneve.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Gnutella.events.bif.zeek, <...>/Zeek_Gnutella.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_HTTP.events.bif.zeek, <...>/Zeek_HTTP.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_HTTP.functions.bif.zeek, <...>/Zeek_HTTP.functions.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_ICMP.events.bif.zeek, <...>/Zeek_ICMP.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_IMAP.events.bif.zeek, <...>/Zeek_IMAP.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_IRC.events.bif.zeek, <...>/Zeek_IRC.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Ident.events.bif.zeek, <...>/Zeek_Ident.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_KRB.events.bif.zeek, <...>/Zeek_KRB.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_KRB.types.bif.zeek, <...>/Zeek_KRB.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Login.events.bif.zeek, <...>/Zeek_Login.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Login.functions.bif.zeek, <...>/Zeek_Login.functions.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_MIME.events.bif.zeek, <...>/Zeek_MIME.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_MQTT.events.bif.zeek, <...>/Zeek_MQTT.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_MQTT.types.bif.zeek, <...>/Zeek_MQTT.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Modbus.events.bif.zeek, <...>/Zeek_Modbus.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_MySQL.events.bif.zeek, <...>/Zeek_MySQL.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_NCP.consts.bif.zeek, <...>/Zeek_NCP.consts.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_NCP.events.bif.zeek, <...>/Zeek_NCP.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_NTLM.events.bif.zeek, <...>/Zeek_NTLM.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_NTLM.types.bif.zeek, <...>/Zeek_NTLM.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_NTP.events.bif.zeek, <...>/Zeek_NTP.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_NTP.types.bif.zeek, <...>/Zeek_NTP.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_NetBIOS.events.bif.zeek, <...>/Zeek_NetBIOS.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_NetBIOS.functions.bif.zeek, <...>/Zeek_NetBIOS.functions.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_NoneWriter.none.bif.zeek, <...>/Zeek_NoneWriter.none.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_PE.events.bif.zeek, <...>/Zeek_PE.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_POP3.events.bif.zeek, <...>/Zeek_POP3.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_RADIUS.events.bif.zeek, <...>/Zeek_RADIUS.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_RDP.events.bif.zeek, <...>/Zeek_RDP.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_RDP.types.bif.zeek, <...>/Zeek_RDP.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_RFB.events.bif.zeek, <...>/Zeek_RFB.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_RPC.events.bif.zeek, <...>/Zeek_RPC.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_RawReader.raw.bif.zeek, <...>/Zeek_RawReader.raw.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SIP.events.bif.zeek, <...>/Zeek_SIP.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.consts.bif.zeek, <...>/Zeek_SMB.consts.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.events.bif.zeek, <...>/Zeek_SMB.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_com_check_directory.bif.zeek, <...>/Zeek_SMB.smb1_com_check_directory.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_com_close.bif.zeek, <...>/Zeek_SMB.smb1_com_close.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_com_create_directory.bif.zeek, <...>/Zeek_SMB.smb1_com_create_directory.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_com_echo.bif.zeek, <...>/Zeek_SMB.smb1_com_echo.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_com_logoff_andx.bif.zeek, <...>/Zeek_SMB.smb1_com_logoff_andx.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_com_negotiate.bif.zeek, <...>/Zeek_SMB.smb1_com_negotiate.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_com_nt_cancel.bif.zeek, <...>/Zeek_SMB.smb1_com_nt_cancel.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_com_nt_create_andx.bif.zeek, <...>/Zeek_SMB.smb1_com_nt_create_andx.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_com_query_information.bif.zeek, <...>/Zeek_SMB.smb1_com_query_information.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_com_read_andx.bif.zeek, <...>/Zeek_SMB.smb1_com_read_andx.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_com_session_setup_andx.bif.zeek, <...>/Zeek_SMB.smb1_com_session_setup_andx.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_com_transaction.bif.zeek, <...>/Zeek_SMB.smb1_com_transaction.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_com_transaction2.bif.zeek, <...>/Zeek_SMB.smb1_com_transaction2.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_com_transaction2_secondary.bif.zeek, <...>/Zeek_SMB.smb1_com_transaction2_secondary.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_com_transaction_secondary.bif.zeek, <...>/Zeek_SMB.smb1_com_transaction_secondary.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_com_tree_connect_andx.bif.zeek, <...>/Zeek_SMB.smb1_com_tree_connect_andx.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_com_tree_disconnect.bif.zeek, <...>/Zeek_SMB.smb1_com_tree_disconnect.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_com_write_andx.bif.zeek, <...>/Zeek_SMB.smb1_com_write_andx.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb1_events.bif.zeek, <...>/Zeek_SMB.smb1_events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb2_com_close.bif.zeek, <...>/Zeek_SMB.smb2_com_close.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb2_com_create.bif.zeek, <...>/Zeek_SMB.smb2_com_create.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb2_com_negotiate.bif.zeek, <...>/Zeek_SMB.smb2_com_negotiate.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb2_com_read.bif.zeek, <...>/Zeek_SMB.smb2_com_read.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb2_com_session_setup.bif.zeek, <...>/Zeek_SMB.smb2_com_session_setup.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb2_com_set_info.bif.zeek, <...>/Zeek_SMB.smb2_com_set_info.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb2_com_transform_header.bif.zeek, <...>/Zeek_SMB.smb2_com_transform_header.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb2_com_tree_connect.bif.zeek, <...>/Zeek_SMB.smb2_com_tree_connect.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb2_com_tree_disconnect.bif.zeek, <...>/Zeek_SMB.smb2_com_tree_disconnect.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb2_com_write.bif.zeek, <...>/Zeek_SMB.smb2_com_write.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.smb2_events.bif.zeek, <...>/Zeek_SMB.smb2_events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMB.types.bif.zeek, <...>/Zeek_SMB.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMTP.events.bif.zeek, <...>/Zeek_SMTP.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SMTP.functions.bif.zeek, <...>/Zeek_SMTP.functions.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SNMP.events.bif.zeek, <...>/Zeek_SNMP.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SNMP.types.bif.zeek, <...>/Zeek_SNMP.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SOCKS.events.bif.zeek, <...>/Zeek_SOCKS.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SQLiteReader.sqlite.bif.zeek, <...>/Zeek_SQLiteReader.sqlite.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SQLiteWriter.sqlite.bif.zeek, <...>/Zeek_SQLiteWriter.sqlite.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SSH.events.bif.zeek, <...>/Zeek_SSH.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SSH.types.bif.zeek, <...>/Zeek_SSH.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SSL.consts.bif.zeek, <...>/Zeek_SSL.consts.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Syslog.events.bif.zeek, <...>/Zeek_Syslog.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_TCP.types.bif.zeek, <...>/Zeek_TCP.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Teredo.events.bif.zeek, <...>/Zeek_Teredo.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_UDP.events.bif.zeek, <...>/Zeek_UDP.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Unified2.events.bif.zeek, <...>/Zeek_Unified2.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Unified2.types.bif.zeek, <...>/Zeek_Unified2.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_VXLAN.events.bif.zeek, <...>/Zeek_VXLAN.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_X509.events.bif.zeek, <...>/Zeek_X509.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_X509.functions.bif.zeek, <...>/Zeek_X509.functions.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_X509.ocsp_events.bif.zeek, <...>/Zeek_X509.ocsp_events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_X509.types.bif.zeek, <...>/Zeek_X509.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_XMPP.events.bif.zeek, <...>/Zeek_XMPP.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./acld, <...>/acld.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./addrs, <...>/addrs.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./analyzer.bif.zeek, <...>/analyzer.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./api, <...>/api.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./average, <...>/average.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./bloom-filter.bif.zeek, <...>/bloom-filter.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./broker, <...>/broker.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./certificate-event-cache, <...>/certificate-event-cache.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./comm.bif.zeek, <...>/comm.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./const-dos-error, <...>/const-dos-error.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./const-nt-status, <...>/const-nt-status.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./const.bif.zeek, <...>/const.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./consts, <...>/consts.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./contents, <...>/contents.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./control, <...>/control.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./ct-list, <...>/ct-list.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./data.bif.zeek, <...>/data.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./dcc-send, <...>/dcc-send.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./debug, <...>/debug.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./drop, <...>/drop.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./entities, <...>/entities.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./event.bif.zeek, <...>/event.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./exec, <...>/exec.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./file_analysis.bif.zeek, <...>/file_analysis.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./files, <...>/files.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./gridftp, <...>/gridftp.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./hll_unique, <...>/hll_unique.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./hooks.bif.zeek, <...>/hooks.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./inactivity, <...>/inactivity.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./info, <...>/info.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./input, <...>/input.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./input.bif.zeek, <...>/input.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./last, <...>/last.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./log, <...>/log.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./log-ocsp, <...>/log-ocsp.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./logging.bif.zeek, <...>/logging.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./magic, <...>/magic) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./main, <...>/main.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./max, <...>/max.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./messaging.bif.zeek, <...>/messaging.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./min, <...>/min.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./mozilla-ca-list, <...>/mozilla-ca-list.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./netstats, <...>/netstats.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./non-cluster, <...>/non-cluster.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./openflow, <...>/openflow.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./option.bif.zeek, <...>/option.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./packet_analysis.bif.zeek, <...>/packet_analysis.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./packetfilter, <...>/packetfilter.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./patterns, <...>/patterns.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./pcap.bif.zeek, <...>/pcap.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./plugin, <...>/plugin.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./plugins, <...>/plugins) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./polling, <...>/polling.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./pools, <...>/pools.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./postprocessors, <...>/postprocessors) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./removal-hooks, <...>/removal-hooks.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./reporter.bif.zeek, <...>/reporter.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./ryu, <...>/ryu.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./sample, <...>/sample.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./scp, <...>/scp.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./sftp, <...>/sftp.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./shunt, <...>/shunt.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./site, <...>/site.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./smb1-main, <...>/smb1-main.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./smb2-main, <...>/smb2-main.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./std-dev, <...>/std-dev.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./store, <...>/store.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./store.bif.zeek, <...>/store.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./strings.bif.zeek, <...>/strings.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./sum, <...>/sum.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./supervisor.bif.zeek, <...>/supervisor.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./telemetry.bif.zeek, <...>/telemetry.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./thresholds, <...>/thresholds.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./top-k.bif.zeek, <...>/top-k.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./topk, <...>/topk.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./types, <...>/types.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./types.bif.zeek, <...>/types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./unique, <...>/unique.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./utils, <...>/utils.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./utils-commands, <...>/utils-commands.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./variance, <...>/variance.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./weird, <...>/weird.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./zeek.bif.zeek, <...>/zeek.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./zeekygen.bif.zeek, <...>/zeekygen.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, .<...>/add-geodata, <...>/add-geodata.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, .<...>/ascii, <...>/ascii.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, .<...>/benchmark, <...>/benchmark.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, .<...>/binary, <...>/binary.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, .<...>/config, <...>/config.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, .<...>/email_admin, <...>/email_admin.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, .<...>/none, <...>/none.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, .<...>/page, <...>/page.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, .<...>/pp-alarms, <...>/pp-alarms.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, .<...>/raw, <...>/raw.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, .<...>/sqlite, <...>/sqlite.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, <...>/__load__.zeek, <...>/__load__.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, <...>/__preload__.zeek, <...>/__preload__.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, <...>/hooks.zeek, <...>/hooks.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base/bif, <...>/bif) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base/init-default, <...>/init-default.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base/packet-protocols, <...>/packet-protocols) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/CPP-load.bif, <...>/CPP-load.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/Zeek_KRB.types.bif, <...>/Zeek_KRB.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/Zeek_SNMP.types.bif, <...>/Zeek_SNMP.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/active-http, <...>/active-http.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/addrs, <...>/addrs.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/analyzer, <...>/analyzer) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/analyzer.bif, <...>/analyzer.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/api, <...>/api.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/backtrace, <...>/backtrace.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/broker, <...>/broker) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/cluster, <...>/cluster) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/comm.bif, <...>/comm.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/config, <...>/config) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/conn, <...>/conn) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/conn-ids, <...>/conn-ids.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/const.bif, <...>/const.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/control, <...>/control) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/data.bif, <...>/data.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/dce-rpc, <...>/dce-rpc) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/dhcp, <...>/dhcp) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/dir, <...>/dir.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/directions-and-hosts, <...>/directions-and-hosts.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/dnp3, <...>/dnp3) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/dns, <...>/dns) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/dpd, <...>/dpd) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/email, <...>/email.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/ethernet, <...>/ethernet) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/event.bif, <...>/event.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/exec, <...>/exec.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/extract, <...>/extract) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/fddi, <...>/fddi) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/file_analysis.bif, <...>/file_analysis.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/files, <...>/files) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/files, <...>/files.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/find-checksum-offloading, <...>/find-checksum-offloading.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/find-filtered-trace, <...>/find-filtered-trace.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/ftp, <...>/ftp) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/geoip-distance, <...>/geoip-distance.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/gre, <...>/gre) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/hash, <...>/hash) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/hash_hrw, <...>/hash_hrw.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/http, <...>/http) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/icmp, <...>/icmp) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/ieee802_11, <...>/ieee802_11) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/ieee802_11_radio, <...>/ieee802_11_radio) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/imap, <...>/imap) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/input, <...>/input) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/input.bif, <...>/input.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/intel, <...>/intel) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/ip, <...>/ip) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/iptunnel, <...>/iptunnel) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/irc, <...>/irc) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/krb, <...>/krb) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/linux_sll, <...>/linux_sll) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/logging, <...>/logging) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/logging.bif, <...>/logging.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/main, <...>/main.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/messaging.bif, <...>/messaging.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/modbus, <...>/modbus) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/mpls, <...>/mpls) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/mqtt, <...>/mqtt) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/mysql, <...>/mysql) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/netcontrol, <...>/netcontrol) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/nflog, <...>/nflog) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/notice, <...>/notice) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/ntlm, <...>/ntlm) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/ntp, <...>/ntp) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/null, <...>/null) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/numbers, <...>/numbers.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/openflow, <...>/openflow) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/option.bif, <...>/option.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/packet-filter, <...>/packet-filter) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/packet_analysis.bif, <...>/packet_analysis.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/paths, <...>/paths.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/patterns, <...>/patterns.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/pe, <...>/pe) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/plugins, <...>/plugins) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/pop3, <...>/pop3) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/ppp_serial, <...>/ppp_serial) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/pppoe, <...>/pppoe) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/queue, <...>/queue.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/radius, <...>/radius) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/rdp, <...>/rdp) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/removal-hooks, <...>/removal-hooks.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/reporter, <...>/reporter) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/reporter.bif, <...>/reporter.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/rfb, <...>/rfb) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/root, <...>/root) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/signatures, <...>/signatures) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/sip, <...>/sip) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/site, <...>/site.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/skip, <...>/skip) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/smb, <...>/smb) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/smtp, <...>/smtp) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/snmp, <...>/snmp) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/socks, <...>/socks) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/software, <...>/software) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/ssh, <...>/ssh) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/ssl, <...>/ssl) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/stats.bif, <...>/stats.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/store.bif, <...>/store.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/strings, <...>/strings.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/strings.bif, <...>/strings.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/sumstats, <...>/sumstats) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/supervisor, <...>/supervisor) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/supervisor.bif, <...>/supervisor.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/syslog, <...>/syslog) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/tcp, <...>/tcp) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/thresholds, <...>/thresholds.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/time, <...>/time.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/tunnels, <...>/tunnels) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/types.bif, <...>/types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/udp, <...>/udp) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/urls, <...>/urls.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/utils, <...>/utils.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/version, <...>/version.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/vlan, <...>/vlan) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/vntag, <...>/vntag) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/weird, <...>/weird.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/x509, <...>/x509) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/xmpp, <...>/xmpp) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/zeek.bif, <...>/zeek.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, builtin-plugins/__load__.zeek, <...>/__load__.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, builtin-plugins/__preload__.zeek, <...>/__preload__.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, s1.sig, ./s1.sig) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(1, ./archive, <...>/archive.sig) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(1, ./audio, <...>/audio.sig) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(1, ./dpd.sig, <...>/dpd.sig) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(1, ./executable, <...>/executable.sig) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(1, ./font, <...>/font.sig) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(1, ./general, <...>/general.sig) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(1, ./image, <...>/image.sig) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(1, ./java, <...>/java.sig) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(1, ./libmagic, <...>/libmagic.sig) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(1, ./office, <...>/office.sig) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(1, ./programming, <...>/programming.sig) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(1, ./video, <...>/video.sig) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(1, s2, ./s2.sig) -> (-1, ) 0.000000 MetaHookPost LogInit(Log::WRITER_ASCII, default, true, true, packet_filter(0.0,0.0,0.0), 5, {ts (time), node (string), filter (string), init (bool), success (bool)}) -> 0.000000 MetaHookPost LogWrite(Log::WRITER_ASCII, default, packet_filter(0.0,0.0,0.0), 5, {ts (time), node (string), filter (string), init (bool), success (bool)}, ) -> true 0.000000 MetaHookPost QueueEvent(NetControl::init()) -> false @@ -2102,6 +2468,372 @@ 0.000000 MetaHookPre LoadFile(1, ./programming, <...>/programming.sig) 0.000000 MetaHookPre LoadFile(1, ./video, <...>/video.sig) 0.000000 MetaHookPre LoadFile(1, s2, ./s2.sig) +0.000000 MetaHookPre LoadFileExtended(0, ../main, <...>/main.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ../plugin, <...>/plugin.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./CPP-load.bif.zeek, <...>/CPP-load.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_ARP.events.bif.zeek, <...>/Zeek_ARP.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_AsciiReader.ascii.bif.zeek, <...>/Zeek_AsciiReader.ascii.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_AsciiWriter.ascii.bif.zeek, <...>/Zeek_AsciiWriter.ascii.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_BenchmarkReader.benchmark.bif.zeek, <...>/Zeek_BenchmarkReader.benchmark.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_BinaryReader.binary.bif.zeek, <...>/Zeek_BinaryReader.binary.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_BitTorrent.events.bif.zeek, <...>/Zeek_BitTorrent.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_ConfigReader.config.bif.zeek, <...>/Zeek_ConfigReader.config.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_ConnSize.events.bif.zeek, <...>/Zeek_ConnSize.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_ConnSize.functions.bif.zeek, <...>/Zeek_ConnSize.functions.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_DCE_RPC.consts.bif.zeek, <...>/Zeek_DCE_RPC.consts.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_DCE_RPC.events.bif.zeek, <...>/Zeek_DCE_RPC.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_DCE_RPC.types.bif.zeek, <...>/Zeek_DCE_RPC.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_DHCP.events.bif.zeek, <...>/Zeek_DHCP.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_DHCP.types.bif.zeek, <...>/Zeek_DHCP.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_DNP3.events.bif.zeek, <...>/Zeek_DNP3.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_DNS.events.bif.zeek, <...>/Zeek_DNS.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_FTP.events.bif.zeek, <...>/Zeek_FTP.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_FTP.functions.bif.zeek, <...>/Zeek_FTP.functions.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_File.events.bif.zeek, <...>/Zeek_File.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_FileEntropy.events.bif.zeek, <...>/Zeek_FileEntropy.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_FileExtract.events.bif.zeek, <...>/Zeek_FileExtract.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_FileExtract.functions.bif.zeek, <...>/Zeek_FileExtract.functions.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_FileHash.events.bif.zeek, <...>/Zeek_FileHash.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Finger.events.bif.zeek, <...>/Zeek_Finger.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_GSSAPI.events.bif.zeek, <...>/Zeek_GSSAPI.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_GTPv1.events.bif.zeek, <...>/Zeek_GTPv1.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Geneve.events.bif.zeek, <...>/Zeek_Geneve.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Gnutella.events.bif.zeek, <...>/Zeek_Gnutella.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_HTTP.events.bif.zeek, <...>/Zeek_HTTP.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_HTTP.functions.bif.zeek, <...>/Zeek_HTTP.functions.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_ICMP.events.bif.zeek, <...>/Zeek_ICMP.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_IMAP.events.bif.zeek, <...>/Zeek_IMAP.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_IRC.events.bif.zeek, <...>/Zeek_IRC.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Ident.events.bif.zeek, <...>/Zeek_Ident.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_KRB.events.bif.zeek, <...>/Zeek_KRB.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_KRB.types.bif.zeek, <...>/Zeek_KRB.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Login.events.bif.zeek, <...>/Zeek_Login.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Login.functions.bif.zeek, <...>/Zeek_Login.functions.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_MIME.events.bif.zeek, <...>/Zeek_MIME.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_MQTT.events.bif.zeek, <...>/Zeek_MQTT.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_MQTT.types.bif.zeek, <...>/Zeek_MQTT.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Modbus.events.bif.zeek, <...>/Zeek_Modbus.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_MySQL.events.bif.zeek, <...>/Zeek_MySQL.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_NCP.consts.bif.zeek, <...>/Zeek_NCP.consts.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_NCP.events.bif.zeek, <...>/Zeek_NCP.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_NTLM.events.bif.zeek, <...>/Zeek_NTLM.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_NTLM.types.bif.zeek, <...>/Zeek_NTLM.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_NTP.events.bif.zeek, <...>/Zeek_NTP.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_NTP.types.bif.zeek, <...>/Zeek_NTP.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_NetBIOS.events.bif.zeek, <...>/Zeek_NetBIOS.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_NetBIOS.functions.bif.zeek, <...>/Zeek_NetBIOS.functions.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_NoneWriter.none.bif.zeek, <...>/Zeek_NoneWriter.none.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_PE.events.bif.zeek, <...>/Zeek_PE.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_POP3.events.bif.zeek, <...>/Zeek_POP3.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_RADIUS.events.bif.zeek, <...>/Zeek_RADIUS.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_RDP.events.bif.zeek, <...>/Zeek_RDP.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_RDP.types.bif.zeek, <...>/Zeek_RDP.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_RFB.events.bif.zeek, <...>/Zeek_RFB.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_RPC.events.bif.zeek, <...>/Zeek_RPC.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_RawReader.raw.bif.zeek, <...>/Zeek_RawReader.raw.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SIP.events.bif.zeek, <...>/Zeek_SIP.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.consts.bif.zeek, <...>/Zeek_SMB.consts.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.events.bif.zeek, <...>/Zeek_SMB.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_com_check_directory.bif.zeek, <...>/Zeek_SMB.smb1_com_check_directory.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_com_close.bif.zeek, <...>/Zeek_SMB.smb1_com_close.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_com_create_directory.bif.zeek, <...>/Zeek_SMB.smb1_com_create_directory.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_com_echo.bif.zeek, <...>/Zeek_SMB.smb1_com_echo.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_com_logoff_andx.bif.zeek, <...>/Zeek_SMB.smb1_com_logoff_andx.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_com_negotiate.bif.zeek, <...>/Zeek_SMB.smb1_com_negotiate.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_com_nt_cancel.bif.zeek, <...>/Zeek_SMB.smb1_com_nt_cancel.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_com_nt_create_andx.bif.zeek, <...>/Zeek_SMB.smb1_com_nt_create_andx.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_com_query_information.bif.zeek, <...>/Zeek_SMB.smb1_com_query_information.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_com_read_andx.bif.zeek, <...>/Zeek_SMB.smb1_com_read_andx.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_com_session_setup_andx.bif.zeek, <...>/Zeek_SMB.smb1_com_session_setup_andx.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_com_transaction.bif.zeek, <...>/Zeek_SMB.smb1_com_transaction.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_com_transaction2.bif.zeek, <...>/Zeek_SMB.smb1_com_transaction2.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_com_transaction2_secondary.bif.zeek, <...>/Zeek_SMB.smb1_com_transaction2_secondary.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_com_transaction_secondary.bif.zeek, <...>/Zeek_SMB.smb1_com_transaction_secondary.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_com_tree_connect_andx.bif.zeek, <...>/Zeek_SMB.smb1_com_tree_connect_andx.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_com_tree_disconnect.bif.zeek, <...>/Zeek_SMB.smb1_com_tree_disconnect.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_com_write_andx.bif.zeek, <...>/Zeek_SMB.smb1_com_write_andx.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb1_events.bif.zeek, <...>/Zeek_SMB.smb1_events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb2_com_close.bif.zeek, <...>/Zeek_SMB.smb2_com_close.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb2_com_create.bif.zeek, <...>/Zeek_SMB.smb2_com_create.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb2_com_negotiate.bif.zeek, <...>/Zeek_SMB.smb2_com_negotiate.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb2_com_read.bif.zeek, <...>/Zeek_SMB.smb2_com_read.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb2_com_session_setup.bif.zeek, <...>/Zeek_SMB.smb2_com_session_setup.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb2_com_set_info.bif.zeek, <...>/Zeek_SMB.smb2_com_set_info.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb2_com_transform_header.bif.zeek, <...>/Zeek_SMB.smb2_com_transform_header.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb2_com_tree_connect.bif.zeek, <...>/Zeek_SMB.smb2_com_tree_connect.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb2_com_tree_disconnect.bif.zeek, <...>/Zeek_SMB.smb2_com_tree_disconnect.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb2_com_write.bif.zeek, <...>/Zeek_SMB.smb2_com_write.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.smb2_events.bif.zeek, <...>/Zeek_SMB.smb2_events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMB.types.bif.zeek, <...>/Zeek_SMB.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMTP.events.bif.zeek, <...>/Zeek_SMTP.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SMTP.functions.bif.zeek, <...>/Zeek_SMTP.functions.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SNMP.events.bif.zeek, <...>/Zeek_SNMP.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SNMP.types.bif.zeek, <...>/Zeek_SNMP.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SOCKS.events.bif.zeek, <...>/Zeek_SOCKS.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SQLiteReader.sqlite.bif.zeek, <...>/Zeek_SQLiteReader.sqlite.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SQLiteWriter.sqlite.bif.zeek, <...>/Zeek_SQLiteWriter.sqlite.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SSH.events.bif.zeek, <...>/Zeek_SSH.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SSH.types.bif.zeek, <...>/Zeek_SSH.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SSL.consts.bif.zeek, <...>/Zeek_SSL.consts.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Syslog.events.bif.zeek, <...>/Zeek_Syslog.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_TCP.types.bif.zeek, <...>/Zeek_TCP.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Teredo.events.bif.zeek, <...>/Zeek_Teredo.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_UDP.events.bif.zeek, <...>/Zeek_UDP.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Unified2.events.bif.zeek, <...>/Zeek_Unified2.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Unified2.types.bif.zeek, <...>/Zeek_Unified2.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_VXLAN.events.bif.zeek, <...>/Zeek_VXLAN.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_X509.events.bif.zeek, <...>/Zeek_X509.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_X509.functions.bif.zeek, <...>/Zeek_X509.functions.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_X509.ocsp_events.bif.zeek, <...>/Zeek_X509.ocsp_events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_X509.types.bif.zeek, <...>/Zeek_X509.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_XMPP.events.bif.zeek, <...>/Zeek_XMPP.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./acld, <...>/acld.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./addrs, <...>/addrs.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./analyzer.bif.zeek, <...>/analyzer.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./api, <...>/api.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./average, <...>/average.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./bloom-filter.bif.zeek, <...>/bloom-filter.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./broker, <...>/broker.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./certificate-event-cache, <...>/certificate-event-cache.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./comm.bif.zeek, <...>/comm.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./const-dos-error, <...>/const-dos-error.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./const-nt-status, <...>/const-nt-status.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./const.bif.zeek, <...>/const.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./consts, <...>/consts.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./contents, <...>/contents.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./control, <...>/control.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./ct-list, <...>/ct-list.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./data.bif.zeek, <...>/data.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./dcc-send, <...>/dcc-send.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./debug, <...>/debug.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./drop, <...>/drop.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./entities, <...>/entities.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./event.bif.zeek, <...>/event.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./exec, <...>/exec.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./file_analysis.bif.zeek, <...>/file_analysis.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./files, <...>/files.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./gridftp, <...>/gridftp.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./hll_unique, <...>/hll_unique.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./hooks.bif.zeek, <...>/hooks.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./inactivity, <...>/inactivity.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./info, <...>/info.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./input, <...>/input.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./input.bif.zeek, <...>/input.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./last, <...>/last.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./log, <...>/log.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./log-ocsp, <...>/log-ocsp.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./logging.bif.zeek, <...>/logging.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./magic, <...>/magic) +0.000000 MetaHookPre LoadFileExtended(0, ./main, <...>/main.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./max, <...>/max.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./messaging.bif.zeek, <...>/messaging.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./min, <...>/min.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./mozilla-ca-list, <...>/mozilla-ca-list.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./netstats, <...>/netstats.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./non-cluster, <...>/non-cluster.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./openflow, <...>/openflow.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./option.bif.zeek, <...>/option.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./packet_analysis.bif.zeek, <...>/packet_analysis.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./packetfilter, <...>/packetfilter.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./patterns, <...>/patterns.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./pcap.bif.zeek, <...>/pcap.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./plugin, <...>/plugin.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./plugins, <...>/plugins) +0.000000 MetaHookPre LoadFileExtended(0, ./polling, <...>/polling.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./pools, <...>/pools.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./postprocessors, <...>/postprocessors) +0.000000 MetaHookPre LoadFileExtended(0, ./removal-hooks, <...>/removal-hooks.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./reporter.bif.zeek, <...>/reporter.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./ryu, <...>/ryu.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./sample, <...>/sample.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./scp, <...>/scp.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./sftp, <...>/sftp.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./shunt, <...>/shunt.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./site, <...>/site.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./smb1-main, <...>/smb1-main.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./smb2-main, <...>/smb2-main.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./stats.bif.zeek, <...>/stats.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./std-dev, <...>/std-dev.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./store, <...>/store.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./store.bif.zeek, <...>/store.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./strings.bif.zeek, <...>/strings.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./sum, <...>/sum.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./supervisor.bif.zeek, <...>/supervisor.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./telemetry.bif.zeek, <...>/telemetry.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./thresholds, <...>/thresholds.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./top-k.bif.zeek, <...>/top-k.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./topk, <...>/topk.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./types, <...>/types.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./types.bif.zeek, <...>/types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./unique, <...>/unique.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./utils, <...>/utils.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./utils-commands, <...>/utils-commands.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./variance, <...>/variance.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./weird, <...>/weird.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./zeek.bif.zeek, <...>/zeek.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./zeekygen.bif.zeek, <...>/zeekygen.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, .<...>/add-geodata, <...>/add-geodata.zeek) +0.000000 MetaHookPre LoadFileExtended(0, .<...>/ascii, <...>/ascii.zeek) +0.000000 MetaHookPre LoadFileExtended(0, .<...>/benchmark, <...>/benchmark.zeek) +0.000000 MetaHookPre LoadFileExtended(0, .<...>/binary, <...>/binary.zeek) +0.000000 MetaHookPre LoadFileExtended(0, .<...>/config, <...>/config.zeek) +0.000000 MetaHookPre LoadFileExtended(0, .<...>/email_admin, <...>/email_admin.zeek) +0.000000 MetaHookPre LoadFileExtended(0, .<...>/none, <...>/none.zeek) +0.000000 MetaHookPre LoadFileExtended(0, .<...>/page, <...>/page.zeek) +0.000000 MetaHookPre LoadFileExtended(0, .<...>/pp-alarms, <...>/pp-alarms.zeek) +0.000000 MetaHookPre LoadFileExtended(0, .<...>/raw, <...>/raw.zeek) +0.000000 MetaHookPre LoadFileExtended(0, .<...>/sqlite, <...>/sqlite.zeek) +0.000000 MetaHookPre LoadFileExtended(0, <...>/__load__.zeek, <...>/__load__.zeek) +0.000000 MetaHookPre LoadFileExtended(0, <...>/__preload__.zeek, <...>/__preload__.zeek) +0.000000 MetaHookPre LoadFileExtended(0, <...>/hooks.zeek, <...>/hooks.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base/bif, <...>/bif) +0.000000 MetaHookPre LoadFileExtended(0, base/init-default, <...>/init-default.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base/packet-protocols, <...>/packet-protocols) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/CPP-load.bif, <...>/CPP-load.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/Zeek_KRB.types.bif, <...>/Zeek_KRB.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/Zeek_SNMP.types.bif, <...>/Zeek_SNMP.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/active-http, <...>/active-http.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/addrs, <...>/addrs.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/analyzer, <...>/analyzer) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/analyzer.bif, <...>/analyzer.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/api, <...>/api.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/backtrace, <...>/backtrace.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/broker, <...>/broker) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/cluster, <...>/cluster) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/comm.bif, <...>/comm.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/config, <...>/config) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/conn, <...>/conn) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/conn-ids, <...>/conn-ids.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/const.bif, <...>/const.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/control, <...>/control) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/data.bif, <...>/data.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/dce-rpc, <...>/dce-rpc) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/dhcp, <...>/dhcp) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/dir, <...>/dir.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/directions-and-hosts, <...>/directions-and-hosts.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/dnp3, <...>/dnp3) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/dns, <...>/dns) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/dpd, <...>/dpd) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/email, <...>/email.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/ethernet, <...>/ethernet) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/event.bif, <...>/event.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/exec, <...>/exec.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/extract, <...>/extract) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/fddi, <...>/fddi) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/file_analysis.bif, <...>/file_analysis.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/files, <...>/files) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/files, <...>/files.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/find-checksum-offloading, <...>/find-checksum-offloading.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/find-filtered-trace, <...>/find-filtered-trace.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/ftp, <...>/ftp) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/geoip-distance, <...>/geoip-distance.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/gre, <...>/gre) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/hash, <...>/hash) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/hash_hrw, <...>/hash_hrw.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/http, <...>/http) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/icmp, <...>/icmp) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/ieee802_11, <...>/ieee802_11) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/ieee802_11_radio, <...>/ieee802_11_radio) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/imap, <...>/imap) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/input, <...>/input) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/input.bif, <...>/input.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/intel, <...>/intel) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/ip, <...>/ip) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/iptunnel, <...>/iptunnel) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/irc, <...>/irc) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/krb, <...>/krb) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/linux_sll, <...>/linux_sll) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/logging, <...>/logging) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/logging.bif, <...>/logging.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/main, <...>/main.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/messaging.bif, <...>/messaging.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/modbus, <...>/modbus) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/mpls, <...>/mpls) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/mqtt, <...>/mqtt) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/mysql, <...>/mysql) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/netcontrol, <...>/netcontrol) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/nflog, <...>/nflog) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/notice, <...>/notice) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/ntlm, <...>/ntlm) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/ntp, <...>/ntp) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/null, <...>/null) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/numbers, <...>/numbers.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/openflow, <...>/openflow) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/option.bif, <...>/option.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/packet-filter, <...>/packet-filter) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/packet_analysis.bif, <...>/packet_analysis.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/paths, <...>/paths.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/patterns, <...>/patterns.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/pe, <...>/pe) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/plugins, <...>/plugins) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/pop3, <...>/pop3) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/ppp_serial, <...>/ppp_serial) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/pppoe, <...>/pppoe) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/queue, <...>/queue.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/radius, <...>/radius) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/rdp, <...>/rdp) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/removal-hooks, <...>/removal-hooks.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/reporter, <...>/reporter) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/reporter.bif, <...>/reporter.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/rfb, <...>/rfb) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/root, <...>/root) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/signatures, <...>/signatures) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/sip, <...>/sip) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/site, <...>/site.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/skip, <...>/skip) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/smb, <...>/smb) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/smtp, <...>/smtp) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/snmp, <...>/snmp) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/socks, <...>/socks) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/software, <...>/software) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/ssh, <...>/ssh) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/ssl, <...>/ssl) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/stats.bif, <...>/stats.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/store.bif, <...>/store.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/strings, <...>/strings.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/strings.bif, <...>/strings.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/sumstats, <...>/sumstats) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/supervisor, <...>/supervisor) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/supervisor.bif, <...>/supervisor.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/syslog, <...>/syslog) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/tcp, <...>/tcp) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/thresholds, <...>/thresholds.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/time, <...>/time.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/tunnels, <...>/tunnels) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/types.bif, <...>/types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/udp, <...>/udp) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/urls, <...>/urls.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/utils, <...>/utils.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/version, <...>/version.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/vlan, <...>/vlan) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/vntag, <...>/vntag) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/weird, <...>/weird.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/x509, <...>/x509) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/xmpp, <...>/xmpp) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/zeek.bif, <...>/zeek.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, builtin-plugins/__load__.zeek, <...>/__load__.zeek) +0.000000 MetaHookPre LoadFileExtended(0, builtin-plugins/__preload__.zeek, <...>/__preload__.zeek) +0.000000 MetaHookPre LoadFileExtended(0, s1.sig, ./s1.sig) +0.000000 MetaHookPre LoadFileExtended(1, ./archive, <...>/archive.sig) +0.000000 MetaHookPre LoadFileExtended(1, ./audio, <...>/audio.sig) +0.000000 MetaHookPre LoadFileExtended(1, ./dpd.sig, <...>/dpd.sig) +0.000000 MetaHookPre LoadFileExtended(1, ./executable, <...>/executable.sig) +0.000000 MetaHookPre LoadFileExtended(1, ./font, <...>/font.sig) +0.000000 MetaHookPre LoadFileExtended(1, ./general, <...>/general.sig) +0.000000 MetaHookPre LoadFileExtended(1, ./image, <...>/image.sig) +0.000000 MetaHookPre LoadFileExtended(1, ./java, <...>/java.sig) +0.000000 MetaHookPre LoadFileExtended(1, ./libmagic, <...>/libmagic.sig) +0.000000 MetaHookPre LoadFileExtended(1, ./office, <...>/office.sig) +0.000000 MetaHookPre LoadFileExtended(1, ./programming, <...>/programming.sig) +0.000000 MetaHookPre LoadFileExtended(1, ./video, <...>/video.sig) +0.000000 MetaHookPre LoadFileExtended(1, s2, ./s2.sig) 0.000000 MetaHookPre LogInit(Log::WRITER_ASCII, default, true, true, packet_filter(0.0,0.0,0.0), 5, {ts (time), node (string), filter (string), init (bool), success (bool)}) 0.000000 MetaHookPre LogWrite(Log::WRITER_ASCII, default, packet_filter(0.0,0.0,0.0), 5, {ts (time), node (string), filter (string), init (bool), success (bool)}, ) 0.000000 MetaHookPre QueueEvent(NetControl::init()) @@ -3155,6 +3887,372 @@ 0.000000 | HookLoadFile builtin-plugins/__preload__.zeek <...>/__preload__.zeek 0.000000 | HookLoadFile s1.sig ./s1.sig 0.000000 | HookLoadFile s2 ./s2.sig +0.000000 | HookLoadFileExtended ../main <...>/main.zeek +0.000000 | HookLoadFileExtended ../plugin <...>/plugin.zeek +0.000000 | HookLoadFileExtended ./CPP-load.bif.zeek <...>/CPP-load.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_ARP.events.bif.zeek <...>/Zeek_ARP.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_AsciiReader.ascii.bif.zeek <...>/Zeek_AsciiReader.ascii.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_AsciiWriter.ascii.bif.zeek <...>/Zeek_AsciiWriter.ascii.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_BenchmarkReader.benchmark.bif.zeek <...>/Zeek_BenchmarkReader.benchmark.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_BinaryReader.binary.bif.zeek <...>/Zeek_BinaryReader.binary.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_BitTorrent.events.bif.zeek <...>/Zeek_BitTorrent.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_ConfigReader.config.bif.zeek <...>/Zeek_ConfigReader.config.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_ConnSize.events.bif.zeek <...>/Zeek_ConnSize.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_ConnSize.functions.bif.zeek <...>/Zeek_ConnSize.functions.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_DCE_RPC.consts.bif.zeek <...>/Zeek_DCE_RPC.consts.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_DCE_RPC.events.bif.zeek <...>/Zeek_DCE_RPC.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_DCE_RPC.types.bif.zeek <...>/Zeek_DCE_RPC.types.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_DHCP.events.bif.zeek <...>/Zeek_DHCP.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_DHCP.types.bif.zeek <...>/Zeek_DHCP.types.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_DNP3.events.bif.zeek <...>/Zeek_DNP3.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_DNS.events.bif.zeek <...>/Zeek_DNS.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_FTP.events.bif.zeek <...>/Zeek_FTP.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_FTP.functions.bif.zeek <...>/Zeek_FTP.functions.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_File.events.bif.zeek <...>/Zeek_File.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_FileEntropy.events.bif.zeek <...>/Zeek_FileEntropy.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_FileExtract.events.bif.zeek <...>/Zeek_FileExtract.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_FileExtract.functions.bif.zeek <...>/Zeek_FileExtract.functions.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_FileHash.events.bif.zeek <...>/Zeek_FileHash.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_Finger.events.bif.zeek <...>/Zeek_Finger.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_GSSAPI.events.bif.zeek <...>/Zeek_GSSAPI.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_GTPv1.events.bif.zeek <...>/Zeek_GTPv1.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_Geneve.events.bif.zeek <...>/Zeek_Geneve.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_Gnutella.events.bif.zeek <...>/Zeek_Gnutella.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_HTTP.events.bif.zeek <...>/Zeek_HTTP.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_HTTP.functions.bif.zeek <...>/Zeek_HTTP.functions.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_ICMP.events.bif.zeek <...>/Zeek_ICMP.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_IMAP.events.bif.zeek <...>/Zeek_IMAP.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_IRC.events.bif.zeek <...>/Zeek_IRC.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_Ident.events.bif.zeek <...>/Zeek_Ident.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_KRB.events.bif.zeek <...>/Zeek_KRB.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_KRB.types.bif.zeek <...>/Zeek_KRB.types.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_Login.events.bif.zeek <...>/Zeek_Login.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_Login.functions.bif.zeek <...>/Zeek_Login.functions.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_MIME.events.bif.zeek <...>/Zeek_MIME.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_MQTT.events.bif.zeek <...>/Zeek_MQTT.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_MQTT.types.bif.zeek <...>/Zeek_MQTT.types.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_Modbus.events.bif.zeek <...>/Zeek_Modbus.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_MySQL.events.bif.zeek <...>/Zeek_MySQL.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_NCP.consts.bif.zeek <...>/Zeek_NCP.consts.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_NCP.events.bif.zeek <...>/Zeek_NCP.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_NTLM.events.bif.zeek <...>/Zeek_NTLM.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_NTLM.types.bif.zeek <...>/Zeek_NTLM.types.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_NTP.events.bif.zeek <...>/Zeek_NTP.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_NTP.types.bif.zeek <...>/Zeek_NTP.types.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_NetBIOS.events.bif.zeek <...>/Zeek_NetBIOS.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_NetBIOS.functions.bif.zeek <...>/Zeek_NetBIOS.functions.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_NoneWriter.none.bif.zeek <...>/Zeek_NoneWriter.none.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_PE.events.bif.zeek <...>/Zeek_PE.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_POP3.events.bif.zeek <...>/Zeek_POP3.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_RADIUS.events.bif.zeek <...>/Zeek_RADIUS.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_RDP.events.bif.zeek <...>/Zeek_RDP.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_RDP.types.bif.zeek <...>/Zeek_RDP.types.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_RFB.events.bif.zeek <...>/Zeek_RFB.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_RPC.events.bif.zeek <...>/Zeek_RPC.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_RawReader.raw.bif.zeek <...>/Zeek_RawReader.raw.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SIP.events.bif.zeek <...>/Zeek_SIP.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.consts.bif.zeek <...>/Zeek_SMB.consts.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.events.bif.zeek <...>/Zeek_SMB.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_com_check_directory.bif.zeek <...>/Zeek_SMB.smb1_com_check_directory.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_com_close.bif.zeek <...>/Zeek_SMB.smb1_com_close.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_com_create_directory.bif.zeek <...>/Zeek_SMB.smb1_com_create_directory.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_com_echo.bif.zeek <...>/Zeek_SMB.smb1_com_echo.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_com_logoff_andx.bif.zeek <...>/Zeek_SMB.smb1_com_logoff_andx.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_com_negotiate.bif.zeek <...>/Zeek_SMB.smb1_com_negotiate.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_com_nt_cancel.bif.zeek <...>/Zeek_SMB.smb1_com_nt_cancel.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_com_nt_create_andx.bif.zeek <...>/Zeek_SMB.smb1_com_nt_create_andx.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_com_query_information.bif.zeek <...>/Zeek_SMB.smb1_com_query_information.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_com_read_andx.bif.zeek <...>/Zeek_SMB.smb1_com_read_andx.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_com_session_setup_andx.bif.zeek <...>/Zeek_SMB.smb1_com_session_setup_andx.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_com_transaction.bif.zeek <...>/Zeek_SMB.smb1_com_transaction.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_com_transaction2.bif.zeek <...>/Zeek_SMB.smb1_com_transaction2.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_com_transaction2_secondary.bif.zeek <...>/Zeek_SMB.smb1_com_transaction2_secondary.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_com_transaction_secondary.bif.zeek <...>/Zeek_SMB.smb1_com_transaction_secondary.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_com_tree_connect_andx.bif.zeek <...>/Zeek_SMB.smb1_com_tree_connect_andx.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_com_tree_disconnect.bif.zeek <...>/Zeek_SMB.smb1_com_tree_disconnect.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_com_write_andx.bif.zeek <...>/Zeek_SMB.smb1_com_write_andx.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb1_events.bif.zeek <...>/Zeek_SMB.smb1_events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb2_com_close.bif.zeek <...>/Zeek_SMB.smb2_com_close.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb2_com_create.bif.zeek <...>/Zeek_SMB.smb2_com_create.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb2_com_negotiate.bif.zeek <...>/Zeek_SMB.smb2_com_negotiate.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb2_com_read.bif.zeek <...>/Zeek_SMB.smb2_com_read.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb2_com_session_setup.bif.zeek <...>/Zeek_SMB.smb2_com_session_setup.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb2_com_set_info.bif.zeek <...>/Zeek_SMB.smb2_com_set_info.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb2_com_transform_header.bif.zeek <...>/Zeek_SMB.smb2_com_transform_header.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb2_com_tree_connect.bif.zeek <...>/Zeek_SMB.smb2_com_tree_connect.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb2_com_tree_disconnect.bif.zeek <...>/Zeek_SMB.smb2_com_tree_disconnect.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb2_com_write.bif.zeek <...>/Zeek_SMB.smb2_com_write.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.smb2_events.bif.zeek <...>/Zeek_SMB.smb2_events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMB.types.bif.zeek <...>/Zeek_SMB.types.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMTP.events.bif.zeek <...>/Zeek_SMTP.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SMTP.functions.bif.zeek <...>/Zeek_SMTP.functions.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SNMP.events.bif.zeek <...>/Zeek_SNMP.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SNMP.types.bif.zeek <...>/Zeek_SNMP.types.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SOCKS.events.bif.zeek <...>/Zeek_SOCKS.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SQLiteReader.sqlite.bif.zeek <...>/Zeek_SQLiteReader.sqlite.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SQLiteWriter.sqlite.bif.zeek <...>/Zeek_SQLiteWriter.sqlite.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SSH.events.bif.zeek <...>/Zeek_SSH.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SSH.types.bif.zeek <...>/Zeek_SSH.types.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SSL.consts.bif.zeek <...>/Zeek_SSL.consts.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SSL.events.bif.zeek <...>/Zeek_SSL.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SSL.functions.bif.zeek <...>/Zeek_SSL.functions.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_SSL.types.bif.zeek <...>/Zeek_SSL.types.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_Syslog.events.bif.zeek <...>/Zeek_Syslog.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_TCP.events.bif.zeek <...>/Zeek_TCP.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_TCP.functions.bif.zeek <...>/Zeek_TCP.functions.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_TCP.types.bif.zeek <...>/Zeek_TCP.types.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_Teredo.events.bif.zeek <...>/Zeek_Teredo.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_UDP.events.bif.zeek <...>/Zeek_UDP.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_Unified2.events.bif.zeek <...>/Zeek_Unified2.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_Unified2.types.bif.zeek <...>/Zeek_Unified2.types.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_VXLAN.events.bif.zeek <...>/Zeek_VXLAN.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_X509.events.bif.zeek <...>/Zeek_X509.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_X509.functions.bif.zeek <...>/Zeek_X509.functions.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_X509.ocsp_events.bif.zeek <...>/Zeek_X509.ocsp_events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_X509.types.bif.zeek <...>/Zeek_X509.types.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_XMPP.events.bif.zeek <...>/Zeek_XMPP.events.bif.zeek +0.000000 | HookLoadFileExtended ./acld <...>/acld.zeek +0.000000 | HookLoadFileExtended ./addrs <...>/addrs.zeek +0.000000 | HookLoadFileExtended ./analyzer.bif.zeek <...>/analyzer.bif.zeek +0.000000 | HookLoadFileExtended ./api <...>/api.zeek +0.000000 | HookLoadFileExtended ./archive <...>/archive.sig +0.000000 | HookLoadFileExtended ./audio <...>/audio.sig +0.000000 | HookLoadFileExtended ./average <...>/average.zeek +0.000000 | HookLoadFileExtended ./bloom-filter.bif.zeek <...>/bloom-filter.bif.zeek +0.000000 | HookLoadFileExtended ./broker <...>/broker.zeek +0.000000 | HookLoadFileExtended ./cardinality-counter.bif.zeek <...>/cardinality-counter.bif.zeek +0.000000 | HookLoadFileExtended ./certificate-event-cache <...>/certificate-event-cache.zeek +0.000000 | HookLoadFileExtended ./comm.bif.zeek <...>/comm.bif.zeek +0.000000 | HookLoadFileExtended ./const-dos-error <...>/const-dos-error.zeek +0.000000 | HookLoadFileExtended ./const-nt-status <...>/const-nt-status.zeek +0.000000 | HookLoadFileExtended ./const.bif.zeek <...>/const.bif.zeek +0.000000 | HookLoadFileExtended ./consts <...>/consts.zeek +0.000000 | HookLoadFileExtended ./contents <...>/contents.zeek +0.000000 | HookLoadFileExtended ./control <...>/control.zeek +0.000000 | HookLoadFileExtended ./ct-list <...>/ct-list.zeek +0.000000 | HookLoadFileExtended ./data.bif.zeek <...>/data.bif.zeek +0.000000 | HookLoadFileExtended ./dcc-send <...>/dcc-send.zeek +0.000000 | HookLoadFileExtended ./debug <...>/debug.zeek +0.000000 | HookLoadFileExtended ./dpd.sig <...>/dpd.sig +0.000000 | HookLoadFileExtended ./drop <...>/drop.zeek +0.000000 | HookLoadFileExtended ./entities <...>/entities.zeek +0.000000 | HookLoadFileExtended ./event.bif.zeek <...>/event.bif.zeek +0.000000 | HookLoadFileExtended ./exec <...>/exec.zeek +0.000000 | HookLoadFileExtended ./executable <...>/executable.sig +0.000000 | HookLoadFileExtended ./file_analysis.bif.zeek <...>/file_analysis.bif.zeek +0.000000 | HookLoadFileExtended ./files <...>/files.zeek +0.000000 | HookLoadFileExtended ./font <...>/font.sig +0.000000 | HookLoadFileExtended ./general <...>/general.sig +0.000000 | HookLoadFileExtended ./gridftp <...>/gridftp.zeek +0.000000 | HookLoadFileExtended ./hll_unique <...>/hll_unique.zeek +0.000000 | HookLoadFileExtended ./hooks.bif.zeek <...>/hooks.bif.zeek +0.000000 | HookLoadFileExtended ./image <...>/image.sig +0.000000 | HookLoadFileExtended ./inactivity <...>/inactivity.zeek +0.000000 | HookLoadFileExtended ./info <...>/info.zeek +0.000000 | HookLoadFileExtended ./input <...>/input.zeek +0.000000 | HookLoadFileExtended ./input.bif.zeek <...>/input.bif.zeek +0.000000 | HookLoadFileExtended ./java <...>/java.sig +0.000000 | HookLoadFileExtended ./last <...>/last.zeek +0.000000 | HookLoadFileExtended ./libmagic <...>/libmagic.sig +0.000000 | HookLoadFileExtended ./log <...>/log.zeek +0.000000 | HookLoadFileExtended ./log-ocsp <...>/log-ocsp.zeek +0.000000 | HookLoadFileExtended ./logging.bif.zeek <...>/logging.bif.zeek +0.000000 | HookLoadFileExtended ./magic <...>/magic +0.000000 | HookLoadFileExtended ./main <...>/main.zeek +0.000000 | HookLoadFileExtended ./max <...>/max.zeek +0.000000 | HookLoadFileExtended ./messaging.bif.zeek <...>/messaging.bif.zeek +0.000000 | HookLoadFileExtended ./min <...>/min.zeek +0.000000 | HookLoadFileExtended ./mozilla-ca-list <...>/mozilla-ca-list.zeek +0.000000 | HookLoadFileExtended ./netstats <...>/netstats.zeek +0.000000 | HookLoadFileExtended ./non-cluster <...>/non-cluster.zeek +0.000000 | HookLoadFileExtended ./office <...>/office.sig +0.000000 | HookLoadFileExtended ./openflow <...>/openflow.zeek +0.000000 | HookLoadFileExtended ./option.bif.zeek <...>/option.bif.zeek +0.000000 | HookLoadFileExtended ./packet_analysis.bif.zeek <...>/packet_analysis.bif.zeek +0.000000 | HookLoadFileExtended ./packetfilter <...>/packetfilter.zeek +0.000000 | HookLoadFileExtended ./patterns <...>/patterns.zeek +0.000000 | HookLoadFileExtended ./pcap.bif.zeek <...>/pcap.bif.zeek +0.000000 | HookLoadFileExtended ./plugin <...>/plugin.zeek +0.000000 | HookLoadFileExtended ./plugins <...>/plugins +0.000000 | HookLoadFileExtended ./polling <...>/polling.zeek +0.000000 | HookLoadFileExtended ./pools <...>/pools.zeek +0.000000 | HookLoadFileExtended ./postprocessors <...>/postprocessors +0.000000 | HookLoadFileExtended ./programming <...>/programming.sig +0.000000 | HookLoadFileExtended ./removal-hooks <...>/removal-hooks.zeek +0.000000 | HookLoadFileExtended ./reporter.bif.zeek <...>/reporter.bif.zeek +0.000000 | HookLoadFileExtended ./ryu <...>/ryu.zeek +0.000000 | HookLoadFileExtended ./sample <...>/sample.zeek +0.000000 | HookLoadFileExtended ./scp <...>/scp.zeek +0.000000 | HookLoadFileExtended ./sftp <...>/sftp.zeek +0.000000 | HookLoadFileExtended ./shunt <...>/shunt.zeek +0.000000 | HookLoadFileExtended ./site <...>/site.zeek +0.000000 | HookLoadFileExtended ./smb1-main <...>/smb1-main.zeek +0.000000 | HookLoadFileExtended ./smb2-main <...>/smb2-main.zeek +0.000000 | HookLoadFileExtended ./stats.bif.zeek <...>/stats.bif.zeek +0.000000 | HookLoadFileExtended ./std-dev <...>/std-dev.zeek +0.000000 | HookLoadFileExtended ./store <...>/store.zeek +0.000000 | HookLoadFileExtended ./store.bif.zeek <...>/store.bif.zeek +0.000000 | HookLoadFileExtended ./strings.bif.zeek <...>/strings.bif.zeek +0.000000 | HookLoadFileExtended ./sum <...>/sum.zeek +0.000000 | HookLoadFileExtended ./supervisor.bif.zeek <...>/supervisor.bif.zeek +0.000000 | HookLoadFileExtended ./telemetry.bif.zeek <...>/telemetry.bif.zeek +0.000000 | HookLoadFileExtended ./thresholds <...>/thresholds.zeek +0.000000 | HookLoadFileExtended ./top-k.bif.zeek <...>/top-k.bif.zeek +0.000000 | HookLoadFileExtended ./topk <...>/topk.zeek +0.000000 | HookLoadFileExtended ./types <...>/types.zeek +0.000000 | HookLoadFileExtended ./types.bif.zeek <...>/types.bif.zeek +0.000000 | HookLoadFileExtended ./unique <...>/unique.zeek +0.000000 | HookLoadFileExtended ./utils <...>/utils.zeek +0.000000 | HookLoadFileExtended ./utils-commands <...>/utils-commands.zeek +0.000000 | HookLoadFileExtended ./variance <...>/variance.zeek +0.000000 | HookLoadFileExtended ./video <...>/video.sig +0.000000 | HookLoadFileExtended ./weird <...>/weird.zeek +0.000000 | HookLoadFileExtended ./zeek.bif.zeek <...>/zeek.bif.zeek +0.000000 | HookLoadFileExtended ./zeekygen.bif.zeek <...>/zeekygen.bif.zeek +0.000000 | HookLoadFileExtended .<...>/add-geodata <...>/add-geodata.zeek +0.000000 | HookLoadFileExtended .<...>/ascii <...>/ascii.zeek +0.000000 | HookLoadFileExtended .<...>/benchmark <...>/benchmark.zeek +0.000000 | HookLoadFileExtended .<...>/binary <...>/binary.zeek +0.000000 | HookLoadFileExtended .<...>/config <...>/config.zeek +0.000000 | HookLoadFileExtended .<...>/email_admin <...>/email_admin.zeek +0.000000 | HookLoadFileExtended .<...>/none <...>/none.zeek +0.000000 | HookLoadFileExtended .<...>/page <...>/page.zeek +0.000000 | HookLoadFileExtended .<...>/pp-alarms <...>/pp-alarms.zeek +0.000000 | HookLoadFileExtended .<...>/raw <...>/raw.zeek +0.000000 | HookLoadFileExtended .<...>/sqlite <...>/sqlite.zeek +0.000000 | HookLoadFileExtended <...>/__load__.zeek <...>/__load__.zeek +0.000000 | HookLoadFileExtended <...>/__preload__.zeek <...>/__preload__.zeek +0.000000 | HookLoadFileExtended <...>/hooks.zeek <...>/hooks.zeek +0.000000 | HookLoadFileExtended base/bif <...>/bif +0.000000 | HookLoadFileExtended base/init-default <...>/init-default.zeek +0.000000 | HookLoadFileExtended base/init-frameworks-and-bifs.zeek <...>/init-frameworks-and-bifs.zeek +0.000000 | HookLoadFileExtended base/packet-protocols <...>/packet-protocols +0.000000 | HookLoadFileExtended base<...>/CPP-load.bif <...>/CPP-load.bif.zeek +0.000000 | HookLoadFileExtended base<...>/Zeek_KRB.types.bif <...>/Zeek_KRB.types.bif.zeek +0.000000 | HookLoadFileExtended base<...>/Zeek_SNMP.types.bif <...>/Zeek_SNMP.types.bif.zeek +0.000000 | HookLoadFileExtended base<...>/active-http <...>/active-http.zeek +0.000000 | HookLoadFileExtended base<...>/addrs <...>/addrs.zeek +0.000000 | HookLoadFileExtended base<...>/analyzer <...>/analyzer +0.000000 | HookLoadFileExtended base<...>/analyzer.bif <...>/analyzer.bif.zeek +0.000000 | HookLoadFileExtended base<...>/api <...>/api.zeek +0.000000 | HookLoadFileExtended base<...>/backtrace <...>/backtrace.zeek +0.000000 | HookLoadFileExtended base<...>/broker <...>/broker +0.000000 | HookLoadFileExtended base<...>/cluster <...>/cluster +0.000000 | HookLoadFileExtended base<...>/comm.bif <...>/comm.bif.zeek +0.000000 | HookLoadFileExtended base<...>/config <...>/config +0.000000 | HookLoadFileExtended base<...>/conn <...>/conn +0.000000 | HookLoadFileExtended base<...>/conn-ids <...>/conn-ids.zeek +0.000000 | HookLoadFileExtended base<...>/const.bif <...>/const.bif.zeek +0.000000 | HookLoadFileExtended base<...>/control <...>/control +0.000000 | HookLoadFileExtended base<...>/data.bif <...>/data.bif.zeek +0.000000 | HookLoadFileExtended base<...>/dce-rpc <...>/dce-rpc +0.000000 | HookLoadFileExtended base<...>/dhcp <...>/dhcp +0.000000 | HookLoadFileExtended base<...>/dir <...>/dir.zeek +0.000000 | HookLoadFileExtended base<...>/directions-and-hosts <...>/directions-and-hosts.zeek +0.000000 | HookLoadFileExtended base<...>/dnp3 <...>/dnp3 +0.000000 | HookLoadFileExtended base<...>/dns <...>/dns +0.000000 | HookLoadFileExtended base<...>/dpd <...>/dpd +0.000000 | HookLoadFileExtended base<...>/email <...>/email.zeek +0.000000 | HookLoadFileExtended base<...>/ethernet <...>/ethernet +0.000000 | HookLoadFileExtended base<...>/event.bif <...>/event.bif.zeek +0.000000 | HookLoadFileExtended base<...>/exec <...>/exec.zeek +0.000000 | HookLoadFileExtended base<...>/extract <...>/extract +0.000000 | HookLoadFileExtended base<...>/fddi <...>/fddi +0.000000 | HookLoadFileExtended base<...>/file_analysis.bif <...>/file_analysis.bif.zeek +0.000000 | HookLoadFileExtended base<...>/files <...>/files +0.000000 | HookLoadFileExtended base<...>/files <...>/files.zeek +0.000000 | HookLoadFileExtended base<...>/find-checksum-offloading <...>/find-checksum-offloading.zeek +0.000000 | HookLoadFileExtended base<...>/find-filtered-trace <...>/find-filtered-trace.zeek +0.000000 | HookLoadFileExtended base<...>/ftp <...>/ftp +0.000000 | HookLoadFileExtended base<...>/geoip-distance <...>/geoip-distance.zeek +0.000000 | HookLoadFileExtended base<...>/gre <...>/gre +0.000000 | HookLoadFileExtended base<...>/hash <...>/hash +0.000000 | HookLoadFileExtended base<...>/hash_hrw <...>/hash_hrw.zeek +0.000000 | HookLoadFileExtended base<...>/http <...>/http +0.000000 | HookLoadFileExtended base<...>/icmp <...>/icmp +0.000000 | HookLoadFileExtended base<...>/ieee802_11 <...>/ieee802_11 +0.000000 | HookLoadFileExtended base<...>/ieee802_11_radio <...>/ieee802_11_radio +0.000000 | HookLoadFileExtended base<...>/imap <...>/imap +0.000000 | HookLoadFileExtended base<...>/input <...>/input +0.000000 | HookLoadFileExtended base<...>/input.bif <...>/input.bif.zeek +0.000000 | HookLoadFileExtended base<...>/intel <...>/intel +0.000000 | HookLoadFileExtended base<...>/ip <...>/ip +0.000000 | HookLoadFileExtended base<...>/iptunnel <...>/iptunnel +0.000000 | HookLoadFileExtended base<...>/irc <...>/irc +0.000000 | HookLoadFileExtended base<...>/krb <...>/krb +0.000000 | HookLoadFileExtended base<...>/linux_sll <...>/linux_sll +0.000000 | HookLoadFileExtended base<...>/logging <...>/logging +0.000000 | HookLoadFileExtended base<...>/logging.bif <...>/logging.bif.zeek +0.000000 | HookLoadFileExtended base<...>/main <...>/main.zeek +0.000000 | HookLoadFileExtended base<...>/messaging.bif <...>/messaging.bif.zeek +0.000000 | HookLoadFileExtended base<...>/modbus <...>/modbus +0.000000 | HookLoadFileExtended base<...>/mpls <...>/mpls +0.000000 | HookLoadFileExtended base<...>/mqtt <...>/mqtt +0.000000 | HookLoadFileExtended base<...>/mysql <...>/mysql +0.000000 | HookLoadFileExtended base<...>/netcontrol <...>/netcontrol +0.000000 | HookLoadFileExtended base<...>/nflog <...>/nflog +0.000000 | HookLoadFileExtended base<...>/notice <...>/notice +0.000000 | HookLoadFileExtended base<...>/ntlm <...>/ntlm +0.000000 | HookLoadFileExtended base<...>/ntp <...>/ntp +0.000000 | HookLoadFileExtended base<...>/null <...>/null +0.000000 | HookLoadFileExtended base<...>/numbers <...>/numbers.zeek +0.000000 | HookLoadFileExtended base<...>/openflow <...>/openflow +0.000000 | HookLoadFileExtended base<...>/option.bif <...>/option.bif.zeek +0.000000 | HookLoadFileExtended base<...>/packet-filter <...>/packet-filter +0.000000 | HookLoadFileExtended base<...>/packet_analysis.bif <...>/packet_analysis.bif.zeek +0.000000 | HookLoadFileExtended base<...>/paths <...>/paths.zeek +0.000000 | HookLoadFileExtended base<...>/patterns <...>/patterns.zeek +0.000000 | HookLoadFileExtended base<...>/pe <...>/pe +0.000000 | HookLoadFileExtended base<...>/plugins <...>/plugins +0.000000 | HookLoadFileExtended base<...>/pop3 <...>/pop3 +0.000000 | HookLoadFileExtended base<...>/ppp_serial <...>/ppp_serial +0.000000 | HookLoadFileExtended base<...>/pppoe <...>/pppoe +0.000000 | HookLoadFileExtended base<...>/queue <...>/queue.zeek +0.000000 | HookLoadFileExtended base<...>/radius <...>/radius +0.000000 | HookLoadFileExtended base<...>/rdp <...>/rdp +0.000000 | HookLoadFileExtended base<...>/removal-hooks <...>/removal-hooks.zeek +0.000000 | HookLoadFileExtended base<...>/reporter <...>/reporter +0.000000 | HookLoadFileExtended base<...>/reporter.bif <...>/reporter.bif.zeek +0.000000 | HookLoadFileExtended base<...>/rfb <...>/rfb +0.000000 | HookLoadFileExtended base<...>/root <...>/root +0.000000 | HookLoadFileExtended base<...>/signatures <...>/signatures +0.000000 | HookLoadFileExtended base<...>/sip <...>/sip +0.000000 | HookLoadFileExtended base<...>/site <...>/site.zeek +0.000000 | HookLoadFileExtended base<...>/skip <...>/skip +0.000000 | HookLoadFileExtended base<...>/smb <...>/smb +0.000000 | HookLoadFileExtended base<...>/smtp <...>/smtp +0.000000 | HookLoadFileExtended base<...>/snmp <...>/snmp +0.000000 | HookLoadFileExtended base<...>/socks <...>/socks +0.000000 | HookLoadFileExtended base<...>/software <...>/software +0.000000 | HookLoadFileExtended base<...>/ssh <...>/ssh +0.000000 | HookLoadFileExtended base<...>/ssl <...>/ssl +0.000000 | HookLoadFileExtended base<...>/stats.bif <...>/stats.bif.zeek +0.000000 | HookLoadFileExtended base<...>/store.bif <...>/store.bif.zeek +0.000000 | HookLoadFileExtended base<...>/strings <...>/strings.zeek +0.000000 | HookLoadFileExtended base<...>/strings.bif <...>/strings.bif.zeek +0.000000 | HookLoadFileExtended base<...>/sumstats <...>/sumstats +0.000000 | HookLoadFileExtended base<...>/supervisor <...>/supervisor +0.000000 | HookLoadFileExtended base<...>/supervisor.bif <...>/supervisor.bif.zeek +0.000000 | HookLoadFileExtended base<...>/syslog <...>/syslog +0.000000 | HookLoadFileExtended base<...>/tcp <...>/tcp +0.000000 | HookLoadFileExtended base<...>/thresholds <...>/thresholds.zeek +0.000000 | HookLoadFileExtended base<...>/time <...>/time.zeek +0.000000 | HookLoadFileExtended base<...>/tunnels <...>/tunnels +0.000000 | HookLoadFileExtended base<...>/types.bif <...>/types.bif.zeek +0.000000 | HookLoadFileExtended base<...>/udp <...>/udp +0.000000 | HookLoadFileExtended base<...>/urls <...>/urls.zeek +0.000000 | HookLoadFileExtended base<...>/utils <...>/utils.zeek +0.000000 | HookLoadFileExtended base<...>/version <...>/version.zeek +0.000000 | HookLoadFileExtended base<...>/vlan <...>/vlan +0.000000 | HookLoadFileExtended base<...>/vntag <...>/vntag +0.000000 | HookLoadFileExtended base<...>/weird <...>/weird.zeek +0.000000 | HookLoadFileExtended base<...>/x509 <...>/x509 +0.000000 | HookLoadFileExtended base<...>/xmpp <...>/xmpp +0.000000 | HookLoadFileExtended base<...>/zeek.bif <...>/zeek.bif.zeek +0.000000 | HookLoadFileExtended builtin-plugins/__load__.zeek <...>/__load__.zeek +0.000000 | HookLoadFileExtended builtin-plugins/__preload__.zeek <...>/__preload__.zeek +0.000000 | HookLoadFileExtended s1.sig ./s1.sig +0.000000 | HookLoadFileExtended s2 ./s2.sig 0.000000 | HookLogInit packet_filter 1/1 {ts (time), node (string), filter (string), init (bool), success (bool)} 0.000000 | HookLogWrite packet_filter [ts=XXXXXXXXXX.XXXXXX, node=zeek, filter=ip or not ip, init=T, success=T] 0.000000 | HookQueueEvent NetControl::init() diff --git a/testing/btest/Baseline/plugins.plugin-load-file-extended/output b/testing/btest/Baseline/plugins.plugin-load-file-extended/output new file mode 100644 index 0000000000..d32f2141d1 --- /dev/null +++ b/testing/btest/Baseline/plugins.plugin-load-file-extended/output @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +HookLoadExtended/script: file=|xxx| resolved=|./xxx.zeek| +HookLoadExtended/script: file=|yyy| resolved=|| +HookLoadExtended/signature: file=|abc.sig| resolved=|./abc.sig| +new zeek_init(): script has been replaced +new zeek_init(): script has been added +signature works! diff --git a/testing/btest/plugins/hooks-plugin/src/Plugin.cc b/testing/btest/plugins/hooks-plugin/src/Plugin.cc index 852d9656bf..1fd4054f38 100644 --- a/testing/btest/plugins/hooks-plugin/src/Plugin.cc +++ b/testing/btest/plugins/hooks-plugin/src/Plugin.cc @@ -15,6 +15,7 @@ using namespace btest::plugin::Demo_Hooks; zeek::plugin::Configuration Plugin::Configure() { EnableHook(zeek::plugin::HOOK_LOAD_FILE); + EnableHook(zeek::plugin::HOOK_LOAD_FILE_EXT); EnableHook(zeek::plugin::HOOK_CALL_FUNCTION); EnableHook(zeek::plugin::HOOK_QUEUE_EVENT); EnableHook(zeek::plugin::HOOK_DRAIN_EVENTS); @@ -56,6 +57,13 @@ int Plugin::HookLoadFile(const LoadType type, const std::string& file, const std return -1; } +std::pair> Plugin::HookLoadFileExtended(const LoadType type, const std::string& file, const std::string& resolved) + { + fprintf(stderr, "%.6f %-15s %s %s\n", zeek::run_state::network_time, "| HookLoadFileExtended", + file.c_str(), resolved.c_str()); + return std::make_pair(-1, std::nullopt); + } + std::pair Plugin::HookFunctionCall(const zeek::Func* func, zeek::detail::Frame* frame, zeek::Args* args) { diff --git a/testing/btest/plugins/hooks-plugin/src/Plugin.h b/testing/btest/plugins/hooks-plugin/src/Plugin.h index 94a7f16aee..b721608a6f 100644 --- a/testing/btest/plugins/hooks-plugin/src/Plugin.h +++ b/testing/btest/plugins/hooks-plugin/src/Plugin.h @@ -9,6 +9,7 @@ class Plugin : public zeek::plugin::Plugin { protected: int HookLoadFile(const LoadType type, const std::string& file, const std::string& resolved) override; + std::pair> HookLoadFileExtended(const LoadType type, const std::string& file, const std::string& resolved) override; std::pair HookFunctionCall(const zeek::Func* func, zeek::detail::Frame* parent, zeek::Args* args) override; bool HookQueueEvent(zeek::Event* event) override; diff --git a/testing/btest/plugins/plugin-load-file-extended.zeek b/testing/btest/plugins/plugin-load-file-extended.zeek new file mode 100644 index 0000000000..0d3d297c5a --- /dev/null +++ b/testing/btest/plugins/plugin-load-file-extended.zeek @@ -0,0 +1,17 @@ +# @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Testing LoadFileExtended +# @TEST-EXEC: cp -r %DIR/plugin-load-file-extended/* . +# @TEST-EXEC: ./configure --zeek-dist=${DIST} && make +# @TEST-EXEC: ZEEK_PLUGIN_PATH=$(pwd) zeek -r $TRACES/wikipedia.trace -b Testing::LoadFileExtended xxx yyy -s abc.sig >> output +# @TEST-EXEC: btest-diff output + +# @TEST-START-FILE xxx.zeek + +event zeek_init() { + print "original script"; +} + +# @TEST-END-FILE + +# @TEST-START-FILE abc.sig +# empty +# @TEST-END-FILE diff --git a/testing/btest/plugins/plugin-load-file-extended/.btest-ignore b/testing/btest/plugins/plugin-load-file-extended/.btest-ignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/plugins/plugin-load-file-extended/src/Plugin.cc b/testing/btest/plugins/plugin-load-file-extended/src/Plugin.cc new file mode 100644 index 0000000000..a86d4f50f2 --- /dev/null +++ b/testing/btest/plugins/plugin-load-file-extended/src/Plugin.cc @@ -0,0 +1,70 @@ + +#include "Plugin.h" + +namespace btest::plugin::Testing_LoadFileExtended + { +Plugin plugin; + } + +using namespace btest::plugin::Testing_LoadFileExtended; + +zeek::plugin::Configuration Plugin::Configure() + { + EnableHook(zeek::plugin::HOOK_LOAD_FILE_EXT); + + zeek::plugin::Configuration config; + config.name = "Testing::LoadFileExtended"; + config.version.major = 0; + config.version.minor = 1; + config.version.patch = 4; + return config; + } + +#include + +std::pair> Plugin::HookLoadFileExtended(const LoadType type, + const std::string& file, + const std::string& resolved) + { + if ( type == LoadType::SCRIPT && file == "xxx" ) + { + printf("HookLoadExtended/script: file=|%s| resolved=|%s|\n", file.c_str(), resolved.c_str()); + + return std::make_pair(1, R"( + event zeek_init() { + print "new zeek_init(): script has been replaced"; + } + + event signature_match(state: signature_state, msg: string, data: string) { + print msg; + } + )"); + } + + if ( type == LoadType::SCRIPT && file == "yyy" ) + { + printf("HookLoadExtended/script: file=|%s| resolved=|%s|\n", file.c_str(), resolved.c_str()); + + return std::make_pair(1, R"( + event zeek_init() { + print "new zeek_init(): script has been added"; + } + )"); + } + + if ( type == LoadType::SIGNATURES && file == "abc.sig" ) + { + printf("HookLoadExtended/signature: file=|%s| resolved=|%s|\n", file.c_str(), resolved.c_str()); + + return std::make_pair(1, R"( + signature my-sig { + ip-proto == tcp + payload /GET \/images/ + event "signature works!" + } + )"); + } + + return std::make_pair(-1, std::nullopt); + } + diff --git a/testing/btest/plugins/plugin-load-file-extended/src/Plugin.h b/testing/btest/plugins/plugin-load-file-extended/src/Plugin.h new file mode 100644 index 0000000000..83137bce6f --- /dev/null +++ b/testing/btest/plugins/plugin-load-file-extended/src/Plugin.h @@ -0,0 +1,18 @@ + +#pragma once + +#include + +namespace btest::plugin::Testing_LoadFileExtended { + +class Plugin : public zeek::plugin::Plugin +{ +protected: + // Overridden from zeek::plugin::Plugin. + zeek::plugin::Configuration Configure() override; + std::pair> HookLoadFileExtended(const Plugin::LoadType type, const std::string& file, const std::string& resolved) override; +}; + +extern Plugin plugin; + +} From c39f33527a306d08356b3ca3a0a92f7c1f2ee2b7 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 5 Nov 2021 12:49:24 -0700 Subject: [PATCH 035/210] Update broker and zeek-archiver submodules [nomail] [skip ci] --- auxil/broker | 2 +- auxil/zeek-archiver | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/auxil/broker b/auxil/broker index 47cac80cbe..32d97ec752 160000 --- a/auxil/broker +++ b/auxil/broker @@ -1 +1 @@ -Subproject commit 47cac80cbe1e1bde8e3b425903e50d62715972a2 +Subproject commit 32d97ec7522951a28aeb521120cc512acd47e324 diff --git a/auxil/zeek-archiver b/auxil/zeek-archiver index f3a1e8fe46..479e8a85fd 160000 --- a/auxil/zeek-archiver +++ b/auxil/zeek-archiver @@ -1 +1 @@ -Subproject commit f3a1e8fe464c0425688eff67e30f35c678914ad2 +Subproject commit 479e8a85fd58936c16d361dbf3de4e7268d751f8 From 1a6da54aedcf2afc52b8f61fa592e7f78657cbe0 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 5 Nov 2021 13:40:04 -0700 Subject: [PATCH 036/210] Update broker submodule [nomail] [skip ci] --- auxil/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auxil/broker b/auxil/broker index 32d97ec752..07842d8bcd 160000 --- a/auxil/broker +++ b/auxil/broker @@ -1 +1 @@ -Subproject commit 32d97ec7522951a28aeb521120cc512acd47e324 +Subproject commit 07842d8bcdd4cf90fb7355be53fa3d14146147dd From f2ea56379bdbe1e8ef2504fb42dccd0ed7116fab Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 3 Nov 2021 11:15:50 -0700 Subject: [PATCH 037/210] Remove no-op false-teredo test --- .../known_services.log | 4 +- .../btest/Traces/tunnels/false-teredo.pcap | Bin 3098 -> 0 bytes testing/btest/core/tunnels/false-teredo.zeek | 47 ------------------ .../core/tunnels/teredo-known-services.test | 2 +- 4 files changed, 4 insertions(+), 49 deletions(-) delete mode 100644 testing/btest/Traces/tunnels/false-teredo.pcap delete mode 100644 testing/btest/core/tunnels/false-teredo.zeek diff --git a/testing/btest/Baseline/core.tunnels.teredo-known-services/known_services.log b/testing/btest/Baseline/core.tunnels.teredo-known-services/known_services.log index 140fed3fc9..d7565fafdf 100644 --- a/testing/btest/Baseline/core.tunnels.teredo-known-services/known_services.log +++ b/testing/btest/Baseline/core.tunnels.teredo-known-services/known_services.log @@ -7,5 +7,7 @@ #open XXXX-XX-XX-XX-XX-XX #fields ts host port_num port_proto service #types time addr port enum set[string] -XXXXXXXXXX.XXXXXX 192.168.1.1 53 udp DNS +XXXXXXXXXX.XXXXXX 192.168.2.1 53 udp DNS +XXXXXXXXXX.XXXXXX 192.168.2.16 1577 tcp (empty) +XXXXXXXXXX.XXXXXX 192.168.2.16 1576 tcp (empty) #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Traces/tunnels/false-teredo.pcap b/testing/btest/Traces/tunnels/false-teredo.pcap deleted file mode 100644 index e82a6f41693b06517d460a0f976cbed141f9e8fb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3098 zcmd6pe`r%z6vyv-xlLkiV`IB%WtO28hO}auCKj!;^oNtCLtFF&=(wfB$;N*C;fihybY)04n2zCG9nqchUXqvgvHUrf za-iqj`|kIi&pG$LcWvVA37$}r|2=v{2nWt0KMj<7>|`tak49>zZojykIFl{hIVb5N zq|0`QkYV>ny=;{02jgz-AU@)`oenL9=Wz-6Q6V0ZTtg{QOZF>rTu=waq%EYxaRRqz z;8F%%O6qpCT^BBVT@S+I=G&a}19huZ& z=z0E|LLu)uLOY-N_86-+^0vK792C;_uxbf5(6Nt@ZqP=vq}Wpa08>0g@+oFFLW=bV z>b+xTikv_rT39ZmcB}4aAFdpL)@p@mXmeBJRy9Aw)W+>*YT1q4l0!$lV|r>fG7p+& z&dQKd8J5mfMHUy0ImB>O927G)Sxgf4lriV|9i)beEo+*%Xt^*b#D#vG#n!Q&Tzk|I zy2Czoifu8lIGn_8lUt&!cXnj8CE~M0nkmWUT3WdaR!&e-N2&G|oU z9;`Bs=D-E4QjZ^4O2|{MlIMx@QWjt#eJnskQIbI+7UNTc4pmGgMO7=5?BtQguTiGq zri1(rlhKGSP;@k>_E%uZBAvOCNyG<2-(*in+1$~z5x%5MzJ8;O)XfXUQW|W+o|KBT|~3G*3Mf~wGo-|kkij^Zk59@bG5sjB{HG%*&_BtcDx?8%~W zs4G0No1HM$tdR-5@t*J&8!`Rfdn@^LmM}HOqntP+S2Lt0$I?cl#mo-VpJ6pQlU}(+{LF+CUvfo zl)*D-7J&CZ`56JWZZ!dd&{Q8)rV0RQNtt}3@_mG7RjkTr6!2)`(DgEp5vcQUaA+3!|O_Sp&U-QB;4mv#1V%MyD@nLK;ABVAG|jIq9tVFNYd31{@#|;A z3yQNN&6bGI67g6fO_qqCwU6U_9X@s5A0l+qbeC(q+Vp$6kH5*jH2t<^XO6QUb)UfA zV7?S&cTB(Q^k46cuQ2_#WvBQ2dVvPkw!5so5_;j9@j~H!*emDoyCaH)oc)*JgIY)NTj zt|g6M_><7<-`||heM)i5A15wHG#PJ{Ga;MsauoW~ X_{hSM`A!^xS5kyMizwXx^F96p8SMi; diff --git a/testing/btest/core/tunnels/false-teredo.zeek b/testing/btest/core/tunnels/false-teredo.zeek deleted file mode 100644 index 818b543d95..0000000000 --- a/testing/btest/core/tunnels/false-teredo.zeek +++ /dev/null @@ -1,47 +0,0 @@ -# @TEST-EXEC: zeek -r $TRACES/tunnels/false-teredo.pcap %INPUT >output -# @TEST-EXEC: test ! -e weird.log -# @TEST-EXEC: test ! -e dpd.log - -# In the first case, there isn't any weird or protocol violation logged -# since the teredo analyzer recognizes that the DNS analyzer has confirmed -# the protocol and yields. - -# In the second case, there are weirds since the teredo analyzer decapsulates -# despite the presence of the confirmed DNS analyzer and the resulting -# inner packets are malformed (no surprise there). There's also no dpd.log -# since the teredo analyzer doesn't confirm until it's seen a valid teredo -# encapsulation in both directions and protocol violations aren't logged -# until there's been a confirmation. - -# In either case, the analyzer doesn't, by default, get disabled as a result -# of the protocol violations. - -function print_teredo(name: string, outer: connection, inner: teredo_hdr) - { - print fmt("%s: %s", name, outer$id); - print fmt(" ip6: %s", inner$hdr$ip6); - if ( inner?$auth ) - print fmt(" auth: %s", inner$auth); - if ( inner?$origin ) - print fmt(" origin: %s", inner$origin); - } - -event teredo_packet(outer: connection, inner: teredo_hdr) - { - print_teredo("packet", outer, inner); - } - -event teredo_authentication(outer: connection, inner: teredo_hdr) - { - print_teredo("auth", outer, inner); - } - -event teredo_origin_indication(outer: connection, inner: teredo_hdr) - { - print_teredo("origin", outer, inner); - } - -event teredo_bubble(outer: connection, inner: teredo_hdr) - { - print_teredo("bubble", outer, inner); - } diff --git a/testing/btest/core/tunnels/teredo-known-services.test b/testing/btest/core/tunnels/teredo-known-services.test index 8e0a09862b..c5a687527c 100644 --- a/testing/btest/core/tunnels/teredo-known-services.test +++ b/testing/btest/core/tunnels/teredo-known-services.test @@ -1,4 +1,4 @@ -# @TEST-EXEC: zeek -b -r $TRACES/tunnels/false-teredo.pcap base/frameworks/dpd base/protocols/tunnels base/protocols/dns protocols/conn/known-services Tunnel::delay_teredo_confirmation=T "Site::local_nets+={192.168.1.0/24}" +# @TEST-EXEC: zeek -b -r $TRACES/tunnels/Teredo.pcap base/frameworks/dpd base/protocols/tunnels base/protocols/dns protocols/conn/known-services Tunnel::delay_teredo_confirmation=T "Site::local_nets+={192.168.2.0/24}" # @TEST-EXEC: btest-diff known_services.log # Expect known_services.log to NOT indicate any service using teredo. From 14abfc68315faae7cd00bae3aa12654ad7f0afa0 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Sun, 7 Nov 2021 16:56:57 -0800 Subject: [PATCH 038/210] removing unused SubNetType class --- src/Type.cc | 22 ---------------------- src/Type.h | 12 ------------ 2 files changed, 34 deletions(-) diff --git a/src/Type.cc b/src/Type.cc index 80a6e9c678..b03e3b8b7d 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -123,18 +123,6 @@ RecordType* Type::AsRecordType() return (RecordType*)this; } -const SubNetType* Type::AsSubNetType() const - { - CHECK_TYPE_TAG(TYPE_SUBNET, "Type::AsSubNetType"); - return (const SubNetType*)this; - } - -SubNetType* Type::AsSubNetType() - { - CHECK_TYPE_TAG(TYPE_SUBNET, "Type::AsSubNetType"); - return (SubNetType*)this; - } - const FuncType* Type::AsFuncType() const { CHECK_TYPE_TAG(TYPE_FUNC, "Type::AsFuncType"); @@ -1442,16 +1430,6 @@ string RecordType::GetFieldDeprecationWarning(int field, bool has_check) const return ""; } -SubNetType::SubNetType() : Type(TYPE_SUBNET) { } - -void SubNetType::Describe(ODesc* d) const - { - if ( d->IsReadable() ) - d->Add("subnet"); - else - d->Add(int(Tag())); - } - FileType::FileType(TypePtr yield_type) : Type(TYPE_FILE), yield(std::move(yield_type)) { } FileType::~FileType() = default; diff --git a/src/Type.h b/src/Type.h index e81ed9eec9..cc6b6c9106 100644 --- a/src/Type.h +++ b/src/Type.h @@ -152,7 +152,6 @@ class TypeList; class TableType; class SetType; class RecordType; -class SubNetType; class FuncType; class EnumType; class VectorType; @@ -165,7 +164,6 @@ using TypeListPtr = IntrusivePtr; using TableTypePtr = IntrusivePtr; using SetTypePtr = IntrusivePtr; using RecordTypePtr = IntrusivePtr; -using SubNetTypePtr = IntrusivePtr; using FuncTypePtr = IntrusivePtr; using EnumTypePtr = IntrusivePtr; using VectorTypePtr = IntrusivePtr; @@ -226,9 +224,6 @@ public: const RecordType* AsRecordType() const; RecordType* AsRecordType(); - const SubNetType* AsSubNetType() const; - SubNetType* AsSubNetType(); - const FuncType* AsFuncType() const; FuncType* AsFuncType(); @@ -700,13 +695,6 @@ protected: type_decl_list* types; }; -class SubNetType final : public Type - { -public: - SubNetType(); - void Describe(ODesc* d) const override; - }; - class FileType final : public Type { public: From bc3bf4ea6ca53024799d2a37deb2f12a841194a9 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Sun, 7 Nov 2021 16:57:52 -0800 Subject: [PATCH 039/210] factoring out CPPEscape to be a standalone function --- src/script_opt/CPP/Util.cc | 56 ++++++++++++++++++++++++++++++++++++++ src/script_opt/CPP/Util.h | 8 ++++++ 2 files changed, 64 insertions(+) diff --git a/src/script_opt/CPP/Util.cc b/src/script_opt/CPP/Util.cc index c2a6fed195..1c64c9af21 100644 --- a/src/script_opt/CPP/Util.cc +++ b/src/script_opt/CPP/Util.cc @@ -75,4 +75,60 @@ void unlock_file(const string& fname, FILE* f) } } +string CPPEscape(const char* b, int len) + { + string res; + + for ( int i = 0; i < len; ++i ) + { + unsigned char c = b[i]; + + switch ( c ) + { + case '\a': + res += "\\a"; + break; + case '\b': + res += "\\b"; + break; + case '\f': + res += "\\f"; + break; + case '\n': + res += "\\n"; + break; + case '\r': + res += "\\r"; + break; + case '\t': + res += "\\t"; + break; + case '\v': + res += "\\v"; + break; + + case '\\': + res += "\\\\"; + break; + case '"': + res += "\\\""; + break; + + default: + if ( isprint(c) ) + res += c; + else + { + char buf[8192]; + snprintf(buf, sizeof buf, "%03o", c); + res += "\\"; + res += buf; + } + break; + } + } + + return res; + } + } // zeek::detail diff --git a/src/script_opt/CPP/Util.h b/src/script_opt/CPP/Util.h index 6ea06e6752..23e2533922 100644 --- a/src/script_opt/CPP/Util.h +++ b/src/script_opt/CPP/Util.h @@ -36,4 +36,12 @@ extern bool is_CPP_compilable(const ProfileFunc* pf, const char** reason = nullp extern void lock_file(const std::string& fname, FILE* f); extern void unlock_file(const std::string& fname, FILE* f); +// For the given byte array / string, returns a version expanded +// with escape sequences in order to represent it as a C++ string. +extern std::string CPPEscape(const char* b, int len); +inline std::string CPPEscape(const char* s) + { + return CPPEscape(s, strlen(s)); + } + } // zeek::detail From e1a760e6747e6b14f5fddaacdf5ecc6b7f3d762b Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Sun, 7 Nov 2021 17:00:19 -0800 Subject: [PATCH 040/210] extensive rewrite of generation & execution of run-time initialization --- src/CMakeLists.txt | 4 +- src/script_opt/CPP/Attrs.cc | 136 +--- src/script_opt/CPP/Attrs.h | 19 + src/script_opt/CPP/Compile.h | 687 +++++++++-------- src/script_opt/CPP/Consts.cc | 284 ++----- src/script_opt/CPP/DeclFunc.cc | 271 ++++--- src/script_opt/CPP/Driver.cc | 298 +++++--- src/script_opt/CPP/Emit.cc | 65 +- src/script_opt/CPP/Exprs.cc | 13 +- src/script_opt/CPP/GenFunc.cc | 16 +- src/script_opt/CPP/HashMgr.cc | 21 +- src/script_opt/CPP/HashMgr.h | 8 +- src/script_opt/CPP/Inits.cc | 426 ++--------- src/script_opt/CPP/InitsInfo.cc | 577 +++++++++++++++ src/script_opt/CPP/InitsInfo.h | 693 ++++++++++++++++++ src/script_opt/CPP/Runtime.h | 7 +- .../{RuntimeInit.cc => RuntimeInitSupport.cc} | 30 +- .../{RuntimeInit.h => RuntimeInitSupport.h} | 1 + src/script_opt/CPP/RuntimeInits.cc | 528 +++++++++++++ src/script_opt/CPP/RuntimeInits.h | 542 ++++++++++++++ src/script_opt/CPP/Stmts.cc | 2 +- src/script_opt/CPP/Tracker.cc | 14 +- src/script_opt/CPP/Tracker.h | 21 +- src/script_opt/CPP/Types.cc | 278 +------ src/script_opt/CPP/Vars.cc | 78 +- src/script_opt/ScriptOpt.cc | 20 +- 26 files changed, 3459 insertions(+), 1580 deletions(-) create mode 100644 src/script_opt/CPP/Attrs.h create mode 100644 src/script_opt/CPP/InitsInfo.cc create mode 100644 src/script_opt/CPP/InitsInfo.h rename src/script_opt/CPP/{RuntimeInit.cc => RuntimeInitSupport.cc} (89%) rename src/script_opt/CPP/{RuntimeInit.h => RuntimeInitSupport.h} (98%) create mode 100644 src/script_opt/CPP/RuntimeInits.cc create mode 100644 src/script_opt/CPP/RuntimeInits.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c0f520fbeb..3646f8fa16 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -401,7 +401,9 @@ set(MAIN_SRCS script_opt/CPP/GenFunc.cc script_opt/CPP/HashMgr.cc script_opt/CPP/Inits.cc - script_opt/CPP/RuntimeInit.cc + script_opt/CPP/InitsInfo.cc + script_opt/CPP/RuntimeInits.cc + script_opt/CPP/RuntimeInitSupport.cc script_opt/CPP/RuntimeOps.cc script_opt/CPP/RuntimeVec.cc script_opt/CPP/Stmts.cc diff --git a/src/script_opt/CPP/Attrs.cc b/src/script_opt/CPP/Attrs.cc index b1d12978ac..6c2594be2d 100644 --- a/src/script_opt/CPP/Attrs.cc +++ b/src/script_opt/CPP/Attrs.cc @@ -7,42 +7,53 @@ namespace zeek::detail using namespace std; -void CPPCompile::RegisterAttributes(const AttributesPtr& attrs) +shared_ptr CPPCompile::RegisterAttributes(const AttributesPtr& attrs) { - if ( ! attrs || attributes.HasKey(attrs) ) - return; + if ( ! attrs ) + return nullptr; + + auto a = attrs.get(); + + if ( processed_attrs.count(a) > 0 ) + return processed_attrs[a]; attributes.AddKey(attrs); - AddInit(attrs); - auto a_rep = attributes.GetRep(attrs); - if ( a_rep != attrs.get() ) + // The cast is just so we can make an IntrusivePtr. + auto a_rep = const_cast(attributes.GetRep(attrs)); + if ( a_rep != a ) { - NoteInitDependency(attrs.get(), a_rep); - return; + AttributesPtr a_rep_ptr = {NewRef{}, a_rep}; + processed_attrs[a] = RegisterAttributes(a_rep_ptr); + return processed_attrs[a]; } for ( const auto& a : attrs->GetAttrs() ) - { - const auto& e = a->GetExpr(); - if ( e ) - { - if ( IsSimpleInitExpr(e) ) - { - // Make sure any dependencies it has get noted. - (void)GenExpr(e, GEN_VAL_PTR); - continue; - } + (void)RegisterAttr(a); - init_exprs.AddKey(e); - AddInit(e); - NoteInitDependency(attrs, e); + shared_ptr gi = make_shared(this, attrs); + attrs_info->AddInstance(gi); + processed_attrs[a] = gi; - auto e_rep = init_exprs.GetRep(e); - if ( e_rep != e.get() ) - NoteInitDependency(e.get(), e_rep); - } - } + return gi; + } + +shared_ptr CPPCompile::RegisterAttr(const AttrPtr& attr) + { + auto a = attr.get(); + + if ( processed_attr.count(a) > 0 ) + return processed_attr[a]; + + const auto& e = a->GetExpr(); + if ( e && ! IsSimpleInitExpr(e) ) + init_exprs.AddKey(e); + + auto gi = make_shared(this, attr); + attr_info->AddInstance(gi); + processed_attr[a] = gi; + + return gi; } void CPPCompile::BuildAttrs(const AttributesPtr& attrs, string& attr_tags, string& attr_vals) @@ -72,78 +83,9 @@ void CPPCompile::BuildAttrs(const AttributesPtr& attrs, string& attr_tags, strin attr_vals = string("{") + attr_vals + "}"; } -void CPPCompile::GenAttrs(const AttributesPtr& attrs) +const char* CPPCompile::AttrName(AttrTag t) { - NL(); - - Emit("AttributesPtr %s", AttrsName(attrs)); - - StartBlock(); - - const auto& avec = attrs->GetAttrs(); - Emit("auto attrs = std::vector();"); - - AddInit(attrs); - - for ( const auto& attr : avec ) - { - const auto& e = attr->GetExpr(); - - if ( ! e ) - { - Emit("attrs.emplace_back(make_intrusive(%s));", AttrName(attr)); - continue; - } - - NoteInitDependency(attrs, e); - AddInit(e); - - string e_arg; - if ( IsSimpleInitExpr(e) ) - e_arg = GenAttrExpr(e); - else - e_arg = InitExprName(e); - - Emit("attrs.emplace_back(make_intrusive(%s, %s));", AttrName(attr), e_arg); - } - - Emit("return make_intrusive(attrs, nullptr, true, false);"); - - EndBlock(); - } - -string CPPCompile::GenAttrExpr(const ExprPtr& e) - { - switch ( e->Tag() ) - { - case EXPR_CONST: - return string("make_intrusive(") + GenExpr(e, GEN_VAL_PTR) + ")"; - - case EXPR_NAME: - NoteInitDependency(e, e->AsNameExpr()->IdPtr()); - return string("make_intrusive(") + globals[e->AsNameExpr()->Id()->Name()] + - ")"; - - case EXPR_RECORD_COERCE: - NoteInitDependency(e, TypeRep(e->GetType())); - return string("make_intrusive(make_intrusive(" - "make_intrusive()), cast_intrusive(") + - GenTypeName(e->GetType()) + "))"; - - default: - reporter->InternalError("bad expr tag in CPPCompile::GenAttrs"); - return "###"; - } - } - -string CPPCompile::AttrsName(const AttributesPtr& a) - { - return attributes.KeyName(a) + "()"; - } - -const char* CPPCompile::AttrName(const AttrPtr& attr) - { - switch ( attr->Tag() ) + switch ( t ) { case ATTR_OPTIONAL: return "ATTR_OPTIONAL"; diff --git a/src/script_opt/CPP/Attrs.h b/src/script_opt/CPP/Attrs.h new file mode 100644 index 0000000000..8a399341e7 --- /dev/null +++ b/src/script_opt/CPP/Attrs.h @@ -0,0 +1,19 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +// Definitions associated with type attributes. + +#pragma once + +namespace zeek::detail + { + +enum AttrExprType + { + AE_NONE, // attribute doesn't have an expression + AE_CONST, // easy expression - a constant (ConstExpr) + AE_NAME, // easy - a global (NameExpr) + AE_RECORD, // an empty record cast to a given type + AE_CALL, // everything else - requires a lambda, essentially + }; + + } // zeek::detail diff --git a/src/script_opt/CPP/Compile.h b/src/script_opt/CPP/Compile.h index 5331d2a9a6..7974c9206a 100644 --- a/src/script_opt/CPP/Compile.h +++ b/src/script_opt/CPP/Compile.h @@ -5,18 +5,20 @@ #include "zeek/Desc.h" #include "zeek/script_opt/CPP/Func.h" #include "zeek/script_opt/CPP/HashMgr.h" +#include "zeek/script_opt/CPP/InitsInfo.h" #include "zeek/script_opt/CPP/Tracker.h" #include "zeek/script_opt/CPP/Util.h" #include "zeek/script_opt/ScriptOpt.h" // We structure the compiler for generating C++ versions of Zeek script -// bodies as a single large class. While we divide the compiler's +// bodies maily as a single large class. While we divide the compiler's // functionality into a number of groups (see below), these interact with // one another, and in particular with various member variables, enough // so that it's not clear there's benefit to further splitting the // functionality into multiple classes. (Some splitting has already been // done for more self-contained functionality, resulting in the CPPTracker -// and CPPHashManager classes.) +// and CPPHashManager classes, and initialization information in +// InitsInfo.{h,cc} and RuntimeInits.{h,cc}.) // // Most aspects of translating to C++ have a straightforward nature. // We can turn many Zeek script statements directly into the C++ that's @@ -45,26 +47,6 @@ // all of the scripts loaded in "bare" mode, plus those for foo.zeek; and // without the "-b" for all of the default scripts plus those in foo.zeek. // -// One of the design goals employed is to support "incremental" compilation, -// i.e., compiling *additional* Zeek scripts at a later point after an -// initial compilation. This comes in two forms. -// -// "-O update-C++" produces C++ code that extends that already compiled, -// in a manner where subsequent compilations can leverage both the original -// and the newly added. Such compilations *must* be done in a consistent -// context (for example, any types extended in the original are extended in -// the same manner - plus then perhaps further extensions - in the updated -// code). -// -// "-O add-C++" instead produces C++ code that (1) will not be leveraged in -// any subsequent compilations, and (2) can be inconsistent with other -// "-O add-C++" code added in the future. The main use of this feature is -// to support compiling polyglot versions of Zeek scripts used to run -// the test suite. -// -// Zeek invocations specifying "-O use-C++" will activate any code compiled -// into the zeek binary; otherwise, the code lies dormant. -// // "-O report-C++" reports on which compiled functions will/won't be used // (including ones that are available but not relevant to the scripts loaded // on the command line). This can be useful when debugging to make sure @@ -104,29 +86,41 @@ // // Emit Low-level code generation. // -// Of these, Inits is probably the most subtle. It turns out to be -// very tricky ensuring that we create run-time variables in the -// proper order. For example, a global might need a record type to be -// defined; one of the record's fields is a table; that table contains -// another record; one of that other record's fields is the original -// record (recursion); another field has an &default expression that -// requires the compiler to generate a helper function to construct -// the expression dynamically; and that helper function might in turn -// refer to other types that require initialization. +// Of these, Inits is the most subtle and complex. There are two major +// challenges in creating run-time values (such as Zeek types and constants). // -// To deal with these dependencies, for every run-time object the compiler -// maintains (1) all of the other run-time objects on which its initialization -// depends, and (2) the C++ statements needed to initialize it, once those -// other objects have been initialized. It then beings initialization with -// objects that have no dependencies, marks those as done (essentially), finds -// objects that now can be initialized and emits their initializations, -// marks those as done, etc. +// First, generating individual code for creating each of these winds up +// incurring unacceptable compile times (for example, clang compiling all +// of the base scripts with optimization takes many hours on a high-end +// laptop). As a result, we employ a table-driven approach that compiles +// much faster (though still taking many minutes on the same high-end laptop, +// running about 40x faster however). // -// Below in declaring the CPPCompiler class, we group methods in accordance -// with those listed above. We also locate member variables with the group -// most relevant for their usage. However, keep in mind that many member -// variables are used by multiple groups, which is why we haven't created -// distinct per-group classes. +// Second, initializations frequently rely upon *other* initializations +// having occurred first. For example, a global might need a record type +// to be defined; one of the record's fields is a table; that table contains +// another record; one of that other record's fields is the original record +// (recursion); another field has an &default expression that requires the +// compiler to generate a helper function to construct the expression +// dynamically; and that helper function might in turn refer to other types +// that require initialization. What's required is a framework for ensuring +// that everything occurs in the proper order. +// +// The logic for dealing with these complexities is isolated into several +// sets of classes. InitsInfo.{h,cc} provides the classes related to tracking +// how to generate initializations in the proper order. RuntimeInits.{h,cc} +// provides the classes used when initialization generated code in order +// to instantiate all of the necessary values. See those files for discussions +// on how they address the points framed above. +// +// In declaring the CPPCompiler class, we group methods in accordance with +// those listed above, locating member variables with the group most relevant +// for their usage. However, keep in mind that many member variables are +// used by multiple groups, which is why we haven't created distinct +// per-group classes. In addition, we make a number of methods public +// in order to avoid the need for numerous "friend" declarations to allow +// associated classes (like those for initialization) access to a the +// necessary compiler methods. namespace zeek::detail { @@ -135,10 +129,124 @@ class CPPCompile { public: CPPCompile(std::vector& _funcs, ProfileFuncs& pfs, const std::string& gen_name, - const std::string& addl_name, CPPHashManager& _hm, bool _update, bool _standalone, + const std::string& addl_name, CPPHashManager& _hm, bool _standalone, bool report_uncompilable); ~CPPCompile(); + // Constructing a CPPCompile object does all of the compilation. + // The public methods here are for use by helper classes. + + // Tracks the given type (with support methods for ones that + // are complicated), recursively including its sub-types, and + // creating initializations for constructing C++ variables + // representing the types. + // + // Returns the initialization info associated with the type. + std::shared_ptr RegisterType(const TypePtr& t); + + // Easy access to the global offset and the initialization + // cohort associated with a given type. + int TypeOffset(const TypePtr& t) { return GI_Offset(RegisterType(t)); } + int TypeCohort(const TypePtr& t) { return GI_Cohort(RegisterType(t)); } + + // Tracks a Zeek ValPtr used as a constant value. These occur + // in two contexts: directly as constant expressions, and indirectly + // as elements within aggregate constants (such as in vector + // initializers). + // + // Returns the associated initialization info. In addition, + // consts_offset returns an offset into an initialization-time + // global that tracks all constructed globals, providing + // general access to them for aggregate constants. + std::shared_ptr RegisterConstant(const ValPtr& vp, int& consts_offset); + + // Tracks a global to generate the necessary initialization. + // Returns the associated initialization info. + std::shared_ptr RegisterGlobal(const ID* g); + + // Tracks a use of the given set of attributes, including + // initialization dependencies and the generation of any + // associated expressions. + // + // Returns the initialization info associated with the set of + // attributes. + std::shared_ptr RegisterAttributes(const AttributesPtr& attrs); + + // Convenient access to the global offset associated with + // a set of Attributes. + int AttributesOffset(const AttributesPtr& attrs) + { + return GI_Offset(RegisterAttributes(attrs)); + } + + // The same, for a single attribute. + std::shared_ptr RegisterAttr(const AttrPtr& attr); + int AttrOffset(const AttrPtr& attr) { return GI_Offset(RegisterAttr(attr)); } + + // Returns a mapping of from Attr objects to their associated + // initialization information. The Attr must have previously + // been registered. + auto ProcessedAttr() { return processed_attr; } + + // True if the given expression is simple enough that we can + // generate code to evaluate it directly, and don't need to + // create a separate function per RegisterInitExpr() to track it. + static bool IsSimpleInitExpr(const ExprPtr& e); + + // Tracks expressions used in attributes (such as &default=). + // + // We need to generate code to evaluate these, via CallExpr's + // that invoke functions that return the value of the expression. + // However, we can't generate that code when first encountering + // the attribute, because doing so will need to refer to the names + // of types, and initially those are unavailable (because the type's + // representatives, per pfs.RepTypes(), might not have yet been + // tracked). So instead we track the associated CallExprInitInfo + // objects, and after all types have been tracked, then spin + // through them to generate the code. + // + // Returns the associated initialization information. + std::shared_ptr RegisterInitExpr(const ExprPtr& e); + + // Tracks a C++ string value needed for initialization. Returns + // an offset into the global vector that will hold these. + int TrackString(std::string s) + { + if ( tracked_strings.count(s) == 0 ) + { + tracked_strings[s] = ordered_tracked_strings.size(); + ordered_tracked_strings.emplace_back(s); + } + + return tracked_strings[s]; + } + + // Tracks a profile hash value needed for initialization. Returns + // an offset into the global vector that will hold these. + int TrackHash(p_hash_type h) + { + if ( tracked_hashes.count(h) == 0 ) + { + tracked_hashes[h] = ordered_tracked_hashes.size(); + ordered_tracked_hashes.emplace_back(h); + } + + return tracked_hashes[h]; + } + + // Returns the hash associated with a given function body. + // It's a fatal error to call this for a body that hasn't + // been compiled. + p_hash_type BodyHash(const Stmt* body); + + // Returns true if at least one of the function bodies associated + // with the function/hook/event handler of the given fname is + // not compilable. + bool NotFullyCompilable(const std::string& fname) const + { + return not_fully_compilable.count(fname) > 0; + } + private: // Start of methods related to driving the overall compilation // process. @@ -148,6 +256,37 @@ private: // Main driver, invoked by constructor. void Compile(bool report_uncompilable); + // The following methods all create objects that track the + // initializations of a given type of value. In each, "tag" + // is the name used to identify the initializer global + // associated with the given type of value, and "type" is + // its C++ representation. Often "tag" is concatenated with + // "type" to designate a specific C++ type. For example, + // "tag" might be "Double" and "type" might be "ValPtr"; + // the resulting global's type is "DoubleValPtr". + + // Creates an object for tracking values associated with Zeek + // constants. "c_type" is the C++ type used in the initializer + // for each object; or, if empty, it specifies that we represent + // the value using an index into a separate vector that holds + // the constant. + std::shared_ptr CreateConstInitInfo(const char* tag, const char* type, + const char* c_type); + + // Creates an object for tracking compound initializers, which + // are whose initialization uses indexes into other vectors. + std::shared_ptr CreateCompoundInitInfo(const char* tag, const char* type); + + // Creates an object for tracking initializers that have custom + // C++ objects to hold their initialization information. + std::shared_ptr CreateCustomInitInfo(const char* tag, const char* type); + + // Generates the declaration associated with a set of initializations + // and tracks the object to facilitate looping over all so + // initializations. As a convenience, returns the object. + std::shared_ptr RegisterInitInfo(const char* tag, const char* type, + std::shared_ptr gi); + // Generate the beginning of the compiled code: run-time functions, // namespace, auxiliary globals. void GenProlog(); @@ -158,7 +297,7 @@ private: void RegisterCompiledBody(const std::string& f); // After compilation, generate the final code. Most of this is - // run-time initialization of various dynamic values. + // in support of run-time initialization of various dynamic values. void GenEpilog(); // True if the given function (plus body and profile) is one @@ -185,9 +324,13 @@ private: // it including some functionality we don't currently support // for compilation. // - // Indexed by the name of the function. + // Indexed by the C++ name of the function. std::unordered_set compilable_funcs; + // Tracks which functions/hooks/events have at least one non-compilable + // body. Indexed by the Zeek name of function. + std::unordered_set not_fully_compilable; + // Maps functions (not hooks or events) to upstream compiled names. std::unordered_map hashed_funcs; @@ -200,10 +343,6 @@ private: // compilation units. int addl_tag = 0; - // If true, then we're updating the C++ base (i.e., generating - // code meant for use by subsequently generated code). - bool update = false; - // If true, the generated code should run "standalone". bool standalone = false; @@ -211,7 +350,7 @@ private: // needed for "seatbelts", to ensure that we can produce a // unique hash relating to this compilation (*and* its // compilation time, which is why these are "seatbelts" and - // likely not important to make distinct. + // likely not important to make distinct). p_hash_type total_hash = 0; // Working directory in which we're compiling. Used to quasi-locate @@ -236,11 +375,6 @@ private: // track it as such. void CreateGlobal(const ID* g); - // For the globals used in the compilation, if new then append - // them to the hash file to make the information available - // to subsequent compilation runs. - void UpdateGlobalHashes(); - // Register the given identifier as a BiF. If is_var is true // then the BiF is also used in a non-call context. void AddBiF(const ID* b, bool is_var); @@ -258,10 +392,9 @@ private: // The following match various forms of identifiers to the // name used for their C++ equivalent. - const char* IDName(const ID& id) { return IDName(&id); } const char* IDName(const IDPtr& id) { return IDName(id.get()); } const char* IDName(const ID* id) { return IDNameStr(id).c_str(); } - const std::string& IDNameStr(const ID* id) const; + const std::string& IDNameStr(const ID* id); // Returns a canonicalized version of a variant of a global made // distinct by the given suffix. @@ -280,12 +413,20 @@ private: // conflict with C++ keywords. std::string Canonicalize(const char* name) const; + // Returns the name of the global corresponding to an expression + // (which must be a EXPR_NAME). + std::string GlobalName(const ExprPtr& e) { return globals[e->AsNameExpr()->Id()->Name()]; } + // Maps global names (not identifiers) to the names we use for them. std::unordered_map globals; // Similar for locals, for the function currently being compiled. std::unordered_map locals; + // Retrieves the initialization information associated with the + // given global. + std::unordered_map> global_gis; + // Maps event names to the names we use for them. std::unordered_map events; @@ -307,14 +448,37 @@ private: // Similar, but for lambdas. void DeclareLambda(const LambdaExpr* l, const ProfileFunc* pf); - // Declares the CPPStmt subclass used for compiling the given + // Generates code to declare the compiled version of a script // function. "ft" gives the functions type, "pf" its profile, // "fname" its C++ name, "body" its AST, "l" if non-nil its // corresponding lambda expression, and "flavor" whether it's // a hook/event/function. + // + // We use two basic approaches. Most functions are represented + // by a "CPPDynStmt" object that's parameterized by a void* pointer + // to the underlying C++ function and an index used to dynamically + // cast the pointer to having the correct type for then calling it. + // Lambdas, however (including "implicit" lambdas used to associate + // complex expressions with &attributes), each have a unique + // subclass derived from CPPStmt that calls the underlying C++ + // function without requiring a cast, and that holds the values + // of the lambda's captures. + // + // It would be cleanest to use the latter approach for all functions, + // but the hundreds/thousands of additional classes required for + // doing so significantly slows down C++ compilation, so we instead + // opt for the uglier dynamic casting approach, which only requires + // one additional class. + void CreateFunction(const FuncTypePtr& ft, const ProfileFunc* pf, const std::string& fname, + const StmtPtr& body, int priority, const LambdaExpr* l, + FunctionFlavor flavor); + + // Used for the case of creating a custom subclass of CPPStmt. void DeclareSubclass(const FuncTypePtr& ft, const ProfileFunc* pf, const std::string& fname, - const StmtPtr& body, int priority, const LambdaExpr* l, - FunctionFlavor flavor); + const std::string& args, const IDPList* lambda_ids); + + // Used for the case of employing an instance of a CPPDynStmt object. + void DeclareDynCPPStmt(); // Generates the declarations (and in-line definitions) associated // with compiling a lambda. @@ -331,11 +495,40 @@ private: // the given type, lambda captures (if non-nil), and profile. std::string ParamDecl(const FuncTypePtr& ft, const IDPList* lambda_ids, const ProfileFunc* pf); + // Returns in p_types the types associated with the parameters for a function + // of the given type, set of lambda captures (if any), and profile. + void GatherParamTypes(std::vector& p_types, const FuncTypePtr& ft, + const IDPList* lambda_ids, const ProfileFunc* pf); + + // Same, but instead returns the parameter's names. + void GatherParamNames(std::vector& p_names, const FuncTypePtr& ft, + const IDPList* lambda_ids, const ProfileFunc* pf); + // Inspects the given profile to find the i'th parameter (starting // at 0). Returns nil if the profile indicates that that parameter // is not used by the function. const ID* FindParam(int i, const ProfileFunc* pf); + // Information associated with a CPPDynStmt dynamic dispatch. + struct DispatchInfo + { + std::string cast; // C++ cast to use for function pointer + std::string args; // arguments to pass to the function + bool is_hook; // whether the function is a hook + TypePtr yield; // what type the function returns, if any + }; + + // An array of cast/invocation pairs used to generate the CPPDynStmt + // Exec method. + std::vector func_casting_glue; + + // Maps casting strings to indices into func_casting_glue. The index + // is what's used to dynamically switch to the right dispatch. + std::unordered_map casting_index; + + // Maps functions (using their C++ name) to their casting strings. + std::unordered_map func_index; + // Names for lambda capture ID's. These require a separate space // that incorporates the lambda's name, to deal with nested lambda's // that refer to the identifiers with the same name. @@ -344,7 +537,7 @@ private: // The function's parameters. Tracked so we don't re-declare them. std::unordered_set params; - // Whether we're parsing a hook. + // Whether we're compiling a hook. bool in_hook = false; // @@ -362,8 +555,12 @@ private: void CompileLambda(const LambdaExpr* l, const ProfileFunc* pf); // Generates the body of the Invoke() method (which supplies the - // "glue" between for calling the C++-generated code). - void GenInvokeBody(const std::string& fname, const TypePtr& t, const std::string& args); + // "glue" for calling the C++-generated code, for CPPStmt subclasses). + void GenInvokeBody(const std::string& fname, const TypePtr& t, const std::string& args) + { + GenInvokeBody(fname + "(" + args + ")", t); + } + void GenInvokeBody(const std::string& call, const TypePtr& t); // Generates the code for the body of a script function with // the given type, profile, C++ name, AST, lambda captures @@ -405,9 +602,6 @@ private: // Maps function bodies to the names we use for them. std::unordered_map body_names; - // Reverse mapping. - std::unordered_map names_to_bodies; - // Maps function names to hashes of bodies. std::unordered_map body_hashes; @@ -426,62 +620,84 @@ private: // // End of methods related to generating compiled script bodies. - // Start of methods related to generating code for representing - // script constants as run-time values. - // See Consts.cc for definitions. - // + // Methods related to generating code for representing script constants + // as run-time values. There's only one nontrivial one of these, + // RegisterConstant() (declared above, as it's public). All the other + // work is done by secondary objects - see InitsInfo.{h,cc} for those. - // Returns an instantiation of a constant - either as a native - // C++ constant, or as a C++ variable that will be bound to - // a Zeek value at run-time initialization - that is needed - // by the given "parent" object (which acquires an initialization - // dependency, if a C++ variable is needed). - std::string BuildConstant(IntrusivePtr parent, const ValPtr& vp) - { - return BuildConstant(parent.get(), vp); - } - std::string BuildConstant(const Obj* parent, const ValPtr& vp); + // Returns the object used to track indices (vectors of integers + // that are used to index various other vectors, including other + // indices). Only used by CPP_InitsInfo objects, but stored + // in the CPPCompile object to make it available across different + // CPP_InitsInfo objects. - // Called to create a constant appropriate for the given expression - // or, more directly, the given value. The second method returns - // "true" if a C++ variable needed to be created to construct the - // constant at run-time initialization, false if can be instantiated - // directly as a C++ constant. - void AddConstant(const ConstExpr* c); - bool AddConstant(const ValPtr& v); - - // Build particular types of C++ variables (with the given name) - // to hold constants initialized at run-time. - void AddStringConstant(const ValPtr& v, std::string& const_name); - void AddPatternConstant(const ValPtr& v, std::string& const_name); - void AddListConstant(const ValPtr& v, std::string& const_name); - void AddRecordConstant(const ValPtr& v, std::string& const_name); - void AddTableConstant(const ValPtr& v, std::string& const_name); - void AddVectorConstant(const ValPtr& v, std::string& const_name); + friend class CPP_InitsInfo; + IndicesManager& IndMgr() { return indices_mgr; } // Maps (non-native) constants to associated C++ globals. std::unordered_map const_exprs; - // Maps the values of (non-native) constants to associated C++ globals. - std::unordered_map const_vals; + // Maps the values of (non-native) constants to associated initializer + // information. + std::unordered_map> const_vals; + + // Same, but for the offset into the vector that tracks all constants + // collectively (to support initialization of compound constants). + std::unordered_map const_offsets; + + // The same as the above pair, but indexed by the string representation + // rather than the Val*. The reason for having both is to enable + // reusing common constants even though their Val*'s differ. + std::unordered_map> constants; + std::unordered_map constants_offsets; // Used for memory management associated with const_vals's index. std::vector cv_indices; - // Maps string representations of (non-native) constants to - // associated C++ globals. - std::unordered_map constants; + // For different types of constants (as indicated by TypeTag), + // provides the associated object that manages the initializers + // for those constants. + std::unordered_map> const_info; - // Maps the same representations to the Val* associated with their - // original creation. This enables us to construct initialization - // dependencies for later Val*'s that are able to reuse the same - // constant. - std::unordered_map constants_to_vals; + // Tracks entries for constructing the vector of all constants + // (regardless of type). Each entry provides a TypeTag, used + // to identify the type-specific vector for a given constant, + // and the offset into that vector. + std::vector> consts; - // Function variables that we need to create dynamically for - // initializing globals, coupled with the name of their associated - // constant. - std::unordered_map func_vars; + // The following objects track initialization information for + // different types of initializers: Zeek types, individual + // attributes, sets of attributes, expressions that call script + // functions (for attribute expressions), registering lambda + // bodies, and registering Zeek globals. + std::shared_ptr type_info; + std::shared_ptr attr_info; + std::shared_ptr attrs_info; + std::shared_ptr call_exprs_info; + std::shared_ptr lambda_reg_info; + std::shared_ptr global_id_info; + + // Tracks all of the above objects (as well as each entry in + // const_info), to facilitate easy iterating over them. + std::set> all_global_info; + + // Tracks the attribute expressions for which we need to generate + // function calls to evaluate them. + std::unordered_map> init_infos; + + // See IndMgr() above for the role of this variable. + IndicesManager indices_mgr; + + // Maps strings to associated offsets. + std::unordered_map tracked_strings; + + // Tracks strings we've registered in order (corresponding to + // their offsets). + std::vector ordered_tracked_strings; + + // The same as the previous two, but for profile hashes. + std::vector ordered_tracked_hashes; + std::unordered_map tracked_hashes; // // End of methods related to generating code for script constants. @@ -649,9 +865,9 @@ private: // not the outer map). int num_rf_mappings = 0; - // For each entry in "field_mapping", the record and TypeDecl - // associated with the mapping. - std::vector> field_decls; + // For each entry in "field_mapping", the record (as a global + // offset) and TypeDecl associated with the mapping. + std::vector> field_decls; // For enums that are extended via redef's, maps each distinct // value (that the compiled scripts refer to) to locations in the @@ -665,9 +881,9 @@ private: // not the outer map). int num_ev_mappings = 0; - // For each entry in "enum_mapping", the record and name - // associated with the mapping. - std::vector> enum_names; + // For each entry in "enum_mapping", the EnumType (as a global + // offset) and name associated with the mapping. + std::vector> enum_names; // // End of methods related to generating code for AST Expr's. @@ -690,24 +906,6 @@ private: // given script type 't', converts it as needed to the given GenType. std::string GenericValPtrToGT(const std::string& expr, const TypePtr& t, GenType gt); - // For a given type, generates the code necessary to initialize - // it at run time. The term "expand" in the method's name refers - // to the fact that the type has already been previously declared - // (necessary to facilitate defining recursive types), so this method - // generates the "meat" of the type but not its original declaration. - void ExpandTypeVar(const TypePtr& t); - - // Methods for expanding specific such types. "tn" is the name - // of the C++ variable used for the particular type. - void ExpandListTypeVar(const TypePtr& t, std::string& tn); - void ExpandRecordTypeVar(const TypePtr& t, std::string& tn); - void ExpandEnumTypeVar(const TypePtr& t, std::string& tn); - void ExpandTableTypeVar(const TypePtr& t, std::string& tn); - void ExpandFuncTypeVar(const TypePtr& t, std::string& tn); - - // The following assumes we're populating a type_decl_list called "tl". - std::string GenTypeDecl(const TypeDecl* td); - // Returns the name of a C++ variable that will hold a TypePtr // of the appropriate flavor. 't' does not need to be a type // representative. @@ -721,21 +919,11 @@ private: const Type* TypeRep(const TypePtr& t) { return TypeRep(t.get()); } // Low-level C++ representations for types, of various flavors. - const char* TypeTagName(TypeTag tag) const; + static const char* TypeTagName(TypeTag tag); const char* TypeName(const TypePtr& t); const char* FullTypeName(const TypePtr& t); const char* TypeType(const TypePtr& t); - // Track the given type (with support methods for onces that - // are complicated), recursively including its sub-types, and - // creating initializations (and dependencies) for constructing - // C++ variables representing the types. - void RegisterType(const TypePtr& t); - void RegisterListType(const TypePtr& t); - void RegisterTableType(const TypePtr& t); - void RegisterRecordType(const TypePtr& t); - void RegisterFuncType(const TypePtr& t); - // Access to a type's underlying values. const char* NativeAccessor(const TypePtr& t); @@ -744,11 +932,13 @@ private: const char* IntrusiveVal(const TypePtr& t); // Maps types to indices in the global "types__CPP" array. - CPPTracker types = {"types", &compiled_items}; + CPPTracker types = {"types", true, &compiled_items}; // Used to prevent analysis of mutually-referring types from - // leading to infinite recursion. - std::unordered_set processed_types; + // leading to infinite recursion. Maps types to their global + // initialization information (or, initially, to nullptr, if + // they're in the process of being registered). + std::unordered_map> processed_types; // // End of methods related to managing script types. @@ -758,11 +948,6 @@ private: // See Attrs.cc for definitions. // - // Tracks a use of the given set of attributes, including - // initialization dependencies and the generation of any - // associated expressions. - void RegisterAttributes(const AttributesPtr& attrs); - // Populates the 2nd and 3rd arguments with C++ representations // of the tags and (optional) values/expressions associated with // the set of attributes. @@ -772,16 +957,17 @@ private: void GenAttrs(const AttributesPtr& attrs); std::string GenAttrExpr(const ExprPtr& e); - // Returns the name of the C++ variable that will hold the given - // attributes at run-time. - std::string AttrsName(const AttributesPtr& attrs); - // Returns a string representation of the name associated with - // different attributes (e.g., "ATTR_DEFAULT"). - const char* AttrName(const AttrPtr& attr); + // different attribute tags (e.g., "ATTR_DEFAULT"). + static const char* AttrName(AttrTag t); // Similar for attributes, so we can reconstruct record types. - CPPTracker attributes = {"attrs", &compiled_items}; + CPPTracker attributes = {"attrs", false, &compiled_items}; + + // Maps Attributes and Attr's to their global initialization + // information. + std::unordered_map> processed_attrs; + std::unordered_map> processed_attr; // // End of methods related to managing script type attributes. @@ -790,121 +976,42 @@ private: // See Inits.cc for definitions. // - // Generates code to construct a CallExpr that can be used to - // evaluate the expression 'e' as an initializer (typically - // for a record &default attribute). - void GenInitExpr(const ExprPtr& e); - - // True if the given expression is simple enough that we can - // generate code to evaluate it directly, and don't need to - // create a separate function per GenInitExpr(). - bool IsSimpleInitExpr(const ExprPtr& e) const; + // Generates code for dynamically generating an expression + // associated with an attribute, via a function call. + void GenInitExpr(std::shared_ptr ce_init); // Returns the name of a function used to evaluate an // initialization expression. std::string InitExprName(const ExprPtr& e); - // Generates code to initializes the global 'g' (with C++ name "gl") - // to the given value *if* on start-up it doesn't already have a value. - void GenGlobalInit(const ID* g, std::string& gl, const ValPtr& v); - - // Generates code to initialize all of the function-valued globals - // (i.e., those pointing to lambdas). - void GenFuncVarInits(); - - // Generates the "pre-initialization" for a given type. For - // extensible types (records, enums, lists), these are empty - // versions that we'll later populate. - void GenPreInit(const Type* t); - - // Generates a function that executes the pre-initializations. - void GenPreInits(); - - // The following all track that for a given object, code associated - // with initializing it. Multiple calls for the same object append - // additional lines of code (the order of the calls is preserved). - // - // Versions with "lhs" and "rhs" arguments provide an initialization - // of the form "lhs = rhs;", as a convenience. - void AddInit(const IntrusivePtr& o, const std::string& lhs, const std::string& rhs) + // Convenience functions for return the offset or initialization cohort + // associated with an initialization. + int GI_Offset(const std::shared_ptr& gi) const { return gi ? gi->Offset() : -1; } + int GI_Cohort(const std::shared_ptr& gi) const { - AddInit(o.get(), lhs + " = " + rhs + ";"); - } - void AddInit(const Obj* o, const std::string& lhs, const std::string& rhs) - { - AddInit(o, lhs + " = " + rhs + ";"); - } - void AddInit(const IntrusivePtr& o, const std::string& init) { AddInit(o.get(), init); } - void AddInit(const Obj* o, const std::string& init); - - // We do consistency checking of initialization dependencies by - // looking for depended-on objects have initializations. Sometimes - // it's unclear whether the object will actually require - // initialization, in which case we add an empty initialization - // for it so that the consistency-checking is happy. - void AddInit(const IntrusivePtr& o) { AddInit(o.get()); } - void AddInit(const Obj* o); - - // This is akin to an initialization, but done separately - // (upon "activation") so it can include initializations that - // rely on parsing having finished (in particular, BiFs having - // been registered). Only used when generating standalone code. - void AddActivation(std::string a) { activations.emplace_back(a); } - - // Records the fact that the initialization of object o1 depends - // on that of object o2. - void NoteInitDependency(const IntrusivePtr& o1, const IntrusivePtr& o2) - { - NoteInitDependency(o1.get(), o2.get()); - } - void NoteInitDependency(const IntrusivePtr& o1, const Obj* o2) - { - NoteInitDependency(o1.get(), o2); - } - void NoteInitDependency(const Obj* o1, const IntrusivePtr& o2) - { - NoteInitDependency(o1, o2.get()); - } - void NoteInitDependency(const Obj* o1, const Obj* o2); - - // Records an initialization dependency of the given object - // on the given type, unless the type is a record. We need - // this notion to protect against circular dependencies in - // the face of recursive records. - void NoteNonRecordInitDependency(const Obj* o, const TypePtr& t) - { - if ( t && t->Tag() != TYPE_RECORD ) - NoteInitDependency(o, TypeRep(t)); - } - void NoteNonRecordInitDependency(const IntrusivePtr o, const TypePtr& t) - { - NoteNonRecordInitDependency(o.get(), t); + return gi ? gi->InitCohort() : 0; } - // Analyzes the initialization dependencies to ensure that they're - // consistent, i.e., every object that either depends on another, - // or is itself depended on, appears in the "to_do" set. - void CheckInitConsistency(std::unordered_set& to_do); - - // Generate initializations for the items in the "to_do" set, - // in accordance with their dependencies. Returns 'n', the - // number of initialization functions generated. They should - // be called in order, from 1 to n. - int GenDependentInits(std::unordered_set& to_do); - - // Generates a function for initializing the nc'th cohort. - void GenInitCohort(int nc, std::unordered_set& cohort); - - // Initialize the mappings for record field offsets for field - // accesses into regions of records that can be extensible (and - // thus can vary at run-time to the offsets encountered during - // compilation). + // Generate code to initialize the mappings for record field + // offsets for field accesses into regions of records that + // can be extensible (and thus can vary at run-time to the + // offsets encountered during compilation). void InitializeFieldMappings(); - // Same, but for enum types. The second form does a single - // initialization corresponding to the given index in the mapping. + // Same, but for enum types. void InitializeEnumMappings(); - void InitializeEnumMappings(const EnumType* et, const std::string& e_name, int index); + + // Generate code to initialize BiFs. + void InitializeBiFs(); + + // Generate code to initialize strings that we track. + void InitializeStrings(); + + // Generate code to initialize hashes that we track. + void InitializeHashes(); + + // Generate code to initialize indirect references to constants. + void InitializeConsts(); // Generate the initialization hook for this set of compiled code. void GenInitHook(); @@ -917,25 +1024,15 @@ private: // what we compiled. void GenLoad(); - // A list of pre-initializations (those potentially required by - // other initializations, and that themselves have no dependencies). - std::vector pre_inits; - - // A list of "activations" (essentially, post-initializations). - // See AddActivation() above. - std::vector activations; + // A list of BiFs to look up during initialization. First + // string is the name of the C++ global holding the BiF, the + // second is its name as known to Zeek. + std::unordered_map BiFs; // Expressions for which we need to generate initialization-time // code. Currently, these are only expressions appearing in // attributes. - CPPTracker init_exprs = {"gen_init_expr", &compiled_items}; - - // Maps an object requiring initialization to its initializers. - std::unordered_map> obj_inits; - - // Maps an object requiring initializations to its dependencies - // on other such objects. - std::unordered_map> obj_deps; + CPPTracker init_exprs = {"gen_init_expr", false, &compiled_items}; // // End of methods related to run-time initialization. @@ -944,12 +1041,20 @@ private: // See Emit.cc for definitions. // + // The following all need to be able to emit code. + friend class CPP_BasicConstInitsInfo; + friend class CPP_CompoundInitsInfo; + friend class IndicesManager; + // Used to create (indented) C++ {...} code blocks. "needs_semi" // controls whether to terminate the block with a ';' (such as // for class definitions. void StartBlock(); void EndBlock(bool needs_semi = false); + void IndentUp() { ++block_level; } + void IndentDown() { --block_level; } + // Various ways of generating code. The multi-argument methods // assume that the first argument is a printf-style format // (but one that can only have %s specifiers). @@ -960,11 +1065,12 @@ private: NL(); } - void Emit(const std::string& fmt, const std::string& arg) const + void Emit(const std::string& fmt, const std::string& arg, bool do_NL = true) const { Indent(); fprintf(write_file, fmt.c_str(), arg.c_str()); - NL(); + if ( do_NL ) + NL(); } void Emit(const std::string& fmt, const std::string& arg1, const std::string& arg2) const @@ -999,14 +1105,15 @@ private: NL(); } - // Returns an expression for constructing a Zeek String object - // corresponding to the given byte array. - std::string GenString(const char* b, int len) const; - - // For the given byte array / string, returns a version expanded - // with escape sequences in order to represent it as a C++ string. - std::string CPPEscape(const char* b, int len) const; - std::string CPPEscape(const char* s) const { return CPPEscape(s, strlen(s)); } + void Emit(const std::string& fmt, const std::string& arg1, const std::string& arg2, + const std::string& arg3, const std::string& arg4, const std::string& arg5, + const std::string& arg6) const + { + Indent(); + fprintf(write_file, fmt.c_str(), arg1.c_str(), arg2.c_str(), arg3.c_str(), arg4.c_str(), + arg5.c_str(), arg6.c_str()); + NL(); + } void NL() const { fputc('\n', write_file); } diff --git a/src/script_opt/CPP/Consts.cc b/src/script_opt/CPP/Consts.cc index c21a1db9b8..30b758df99 100644 --- a/src/script_opt/CPP/Consts.cc +++ b/src/script_opt/CPP/Consts.cc @@ -4,55 +4,26 @@ #include "zeek/RE.h" #include "zeek/script_opt/CPP/Compile.h" +using namespace std; + namespace zeek::detail { -using namespace std; - -string CPPCompile::BuildConstant(const Obj* parent, const ValPtr& vp) +shared_ptr CPPCompile::RegisterConstant(const ValPtr& vp, int& consts_offset) { - if ( ! vp ) - return "nullptr"; + // Make sure the value pointer, which might be transient + // in construction, sticks around so we can track its + // value. + cv_indices.push_back(vp); - if ( AddConstant(vp) ) - { - auto v = vp.get(); - AddInit(parent); - NoteInitDependency(parent, v); - - // Make sure the value pointer, which might be transient - // in construction, sticks around so we can track its - // value. - cv_indices.push_back(vp); - - return const_vals[v]; - } - else - return NativeToGT(GenVal(vp), vp->GetType(), GEN_VAL_PTR); - } - -void CPPCompile::AddConstant(const ConstExpr* c) - { - auto v = c->ValuePtr(); - - if ( AddConstant(v) ) - { - AddInit(c); - NoteInitDependency(c, v.get()); - } - } - -bool CPPCompile::AddConstant(const ValPtr& vp) - { auto v = vp.get(); - if ( IsNativeType(v->GetType()) ) - // These we instantiate directly. - return false; - if ( const_vals.count(v) > 0 ) + { // Already did this one. - return true; + consts_offset = const_offsets[v]; + return const_vals[v]; + } // Formulate a key that's unique per distinct constant. @@ -82,213 +53,100 @@ bool CPPCompile::AddConstant(const ValPtr& vp) if ( constants.count(c_desc) > 0 ) { const_vals[v] = constants[c_desc]; - - auto orig_v = constants_to_vals[c_desc]; - ASSERT(v != orig_v); - AddInit(v); - NoteInitDependency(v, orig_v); - - return true; + consts_offset = const_offsets[v] = constants_offsets[c_desc]; + return const_vals[v]; } - // Need a C++ global for this constant. - auto const_name = string("CPP__const__") + Fmt(int(constants.size())); - - const_vals[v] = constants[c_desc] = const_name; - constants_to_vals[c_desc] = v; - auto tag = t->Tag(); + auto const_name = const_info[tag]->NextName(); + shared_ptr gi; switch ( tag ) { - case TYPE_STRING: - AddStringConstant(vp, const_name); + case TYPE_BOOL: + gi = make_shared(vp->AsBool() ? "true" : "false"); break; - case TYPE_PATTERN: - AddPatternConstant(vp, const_name); + case TYPE_INT: + gi = make_shared(to_string(vp->AsInt())); break; - case TYPE_LIST: - AddListConstant(vp, const_name); + case TYPE_COUNT: + gi = make_shared(to_string(vp->AsCount()) + "ULL"); break; - case TYPE_RECORD: - AddRecordConstant(vp, const_name); + case TYPE_DOUBLE: + gi = make_shared(to_string(vp->AsDouble())); break; - case TYPE_TABLE: - AddTableConstant(vp, const_name); + case TYPE_TIME: + gi = make_shared(to_string(vp->AsDouble())); break; - case TYPE_VECTOR: - AddVectorConstant(vp, const_name); + case TYPE_INTERVAL: + gi = make_shared(to_string(vp->AsDouble())); break; case TYPE_ADDR: - case TYPE_SUBNET: - { - auto prefix = (tag == TYPE_ADDR) ? "Addr" : "SubNet"; - - Emit("%sValPtr %s;", prefix, const_name); - - ODesc d; - v->Describe(&d); - - AddInit(v, const_name, - string("make_intrusive<") + prefix + "Val>(\"" + d.Description() + "\")"); - } + gi = make_shared(this, vp); break; - case TYPE_FUNC: - Emit("FuncValPtr %s;", const_name); + case TYPE_SUBNET: + gi = make_shared(this, vp); + break; - // We can't generate the initialization now because it - // depends on first having compiled the associated body, - // so we know its hash. So for now we just note it - // to deal with later. - func_vars[v->AsFuncVal()] = const_name; + case TYPE_ENUM: + gi = make_shared(this, vp); + break; + + case TYPE_STRING: + gi = make_shared(this, vp); + break; + + case TYPE_PATTERN: + gi = make_shared(this, vp); + break; + + case TYPE_PORT: + gi = make_shared(vp); + break; + + case TYPE_LIST: + gi = make_shared(this, vp); + break; + + case TYPE_VECTOR: + gi = make_shared(this, vp); + break; + + case TYPE_RECORD: + gi = make_shared(this, vp); + break; + + case TYPE_TABLE: + gi = make_shared(this, vp); break; case TYPE_FILE: - { - Emit("FileValPtr %s;", const_name); + gi = make_shared(this, vp); + break; - auto f = cast_intrusive(vp)->Get(); - - AddInit(v, const_name, - string("make_intrusive(") + "make_intrusive(\"" + f->Name() + - "\", \"w\"))"); - } + case TYPE_FUNC: + gi = make_shared(this, vp); break; default: reporter->InternalError("bad constant type in CPPCompile::AddConstant"); + break; } - return true; - } + const_info[tag]->AddInstance(gi); + const_vals[v] = constants[c_desc] = gi; -void CPPCompile::AddStringConstant(const ValPtr& v, string& const_name) - { - Emit("StringValPtr %s;", const_name); + consts_offset = const_offsets[v] = constants_offsets[c_desc] = consts.size(); + consts.emplace_back(pair(tag, gi->Offset())); - auto s = v->AsString(); - const char* b = (const char*)(s->Bytes()); - auto len = s->Len(); - - AddInit(v, const_name, GenString(b, len)); - } - -void CPPCompile::AddPatternConstant(const ValPtr& v, string& const_name) - { - Emit("PatternValPtr %s;", const_name); - - auto re = v->AsPatternVal()->Get(); - - AddInit(v, string("{ auto re = new RE_Matcher(") + CPPEscape(re->OrigText()) + ");"); - - if ( re->IsCaseInsensitive() ) - AddInit(v, "re->MakeCaseInsensitive();"); - - AddInit(v, "re->Compile();"); - AddInit(v, const_name, "make_intrusive(re)"); - AddInit(v, "}"); - } - -void CPPCompile::AddListConstant(const ValPtr& v, string& const_name) - { - Emit("ListValPtr %s;", const_name); - - // No initialization dependency on the main type since we don't - // use the underlying TypeList. However, we *do* use the types of - // the elements. - - AddInit(v, const_name, string("make_intrusive(TYPE_ANY)")); - - auto lv = cast_intrusive(v); - auto n = lv->Length(); - - for ( auto i = 0; i < n; ++i ) - { - const auto& l_i = lv->Idx(i); - auto l_i_c = BuildConstant(v, l_i); - AddInit(v, const_name + "->Append(" + l_i_c + ");"); - NoteInitDependency(v, TypeRep(l_i->GetType())); - } - } - -void CPPCompile::AddRecordConstant(const ValPtr& v, string& const_name) - { - const auto& t = v->GetType(); - - Emit("RecordValPtr %s;", const_name); - - NoteInitDependency(v, TypeRep(t)); - - AddInit(v, const_name, - string("make_intrusive(") + "cast_intrusive(" + GenTypeName(t) + - "))"); - - auto r = cast_intrusive(v); - auto n = r->NumFields(); - - for ( auto i = 0u; i < n; ++i ) - { - const auto& r_i = r->GetField(i); - - if ( r_i ) - { - auto r_i_c = BuildConstant(v, r_i); - AddInit(v, const_name + "->Assign(" + Fmt(static_cast(i)) + ", " + r_i_c + ");"); - } - } - } - -void CPPCompile::AddTableConstant(const ValPtr& v, string& const_name) - { - const auto& t = v->GetType(); - - Emit("TableValPtr %s;", const_name); - - NoteInitDependency(v, TypeRep(t)); - - AddInit(v, const_name, - string("make_intrusive(") + "cast_intrusive(" + GenTypeName(t) + - "))"); - - auto tv = cast_intrusive(v); - auto tv_map = tv->ToMap(); - - for ( auto& tv_i : tv_map ) - { - auto ind = BuildConstant(v, tv_i.first); - auto val = BuildConstant(v, tv_i.second); - AddInit(v, const_name + "->Assign(" + ind + ", " + val + ");"); - } - } - -void CPPCompile::AddVectorConstant(const ValPtr& v, string& const_name) - { - const auto& t = v->GetType(); - - Emit("VectorValPtr %s;", const_name); - - NoteInitDependency(v, TypeRep(t)); - - AddInit(v, const_name, - string("make_intrusive(") + "cast_intrusive(" + GenTypeName(t) + - "))"); - - auto vv = cast_intrusive(v); - auto n = vv->Size(); - - for ( auto i = 0u; i < n; ++i ) - { - const auto& v_i = vv->ValAt(i); - auto v_i_c = BuildConstant(v, v_i); - AddInit(v, const_name + "->Append(" + v_i_c + ");"); - } + return gi; } } // zeek::detail diff --git a/src/script_opt/CPP/DeclFunc.cc b/src/script_opt/CPP/DeclFunc.cc index dbca009052..0d9117d2d8 100644 --- a/src/script_opt/CPP/DeclFunc.cc +++ b/src/script_opt/CPP/DeclFunc.cc @@ -22,7 +22,7 @@ void CPPCompile::DeclareFunc(const FuncInfo& func) const auto& body = func.Body(); auto priority = func.Priority(); - DeclareSubclass(f->GetType(), pf, fname, body, priority, nullptr, f->Flavor()); + CreateFunction(f->GetType(), pf, fname, body, priority, nullptr, f->Flavor()); if ( f->GetBodies().size() == 1 ) compiled_simple_funcs[f->Name()] = fname; @@ -40,17 +40,88 @@ void CPPCompile::DeclareLambda(const LambdaExpr* l, const ProfileFunc* pf) for ( auto id : ids ) lambda_names[id] = LocalName(id); - DeclareSubclass(l_id->GetType(), pf, lname, body, 0, l, FUNC_FLAVOR_FUNCTION); + CreateFunction(l_id->GetType(), pf, lname, body, 0, l, FUNC_FLAVOR_FUNCTION); } -void CPPCompile::DeclareSubclass(const FuncTypePtr& ft, const ProfileFunc* pf, const string& fname, - const StmtPtr& body, int priority, const LambdaExpr* l, - FunctionFlavor flavor) +void CPPCompile::CreateFunction(const FuncTypePtr& ft, const ProfileFunc* pf, const string& fname, + const StmtPtr& body, int priority, const LambdaExpr* l, + FunctionFlavor flavor) { const auto& yt = ft->Yield(); in_hook = flavor == FUNC_FLAVOR_HOOK; const IDPList* lambda_ids = l ? &l->OuterIDs() : nullptr; + string args = BindArgs(ft, lambda_ids); + + auto yt_decl = in_hook ? "bool" : FullTypeName(yt); + + vector p_types; + GatherParamTypes(p_types, ft, lambda_ids, pf); + + string cast = string(yt_decl) + "(*)("; + for ( auto& pt : p_types ) + cast += pt + ", "; + cast += string("Frame*)"); + + // We need to distinguish between hooks and non-hooks that happen + // to have matching type signatures. They'll be equivalent if they + // have identical cast's. To keep them separate, we cheat and + // make hook casts different, string-wise, without altering their + // semantics. + if ( in_hook ) + cast += " "; + + func_index[fname] = cast; + + if ( casting_index.count(cast) == 0 ) + { + casting_index[cast] = func_casting_glue.size(); + + DispatchInfo di; + di.cast = cast; + di.args = args; + di.is_hook = in_hook; + di.yield = yt; + + func_casting_glue.emplace_back(di); + } + + if ( lambda_ids ) + { + DeclareSubclass(ft, pf, fname, args, lambda_ids); + BuildLambda(ft, pf, fname, body, l, lambda_ids); + EndBlock(true); + } + else + { + Emit("static %s %s(%s);", yt_decl, fname, ParamDecl(ft, lambda_ids, pf)); + + // Track this function as known to have been compiled. + // We don't track lambda bodies as compiled because they + // can't be instantiated directly without also supplying + // the captures. In principle we could make an exception + // for lambdas that don't take any arguments, but that + // seems potentially more confusing than beneficial. + compiled_funcs.emplace(fname); + + auto loc_f = script_specific_filename(body); + cf_locs[fname] = loc_f; + } + + auto h = pf->HashVal(); + + body_hashes[fname] = h; + body_priorities[fname] = priority; + body_names.emplace(body.get(), fname); + + total_hash = merge_p_hashes(total_hash, h); + } + +void CPPCompile::DeclareSubclass(const FuncTypePtr& ft, const ProfileFunc* pf, const string& fname, + const string& args, const IDPList* lambda_ids) + { + const auto& yt = ft->Yield(); + auto yt_decl = in_hook ? "bool" : FullTypeName(yt); NL(); @@ -76,8 +147,7 @@ void CPPCompile::DeclareSubclass(const FuncTypePtr& ft, const ProfileFunc* pf, c } } - Emit("%s_cl(const char* name%s) : CPPStmt(name)%s { }", fname, addl_args.c_str(), - inits.c_str()); + Emit("%s_cl(const char* name%s) : CPPStmt(name)%s { }", fname, addl_args, inits); // An additional constructor just used to generate place-holder // instances, due to the mis-design that lambdas are identified @@ -92,7 +162,7 @@ void CPPCompile::DeclareSubclass(const FuncTypePtr& ft, const ProfileFunc* pf, c if ( in_hook ) { - Emit("if ( ! %s(%s) )", fname, BindArgs(ft, lambda_ids)); + Emit("if ( ! %s(%s) )", fname, args); StartBlock(); Emit("flow = FLOW_BREAK;"); EndBlock(); @@ -100,42 +170,36 @@ void CPPCompile::DeclareSubclass(const FuncTypePtr& ft, const ProfileFunc* pf, c } else if ( IsNativeType(yt) ) - GenInvokeBody(fname, yt, BindArgs(ft, lambda_ids)); + GenInvokeBody(fname, yt, args); else - Emit("return %s(%s);", fname, BindArgs(ft, lambda_ids)); + Emit("return %s(%s);", fname, args); EndBlock(); + } - if ( lambda_ids ) - BuildLambda(ft, pf, fname, body, l, lambda_ids); - else - { - // Track this function as known to have been compiled. - // We don't track lambda bodies as compiled because they - // can't be instantiated directly without also supplying - // the captures. In principle we could make an exception - // for lambdas that don't take any arguments, but that - // seems potentially more confusing than beneficial. - compiled_funcs.emplace(fname); - - auto loc_f = script_specific_filename(body); - cf_locs[fname] = loc_f; - - // Some guidance for those looking through the generated code. - Emit("// compiled body for: %s", loc_f); - } - - EndBlock(true); - - auto h = pf->HashVal(); - - body_hashes[fname] = h; - body_priorities[fname] = priority; - body_names.emplace(body.get(), fname); - names_to_bodies.emplace(fname, body.get()); - - total_hash = merge_p_hashes(total_hash, h); +void CPPCompile::DeclareDynCPPStmt() + { + Emit("// A version of CPPStmt that manages a function pointer and"); + Emit("// dynamically casts it to a given type to call it via Exec()."); + Emit("// We will later generate a custom Exec method to support this"); + Emit("// dispatch. All of this is ugly, and only needed because clang"); + Emit("// goes nuts (super slow) in the face of thousands of templates"); + Emit("// in a given context (initializers, or a function body)."); + Emit("class CPPDynStmt : public CPPStmt"); + Emit("\t{"); + Emit("public:"); + Emit("\tCPPDynStmt(const char* _name, void* _func, int _type_signature) : CPPStmt(_name), " + "func(_func), type_signature(_type_signature) { }"); + Emit("\tValPtr Exec(Frame* f, StmtFlowType& flow) override final;"); + Emit("private:"); + Emit("\t// The function to call in Exec()."); + Emit("\tvoid* func;"); + Emit("\t// Used via a switch in the dynamically-generated Exec() method"); + Emit("\t// to cast func to the write type, and to call it with the"); + Emit("\t// right arguments pulled out of the frame."); + Emit("\tint type_signature;"); + Emit("\t};"); } void CPPCompile::BuildLambda(const FuncTypePtr& ft, const ProfileFunc* pf, const string& fname, @@ -146,28 +210,17 @@ void CPPCompile::BuildLambda(const FuncTypePtr& ft, const ProfileFunc* pf, const { auto name = lambda_names[id]; auto tn = FullTypeName(id->GetType()); - Emit("%s %s;", tn, name.c_str()); + Emit("%s %s;", tn, name); } // Generate initialization to create and register the lambda. - auto literal_name = string("\"") + l->Name() + "\""; - auto instantiate = string("make_intrusive<") + fname + "_cl>(" + literal_name + ")"; + auto h = pf->HashVal(); + auto nl = lambda_ids->length(); + bool has_captures = nl > 0; - int nl = lambda_ids->length(); - auto h = Fmt(pf->HashVal()); - auto has_captures = nl > 0 ? "true" : "false"; - auto l_init = string("register_lambda__CPP(") + instantiate + ", " + h + ", \"" + l->Name() + - "\", " + GenTypeName(ft) + ", " + has_captures + ");"; - - AddInit(l, l_init); - NoteInitDependency(l, TypeRep(ft)); - - // Make the lambda's body's initialization depend on the lambda's - // initialization. That way GenFuncVarInits() can generate - // initializations with the assurance that the associated body - // hashes will have been registered. - AddInit(body.get()); - NoteInitDependency(body.get(), l); + auto gi = make_shared(this, l->Name(), ft, fname + "_cl", h, + has_captures); + lambda_reg_info->AddInstance(gi); // Generate method to extract the lambda captures from a deserialized // Frame object. @@ -237,17 +290,71 @@ string CPPCompile::BindArgs(const FuncTypePtr& ft, const IDPList* lambda_ids) string CPPCompile::ParamDecl(const FuncTypePtr& ft, const IDPList* lambda_ids, const ProfileFunc* pf) { - const auto& params = ft->Params(); - int n = params->NumFields(); + vector p_types; + vector p_names; + + GatherParamTypes(p_types, ft, lambda_ids, pf); + GatherParamNames(p_names, ft, lambda_ids, pf); + + ASSERT(p_types.size() == p_names.size()); string decl; + for ( auto i = 0U; i < p_types.size(); ++i ) + decl += p_types[i] + " " + p_names[i] + ", "; + + // Add in the declaration of the frame. + return decl + "Frame* f__CPP"; + } + +void CPPCompile::GatherParamTypes(vector& p_types, const FuncTypePtr& ft, + const IDPList* lambda_ids, const ProfileFunc* pf) + { + const auto& params = ft->Params(); + int n = params->NumFields(); + for ( auto i = 0; i < n; ++i ) { const auto& t = params->GetFieldType(i); auto tn = FullTypeName(t); auto param_id = FindParam(i, pf); - string fn; + + if ( IsNativeType(t) ) + // Native types are always pass-by-value. + p_types.emplace_back(tn); + else + { + if ( param_id && pf->Assignees().count(param_id) > 0 ) + // We modify the parameter. + p_types.emplace_back(tn); + else + // Not modified, so pass by const reference. + p_types.emplace_back(string("const ") + tn + "&"); + } + } + + if ( lambda_ids ) + // Add the captures as additional parameters. + for ( auto& id : *lambda_ids ) + { + const auto& t = id->GetType(); + auto tn = FullTypeName(t); + + // Allow the captures to be modified. + p_types.emplace_back(string(tn) + "& "); + } + } + +void CPPCompile::GatherParamNames(vector& p_names, const FuncTypePtr& ft, + const IDPList* lambda_ids, const ProfileFunc* pf) + { + const auto& params = ft->Params(); + int n = params->NumFields(); + + for ( auto i = 0; i < n; ++i ) + { + const auto& t = params->GetFieldType(i); + auto param_id = FindParam(i, pf); if ( param_id ) { @@ -255,50 +362,22 @@ string CPPCompile::ParamDecl(const FuncTypePtr& ft, const IDPList* lambda_ids, // We'll need to translate the parameter // from its current representation to // type "any". - fn = string("any_param__CPP_") + Fmt(i); + p_names.emplace_back(string("any_param__CPP_") + Fmt(i)); else - fn = LocalName(param_id); + p_names.emplace_back(LocalName(param_id)); } else - // Parameters that are unused don't wind up - // in the ProfileFunc. Rather than dig their - // name out of the function's declaration, we - // explicitly name them to reflect that they're - // unused. - fn = string("unused_param__CPP_") + Fmt(i); - - if ( IsNativeType(t) ) - // Native types are always pass-by-value. - decl = decl + tn + " " + fn; - else - { - if ( param_id && pf->Assignees().count(param_id) > 0 ) - // We modify the parameter. - decl = decl + tn + " " + fn; - else - // Not modified, so pass by const reference. - decl = decl + "const " + tn + "& " + fn; - } - - decl += ", "; + // Parameters that are unused don't wind up in the + // ProfileFunc. Rather than dig their name out of + // the function's declaration, we explicitly name + // them to reflect that they're unused. + p_names.emplace_back(string("unused_param__CPP_") + Fmt(i)); } if ( lambda_ids ) - { // Add the captures as additional parameters. for ( auto& id : *lambda_ids ) - { - auto name = lambda_names[id]; - const auto& t = id->GetType(); - auto tn = FullTypeName(t); - - // Allow the captures to be modified. - decl = decl + tn + "& " + name + ", "; - } - } - - // Add in the declaration of the frame. - return decl + "Frame* f__CPP"; + p_names.emplace_back(lambda_names[id]); } const ID* CPPCompile::FindParam(int i, const ProfileFunc* pf) diff --git a/src/script_opt/CPP/Driver.cc b/src/script_opt/CPP/Driver.cc index ccaa0a0190..7e446a302f 100644 --- a/src/script_opt/CPP/Driver.cc +++ b/src/script_opt/CPP/Driver.cc @@ -12,14 +12,13 @@ namespace zeek::detail using namespace std; CPPCompile::CPPCompile(vector& _funcs, ProfileFuncs& _pfs, const string& gen_name, - const string& _addl_name, CPPHashManager& _hm, bool _update, - bool _standalone, bool report_uncompilable) - : funcs(_funcs), pfs(_pfs), hm(_hm), update(_update), standalone(_standalone) + const string& _addl_name, CPPHashManager& _hm, bool _standalone, + bool report_uncompilable) + : funcs(_funcs), pfs(_pfs), hm(_hm), standalone(_standalone) { addl_name = _addl_name; - bool is_addl = hm.IsAppend(); - auto target_name = is_addl ? addl_name.c_str() : gen_name.c_str(); - auto mode = is_addl ? "a" : "w"; + auto target_name = gen_name.c_str(); + auto mode = "w"; write_file = fopen(target_name, mode); if ( ! write_file ) @@ -27,30 +26,6 @@ CPPCompile::CPPCompile(vector& _funcs, ProfileFuncs& _pfs, const strin reporter->Error("can't open C++ target file %s", target_name); exit(1); } - - if ( is_addl ) - { - // We need a unique number to associate with the name - // space for the code we're adding. A convenient way to - // generate this safely is to use the present size of the - // file we're appending to. That guarantees that every - // incremental compilation will wind up with a different - // number. - struct stat st; - if ( fstat(fileno(write_file), &st) != 0 ) - { - char buf[256]; - util::zeek_strerror_r(errno, buf, sizeof(buf)); - reporter->Error("fstat failed on %s: %s", target_name, buf); - exit(1); - } - - // We use a value of "0" to mean "we're not appending, - // we're generating from scratch", so make sure we're - // distinct from that. - addl_tag = st.st_size + 1; - } - else { // Create an empty "additional" file. @@ -83,10 +58,6 @@ void CPPCompile::Compile(bool report_uncompilable) working_dir = buf; - if ( update && addl_tag > 0 && CheckForCollisions() ) - // Inconsistent compilation environment. - exit(1); - GenProlog(); // Determine which functions we can call directly, and reuse @@ -100,9 +71,13 @@ void CPPCompile::Compile(bool report_uncompilable) const char* reason; if ( IsCompilable(func, &reason) ) compilable_funcs.insert(BodyName(func)); - else if ( reason && report_uncompilable ) - fprintf(stderr, "%s cannot be compiled to C++ due to %s\n", func.Func()->Name(), - reason); + else + { + if ( reason && report_uncompilable ) + fprintf(stderr, "%s cannot be compiled to C++ due to %s\n", func.Func()->Name(), + reason); + not_fully_compilable.insert(func.Func()->Name()); + } auto h = func.Profile()->HashVal(); if ( hm.HasHash(h) ) @@ -119,39 +94,24 @@ void CPPCompile::Compile(bool report_uncompilable) { TypePtr tp{NewRef{}, (Type*)(t)}; types.AddKey(tp, pfs.HashType(t)); + (void)RegisterType(tp); } - for ( const auto& t : types.DistinctKeys() ) - if ( ! types.IsInherited(t) ) - // Type is new to this compilation, so we'll - // be generating it. - Emit("TypePtr %s;", types.KeyName(t)); + // ### This doesn't work for -O add-C++ + Emit("TypePtr types__CPP[%s];", Fmt(static_cast(types.DistinctKeys().size()))); NL(); - for ( const auto& c : pfs.Constants() ) - AddConstant(c); +#if 0 + for ( auto gi : all_global_info ) + Emit(gi->Declare()); NL(); +#endif for ( auto& g : pfs.AllGlobals() ) CreateGlobal(g); - // Now that the globals are created, register their attributes, - // if any, and generate their initialization for use in standalone - // scripts. We can't do these in CreateGlobal() because at that - // point it's possible that some of the globals refer to other - // globals not-yet-created. - for ( auto& g : pfs.AllGlobals() ) - { - RegisterAttributes(g->GetAttrs()); - if ( g->HasVal() ) - { - auto gn = string(g->Name()); - GenGlobalInit(g, globals[gn], g->GetVal()); - } - } - for ( const auto& e : pfs.Events() ) if ( AddGlobal(e, "gl", false) ) Emit("EventHandlerPtr %s_ev;", globals[string(e)]); @@ -201,10 +161,13 @@ void CPPCompile::Compile(bool report_uncompilable) lambda_names.insert(n); } + NL(); + Emit("std::vector CPP__bodies_to_register = {"); + for ( const auto& f : compiled_funcs ) RegisterCompiledBody(f); - GenFuncVarInits(); + Emit("};"); GenEpilog(); } @@ -217,12 +180,75 @@ void CPPCompile::GenProlog() Emit("namespace zeek::detail { //\n"); } - Emit("namespace CPP_%s { // %s\n", Fmt(addl_tag), working_dir.c_str()); + Emit("namespace CPP_%s { // %s\n", Fmt(addl_tag), working_dir); // The following might-or-might-not wind up being populated/used. Emit("std::vector field_mapping;"); Emit("std::vector enum_mapping;"); NL(); + + const_info[TYPE_BOOL] = CreateConstInitInfo("Bool", "ValPtr", "bool"); + const_info[TYPE_INT] = CreateConstInitInfo("Int", "ValPtr", "bro_int_t"); + const_info[TYPE_COUNT] = CreateConstInitInfo("Count", "ValPtr", "bro_uint_t"); + const_info[TYPE_DOUBLE] = CreateConstInitInfo("Double", "ValPtr", "double"); + const_info[TYPE_TIME] = CreateConstInitInfo("Time", "ValPtr", "double"); + const_info[TYPE_INTERVAL] = CreateConstInitInfo("Interval", "ValPtr", "double"); + const_info[TYPE_ADDR] = CreateConstInitInfo("Addr", "ValPtr", ""); + const_info[TYPE_SUBNET] = CreateConstInitInfo("SubNet", "ValPtr", ""); + const_info[TYPE_PORT] = CreateConstInitInfo("Port", "ValPtr", "uint32_t"); + + const_info[TYPE_ENUM] = CreateCompoundInitInfo("Enum", "ValPtr"); + const_info[TYPE_STRING] = CreateCompoundInitInfo("String", "ValPtr"); + const_info[TYPE_LIST] = CreateCompoundInitInfo("List", "ValPtr"); + const_info[TYPE_PATTERN] = CreateCompoundInitInfo("Pattern", "ValPtr"); + const_info[TYPE_VECTOR] = CreateCompoundInitInfo("Vector", "ValPtr"); + const_info[TYPE_RECORD] = CreateCompoundInitInfo("Record", "ValPtr"); + const_info[TYPE_TABLE] = CreateCompoundInitInfo("Table", "ValPtr"); + const_info[TYPE_FUNC] = CreateCompoundInitInfo("Func", "ValPtr"); + const_info[TYPE_FILE] = CreateCompoundInitInfo("File", "ValPtr"); + + type_info = CreateCompoundInitInfo("Type", "Ptr"); + attr_info = CreateCompoundInitInfo("Attr", "Ptr"); + attrs_info = CreateCompoundInitInfo("Attributes", "Ptr"); + + call_exprs_info = CreateCustomInitInfo("CallExpr", "Ptr"); + lambda_reg_info = CreateCustomInitInfo("LambdaRegistration", ""); + global_id_info = CreateCustomInitInfo("GlobalID", ""); + + NL(); + DeclareDynCPPStmt(); + NL(); + } + +shared_ptr CPPCompile::CreateConstInitInfo(const char* tag, const char* type, + const char* c_type) + { + auto gi = make_shared(tag, type, c_type); + return RegisterInitInfo(tag, type, gi); + } + +shared_ptr CPPCompile::CreateCompoundInitInfo(const char* tag, const char* type) + { + auto gi = make_shared(tag, type); + return RegisterInitInfo(tag, type, gi); + } + +shared_ptr CPPCompile::CreateCustomInitInfo(const char* tag, const char* type) + { + auto gi = make_shared(tag, type); + if ( type[0] == '\0' ) + gi->SetCPPType("void*"); + + return RegisterInitInfo(tag, type, gi); + } + +shared_ptr CPPCompile::RegisterInitInfo(const char* tag, const char* type, + shared_ptr gi) + { + string v_type = type[0] ? (string(tag) + type) : "void*"; + Emit("std::vector<%s> CPP__%s__;", v_type, string(tag)); + all_global_info.insert(gi); + return gi; } void CPPCompile::RegisterCompiledBody(const string& f) @@ -252,74 +278,135 @@ void CPPCompile::RegisterCompiledBody(const string& f) // same binary). h = merge_p_hashes(h, p_hash(cf_locs[f])); - auto init = string("register_body__CPP(make_intrusive<") + f + "_cl>(\"" + f + "\"), " + - Fmt(p) + ", " + Fmt(h) + ", " + events + ");"; - - AddInit(names_to_bodies[f], init); - - if ( update ) - { - fprintf(hm.HashFile(), "func\n%s%s\n", scope_prefix(addl_tag).c_str(), f.c_str()); - fprintf(hm.HashFile(), "%llu\n", h); - } + ASSERT(func_index.count(f) > 0); + auto type_signature = casting_index[func_index[f]]; + Emit("\tCPP_RegisterBody(\"%s\", (void*) %s, %s, %s, %s, std::vector(%s)),", f, f, + Fmt(type_signature), Fmt(p), Fmt(h), events); } void CPPCompile::GenEpilog() { NL(); + for ( const auto& ii : init_infos ) + GenInitExpr(ii.second); - for ( const auto& e : init_exprs.DistinctKeys() ) + NL(); + Emit("ValPtr CPPDynStmt::Exec(Frame* f, StmtFlowType& flow)"); + StartBlock(); + Emit("flow = FLOW_RETURN;"); + Emit("switch ( type_signature )"); + StartBlock(); + for ( auto i = 0U; i < func_casting_glue.size(); ++i ) { - GenInitExpr(e); - if ( update ) - init_exprs.LogIfNew(e, addl_tag, hm.HashFile()); + Emit("case %s:", to_string(i)); + StartBlock(); + auto& glue = func_casting_glue[i]; + + auto invoke = string("(*(") + glue.cast + ")(func))(" + glue.args + ")"; + + if ( glue.is_hook ) + { + Emit("if ( ! %s )", invoke); + StartBlock(); + Emit("flow = FLOW_BREAK;"); + EndBlock(); + Emit("return nullptr;"); + } + + else if ( IsNativeType(glue.yield) ) + GenInvokeBody(invoke, glue.yield); + + else + Emit("return %s;", invoke); + + EndBlock(); } - for ( const auto& a : attributes.DistinctKeys() ) - { - GenAttrs(a); - if ( update ) - attributes.LogIfNew(a, addl_tag, hm.HashFile()); - } + Emit("default:"); + Emit("\treporter->InternalError(\"invalid type in CPPDynStmt::Exec\");"); + Emit("\treturn nullptr;"); - // Generate the guts of compound types, and preserve type names - // if present. - for ( const auto& t : types.DistinctKeys() ) - { - ExpandTypeVar(t); - if ( update ) - types.LogIfNew(t, addl_tag, hm.HashFile()); - } + EndBlock(); + EndBlock(); - InitializeEnumMappings(); + NL(); - GenPreInits(); - - unordered_set to_do; - for ( const auto& oi : obj_inits ) - to_do.insert(oi.first); - - CheckInitConsistency(to_do); - auto nc = GenDependentInits(to_do); + for ( auto gi : all_global_info ) + gi->GenerateInitializers(this); if ( standalone ) GenStandaloneActivation(); + NL(); + InitializeEnumMappings(); + + NL(); + InitializeFieldMappings(); + + NL(); + InitializeBiFs(); + + NL(); + indices_mgr.Generate(this); + + NL(); + InitializeStrings(); + + NL(); + InitializeHashes(); + + NL(); + InitializeConsts(); + NL(); Emit("void init__CPP()"); StartBlock(); - Emit("enum_mapping.resize(%s);\n", Fmt(int(enum_names.size()))); - Emit("pre_init__CPP();"); + Emit("std::vector> InitIndices;"); + Emit("generate_indices_set(CPP__Indices__init, InitIndices);"); + + Emit("std::map> InitConsts;"); NL(); - for ( auto i = 1; i <= nc; ++i ) - Emit("init_%s__CPP();", Fmt(i)); + for ( const auto& ci : const_info ) + { + auto& gi = ci.second; + Emit("InitConsts.emplace(%s, std::make_shared>(%s));", + TypeTagName(ci.first), gi->CPPType(), gi->InitsName()); + } + + Emit("InitsManager im(CPP__ConstVals, InitConsts, InitIndices, CPP__Strings, CPP__Hashes, " + "CPP__Type__, CPP__Attributes__, CPP__Attr__, CPP__CallExpr__);"); + + NL(); + Emit("for ( auto& b : CPP__bodies_to_register )"); + StartBlock(); + Emit("auto f = make_intrusive(b.func_name.c_str(), b.func, b.type_signature);"); + Emit("register_body__CPP(f, b.priority, b.h, b.events);"); + EndBlock(); + + NL(); + int max_cohort = 0; + for ( auto gi : all_global_info ) + max_cohort = std::max(max_cohort, gi->MaxCohort()); + + for ( auto c = 0; c <= max_cohort; ++c ) + for ( auto gi : all_global_info ) + if ( gi->CohortSize(c) > 0 ) + Emit("%s.InitializeCohort(&im, %s);", gi->InitializersName(), Fmt(c)); + + NL(); + Emit("for ( auto& b : CPP__BiF_lookups__ )"); + Emit("\tb.ResolveBiF();"); // Populate mappings for dynamic offsets. NL(); - InitializeFieldMappings(); + Emit("for ( auto& em : CPP__enum_mappings__ )"); + Emit("\tenum_mapping.push_back(em.ComputeOffset(&im));"); + NL(); + Emit("for ( auto& fm : CPP__field_mappings__ )"); + Emit("\tfield_mapping.push_back(fm.ComputeOffset(&im));"); if ( standalone ) Emit("standalone_init__CPP();"); @@ -328,10 +415,7 @@ void CPPCompile::GenEpilog() GenInitHook(); - Emit("} // %s\n\n", scope_prefix(addl_tag).c_str()); - - if ( update ) - UpdateGlobalHashes(); + Emit("} // %s\n\n", scope_prefix(addl_tag)); if ( addl_tag > 0 ) return; diff --git a/src/script_opt/CPP/Emit.cc b/src/script_opt/CPP/Emit.cc index 91a79b2a2f..84e122f9c0 100644 --- a/src/script_opt/CPP/Emit.cc +++ b/src/script_opt/CPP/Emit.cc @@ -13,75 +13,14 @@ using namespace std; void CPPCompile::StartBlock() { - ++block_level; + IndentUp(); Emit("{"); } void CPPCompile::EndBlock(bool needs_semi) { Emit("}%s", needs_semi ? ";" : ""); - --block_level; - } - -string CPPCompile::GenString(const char* b, int len) const - { - return string("make_intrusive(") + Fmt(len) + ", " + CPPEscape(b, len) + ")"; - } - -string CPPCompile::CPPEscape(const char* b, int len) const - { - string res = "\""; - - for ( int i = 0; i < len; ++i ) - { - unsigned char c = b[i]; - - switch ( c ) - { - case '\a': - res += "\\a"; - break; - case '\b': - res += "\\b"; - break; - case '\f': - res += "\\f"; - break; - case '\n': - res += "\\n"; - break; - case '\r': - res += "\\r"; - break; - case '\t': - res += "\\t"; - break; - case '\v': - res += "\\v"; - break; - - case '\\': - res += "\\\\"; - break; - case '"': - res += "\\\""; - break; - - default: - if ( isprint(c) ) - res += c; - else - { - char buf[8192]; - snprintf(buf, sizeof buf, "%03o", c); - res += "\\"; - res += buf; - } - break; - } - } - - return res + "\""; + IndentDown(); } void CPPCompile::Indent() const diff --git a/src/script_opt/CPP/Exprs.cc b/src/script_opt/CPP/Exprs.cc index c2a9cc2753..3caf7805f3 100644 --- a/src/script_opt/CPP/Exprs.cc +++ b/src/script_opt/CPP/Exprs.cc @@ -232,7 +232,12 @@ string CPPCompile::GenConstExpr(const ConstExpr* c, GenType gt) const auto& t = c->GetType(); if ( ! IsNativeType(t) ) - return NativeToGT(const_vals[c->Value()], t, gt); + { + auto v = c->ValuePtr(); + int consts_offset; // ignored + (void)RegisterConstant(v, consts_offset); + return NativeToGT(const_vals[v.get()]->Name(), t, gt); + } return NativeToGT(GenVal(c->ValuePtr()), t, gt); } @@ -1177,8 +1182,10 @@ string CPPCompile::GenField(const ExprPtr& rec, int field) // New mapping. mapping_slot = num_rf_mappings++; + ASSERT(processed_types.count(rt) > 0); + auto rt_offset = processed_types[rt]->Offset(); string field_name = rt->FieldName(field); - field_decls.emplace_back(pair(rt, rt->FieldDecl(field))); + field_decls.emplace_back(pair(rt_offset, rt->FieldDecl(field))); if ( record_field_mappings.count(rt) > 0 ) // We're already tracking this record. @@ -1217,7 +1224,7 @@ string CPPCompile::GenEnum(const TypePtr& t, const ValPtr& ev) mapping_slot = num_ev_mappings++; string enum_name = et->Lookup(v); - enum_names.emplace_back(pair(et, move(enum_name))); + enum_names.emplace_back(pair(TypeOffset(t), move(enum_name))); if ( enum_val_mappings.count(et) > 0 ) { diff --git a/src/script_opt/CPP/GenFunc.cc b/src/script_opt/CPP/GenFunc.cc index d0cb328f87..4cd57e7c7d 100644 --- a/src/script_opt/CPP/GenFunc.cc +++ b/src/script_opt/CPP/GenFunc.cc @@ -34,10 +34,8 @@ void CPPCompile::CompileLambda(const LambdaExpr* l, const ProfileFunc* pf) DefineBody(l_id->GetType(), pf, lname, body, &ids, FUNC_FLAVOR_FUNCTION); } -void CPPCompile::GenInvokeBody(const string& fname, const TypePtr& t, const string& args) +void CPPCompile::GenInvokeBody(const string& call, const TypePtr& t) { - auto call = fname + "(" + args + ")"; - if ( ! t || t->Tag() == TYPE_VOID ) { Emit("%s;", call); @@ -144,7 +142,7 @@ void CPPCompile::InitializeEvents(const ProfileFunc* pf) // returns an EventHandlerPtr, sigh. Emit("if ( event_registry->Lookup(\"%s\") )", e); StartBlock(); - Emit("%s = event_registry->Register(\"%s\");", ev_name.c_str(), e); + Emit("%s = event_registry->Register(\"%s\");", ev_name, e); EndBlock(); Emit("did_init = true;"); EndBlock(); @@ -233,6 +231,16 @@ string CPPCompile::BodyName(const FuncInfo& func) return fname + "__" + Fmt(static_cast(i)); } +p_hash_type CPPCompile::BodyHash(const Stmt* body) + { + ASSERT(body_names.count(body) > 0); + + auto& body_name = body_names[body]; + ASSERT(body_hashes.count(body_name) > 0); + + return body_hashes[body_name]; + } + string CPPCompile::GenArgs(const RecordTypePtr& params, const Expr* e) { const auto& exprs = e->AsListExpr()->Exprs(); diff --git a/src/script_opt/CPP/HashMgr.cc b/src/script_opt/CPP/HashMgr.cc index 4a6625391a..402b9dc012 100644 --- a/src/script_opt/CPP/HashMgr.cc +++ b/src/script_opt/CPP/HashMgr.cc @@ -12,28 +12,11 @@ using namespace std; VarMapper compiled_items; -CPPHashManager::CPPHashManager(const char* hash_name_base, bool _append) +CPPHashManager::CPPHashManager(const char* hash_name_base) { - append = _append; - hash_name = string(hash_name_base) + ".dat"; - if ( append ) - { - hf_r = fopen(hash_name.c_str(), "r"); - if ( ! hf_r ) - { - reporter->Error("can't open auxiliary C++ hash file %s for reading", hash_name.c_str()); - exit(1); - } - - lock_file(hash_name, hf_r); - LoadHashes(hf_r); - } - - auto mode = append ? "a" : "w"; - - hf_w = fopen(hash_name.c_str(), mode); + hf_w = fopen(hash_name.c_str(), "w"); if ( ! hf_w ) { reporter->Error("can't open auxiliary C++ hash file %s for writing", hash_name.c_str()); diff --git a/src/script_opt/CPP/HashMgr.h b/src/script_opt/CPP/HashMgr.h index 6a495b597e..2ae2e65ace 100644 --- a/src/script_opt/CPP/HashMgr.h +++ b/src/script_opt/CPP/HashMgr.h @@ -27,11 +27,9 @@ public: // end of the file (and the hash file will be locked, to prevent // overlapping updates from concurrent compilation/appends). // Otherwise, the file will be generated afresh. - CPPHashManager(const char* hash_name_base, bool append); + CPPHashManager(const char* hash_name_base); ~CPPHashManager(); - bool IsAppend() const { return append; } - // True if the given hash has already been generated. bool HasHash(p_hash_type h) const { return previously_compiled.count(h) > 0; } @@ -96,10 +94,6 @@ protected: // names, rather than their script-level names. std::unordered_map gv_scopes; - // Whether we're appending to existing hash file(s), or starting - // afresh. - bool append; - // Base for file names. std::string hash_name; diff --git a/src/script_opt/CPP/Inits.cc b/src/script_opt/CPP/Inits.cc index 48f3dd1cc1..61b1c18ca7 100644 --- a/src/script_opt/CPP/Inits.cc +++ b/src/script_opt/CPP/Inits.cc @@ -14,12 +14,30 @@ namespace zeek::detail using namespace std; -void CPPCompile::GenInitExpr(const ExprPtr& e) +std::shared_ptr CPPCompile::RegisterInitExpr(const ExprPtr& ep) + { + auto ename = InitExprName(ep); + + if ( init_infos.count(ename) ) + return init_infos[ename]; + + auto wrapper_cl = string("wrapper_") + ename + "_cl"; + + auto gi = make_shared(this, ep, ename, wrapper_cl); + call_exprs_info->AddInstance(gi); + init_infos[ename] = gi; + + return gi; + } + +void CPPCompile::GenInitExpr(std::shared_ptr ce_init) { NL(); + const auto& e = ce_init->GetExpr(); const auto& t = e->GetType(); - auto ename = InitExprName(e); + const auto& ename = ce_init->Name(); + const auto& wc = ce_init->WrapperClass(); // First, create a CPPFunc that we can compile to compute 'e'. auto name = string("wrapper_") + ename; @@ -29,18 +47,17 @@ void CPPCompile::GenInitExpr(const ExprPtr& e) // Create the Func subclass that can be used in a CallExpr to // evaluate 'e'. - Emit("class %s_cl : public CPPFunc", name); + Emit("class %s : public CPPFunc", wc); StartBlock(); Emit("public:"); - Emit("%s_cl() : CPPFunc(\"%s\", %s)", name, name, e->IsPure() ? "true" : "false"); + Emit("%s() : CPPFunc(\"%s\", %s)", wc, name, e->IsPure() ? "true" : "false"); StartBlock(); Emit("type = make_intrusive(make_intrusive(new type_decl_list()), %s, " "FUNC_FLAVOR_FUNCTION);", GenTypeName(t)); - NoteInitDependency(e, TypeRep(t)); EndBlock(); Emit("ValPtr Invoke(zeek::Args* args, Frame* parent) const override final"); @@ -62,15 +79,9 @@ void CPPCompile::GenInitExpr(const ExprPtr& e) EndBlock(); Emit("CallExprPtr %s;", ename); - - NoteInitDependency(e, TypeRep(t)); - AddInit(e, ename, - string("make_intrusive(make_intrusive(make_intrusive(" - "make_intrusive<") + - name + "_cl>())), make_intrusive(), false)"); } -bool CPPCompile::IsSimpleInitExpr(const ExprPtr& e) const +bool CPPCompile::IsSimpleInitExpr(const ExprPtr& e) { switch ( e->Tag() ) { @@ -101,360 +112,83 @@ string CPPCompile::InitExprName(const ExprPtr& e) return init_exprs.KeyName(e); } -void CPPCompile::GenGlobalInit(const ID* g, string& gl, const ValPtr& v) - { - const auto& t = v->GetType(); - auto tag = t->Tag(); - - if ( tag == TYPE_FUNC ) - // This should get initialized by recognizing hash of - // the function's body. - return; - - string init_val; - if ( tag == TYPE_OPAQUE ) - { - // We can only generate these by reproducing the expression - // (presumably a function call) used to create the value. - // That isn't fully sound, since if the global's value - // was redef'd in terms of its original value (e.g., - // "redef x = f(x)"), then we'll wind up with a broken - // expression. It's difficult to detect that in full - // generality, so um Don't Do That. (Note that this - // only affects execution of standalone compiled code, - // where the original scripts are replaced by load-stubs. - // If the scripts are available, then the HasVal() test - // we generate will mean we don't wind up using this - // expression anyway.) - - // Use the final initialization expression. - auto& init_exprs = g->GetOptInfo()->GetInitExprs(); - init_val = GenExpr(init_exprs.back(), GEN_VAL_PTR, false); - } - else - init_val = BuildConstant(g, v); - - auto& attrs = g->GetAttrs(); - - AddInit(g, string("if ( ! ") + gl + "->HasVal() )"); - - if ( attrs ) - { - RegisterAttributes(attrs); - - AddInit(g, "\t{"); - AddInit(g, "\t" + gl + "->SetVal(" + init_val + ");"); - AddInit(g, "\t" + gl + "->SetAttrs(" + AttrsName(attrs) + ");"); - AddInit(g, "\t}"); - } - else - AddInit(g, "\t" + gl + "->SetVal(" + init_val + ");"); - } - -void CPPCompile::GenFuncVarInits() - { - for ( const auto& fv_init : func_vars ) - { - auto& fv = fv_init.first; - auto& const_name = fv_init.second; - - auto f = fv->AsFunc(); - const auto& fn = f->Name(); - const auto& ft = f->GetType(); - - NoteInitDependency(fv, TypeRep(ft)); - - const auto& bodies = f->GetBodies(); - - string hashes = "{"; - - for ( const auto& b : bodies ) - { - auto body = b.stmts.get(); - - ASSERT(body_names.count(body) > 0); - - auto& body_name = body_names[body]; - ASSERT(body_hashes.count(body_name) > 0); - - NoteInitDependency(fv, body); - - if ( hashes.size() > 1 ) - hashes += ", "; - - hashes += Fmt(body_hashes[body_name]); - } - - hashes += "}"; - - auto init = string("lookup_func__CPP(\"") + fn + "\", " + hashes + ", " + GenTypeName(ft) + - ")"; - - AddInit(fv, const_name, init); - } - } - -void CPPCompile::GenPreInit(const Type* t) - { - string pre_init; - - switch ( t->Tag() ) - { - case TYPE_ADDR: - case TYPE_ANY: - case TYPE_BOOL: - case TYPE_COUNT: - case TYPE_DOUBLE: - case TYPE_ERROR: - case TYPE_INT: - case TYPE_INTERVAL: - case TYPE_PATTERN: - case TYPE_PORT: - case TYPE_STRING: - case TYPE_TIME: - case TYPE_TIMER: - case TYPE_VOID: - pre_init = string("base_type(") + TypeTagName(t->Tag()) + ")"; - break; - - case TYPE_ENUM: - pre_init = string("get_enum_type__CPP(\"") + t->GetName() + "\")"; - break; - - case TYPE_SUBNET: - pre_init = string("make_intrusive()"); - break; - - case TYPE_FILE: - pre_init = string("make_intrusive(") + GenTypeName(t->AsFileType()->Yield()) + - ")"; - break; - - case TYPE_OPAQUE: - pre_init = string("make_intrusive(\"") + t->AsOpaqueType()->Name() + "\")"; - break; - - case TYPE_RECORD: - { - string name; - - if ( t->GetName() != "" ) - name = string("\"") + t->GetName() + string("\""); - else - name = "nullptr"; - - pre_init = string("get_record_type__CPP(") + name + ")"; - } - break; - - case TYPE_LIST: - pre_init = string("make_intrusive()"); - break; - - case TYPE_TYPE: - case TYPE_VECTOR: - case TYPE_TABLE: - case TYPE_FUNC: - // Nothing to do for these, pre-initialization-wise. - return; - - default: - reporter->InternalError("bad type in CPPCompile::GenType"); - } - - pre_inits.emplace_back(GenTypeName(t) + " = " + pre_init + ";"); - } - -void CPPCompile::GenPreInits() - { - NL(); - Emit("void pre_init__CPP()"); - - StartBlock(); - for ( const auto& i : pre_inits ) - Emit(i); - EndBlock(); - } - -void CPPCompile::AddInit(const Obj* o, const string& init) - { - obj_inits[o].emplace_back(init); - } - -void CPPCompile::AddInit(const Obj* o) - { - if ( obj_inits.count(o) == 0 ) - obj_inits[o] = {}; - } - -void CPPCompile::NoteInitDependency(const Obj* o1, const Obj* o2) - { - obj_deps[o1].emplace(o2); - } - -void CPPCompile::CheckInitConsistency(unordered_set& to_do) - { - for ( const auto& od : obj_deps ) - { - const auto& o = od.first; - - if ( to_do.count(o) == 0 ) - { - fprintf(stderr, "object not in to_do: %s\n", obj_desc(o).c_str()); - exit(1); - } - - for ( const auto& d : od.second ) - { - if ( to_do.count(d) == 0 ) - { - fprintf(stderr, "dep object for %s not in to_do: %s\n", obj_desc(o).c_str(), - obj_desc(d).c_str()); - exit(1); - } - } - } - } - -int CPPCompile::GenDependentInits(unordered_set& to_do) - { - int n = 0; - - // The basic approach is fairly brute force: find elements of - // to_do that don't have any pending dependencies; generate those; - // and remove them from the to_do list, freeing up other to_do entries - // to now not having any pending dependencies. Iterate until there - // are no more to-do items. - while ( to_do.size() > 0 ) - { - unordered_set cohort; - - for ( const auto& o : to_do ) - { - const auto& od = obj_deps.find(o); - - bool has_pending_dep = false; - - if ( od != obj_deps.end() ) - { - for ( const auto& d : od->second ) - if ( to_do.count(d) > 0 ) - { - has_pending_dep = true; - break; - } - } - - if ( has_pending_dep ) - continue; - - cohort.insert(o); - } - - ASSERT(cohort.size() > 0); - - GenInitCohort(++n, cohort); - - for ( const auto& o : cohort ) - { - ASSERT(to_do.count(o) > 0); - to_do.erase(o); - } - } - - return n; - } - -void CPPCompile::GenInitCohort(int nc, unordered_set& cohort) - { - NL(); - Emit("void init_%s__CPP()", Fmt(nc)); - StartBlock(); - - // If any script/BiF functions are used for initializing globals, - // the code generated from that will expect the presence of a - // frame pointer, even if nil. - Emit("Frame* f__CPP = nullptr;"); - - // The following is just for making the output readable/pretty: - // add space between initializations for distinct objects, taking - // into account that some objects have empty initializations. - bool did_an_init = false; - - for ( auto o : cohort ) - { - if ( did_an_init ) - { - NL(); - did_an_init = false; - } - - for ( const auto& i : obj_inits.find(o)->second ) - { - Emit("%s", i); - did_an_init = true; - } - } - - EndBlock(); - } - void CPPCompile::InitializeFieldMappings() { - Emit("int fm_offset;"); + Emit("std::vector CPP__field_mappings__ = "); + + StartBlock(); for ( const auto& mapping : field_decls ) { - auto rt = mapping.first; + auto rt_arg = Fmt(mapping.first); auto td = mapping.second; - auto fn = td->id; - auto rt_name = GenTypeName(rt) + "->AsRecordType()"; + auto type_arg = Fmt(TypeOffset(td->type)); + auto attrs_arg = Fmt(AttributesOffset(td->attrs)); - Emit("fm_offset = %s->FieldOffset(\"%s\");", rt_name, fn); - Emit("if ( fm_offset < 0 )"); - - StartBlock(); - Emit("// field does not exist, create it"); - Emit("fm_offset = %s->NumFields();", rt_name); - Emit("type_decl_list tl;"); - Emit(GenTypeDecl(td)); - Emit("%s->AddFieldsDirectly(tl);", rt_name); - EndBlock(); - - Emit("field_mapping.push_back(fm_offset);"); + Emit("CPP_FieldMapping(%s, \"%s\", %s, %s),", rt_arg, td->id, type_arg, attrs_arg); } + + EndBlock(true); } void CPPCompile::InitializeEnumMappings() { - int n = 0; + Emit("std::vector CPP__enum_mappings__ = "); + + StartBlock(); for ( const auto& mapping : enum_names ) - InitializeEnumMappings(mapping.first, mapping.second, n++); + Emit("CPP_EnumMapping(%s, \"%s\"),", Fmt(mapping.first), mapping.second); + + EndBlock(true); } -void CPPCompile::InitializeEnumMappings(const EnumType* et, const string& e_name, int index) +void CPPCompile::InitializeBiFs() { - AddInit(et, "{"); + Emit("std::vector CPP__BiF_lookups__ = "); - auto et_name = GenTypeName(et) + "->AsEnumType()"; - AddInit(et, "int em_offset = " + et_name + "->Lookup(\"" + e_name + "\");"); - AddInit(et, "if ( em_offset < 0 )"); + StartBlock(); - AddInit(et, "\t{"); - AddInit(et, "\tem_offset = " + et_name + "->Names().size();"); - // The following is to catch the case where the offset is already - // in use due to it being specified explicitly for an existing enum. - AddInit(et, "\tif ( " + et_name + "->Lookup(em_offset) )"); - AddInit( - et, - "\t\treporter->InternalError(\"enum inconsistency while initializing compiled scripts\");"); - AddInit(et, "\t" + et_name + "->AddNameInternal(\"" + e_name + "\", em_offset);"); - AddInit(et, "\t}"); + for ( const auto& b : BiFs ) + Emit("CPP_LookupBiF(%s, \"%s\"),", b.first, b.second); - AddInit(et, "enum_mapping[" + Fmt(index) + "] = em_offset;"); + EndBlock(true); + } - AddInit(et, "}"); +void CPPCompile::InitializeStrings() + { + Emit("std::vector CPP__Strings ="); + + StartBlock(); + + for ( const auto& s : ordered_tracked_strings ) + Emit("\"%s\",", s); + + EndBlock(true); + } + +void CPPCompile::InitializeHashes() + { + Emit("std::vector CPP__Hashes ="); + + StartBlock(); + + for ( const auto& h : ordered_tracked_hashes ) + Emit(Fmt(h) + ","); + + EndBlock(true); + } + +void CPPCompile::InitializeConsts() + { + Emit("std::vector CPP__ConstVals ="); + + StartBlock(); + + for ( const auto& c : consts ) + Emit("CPP_ValElem(%s, %s),", TypeTagName(c.first), Fmt(c.second)); + + EndBlock(true); } void CPPCompile::GenInitHook() @@ -482,11 +216,13 @@ void CPPCompile::GenStandaloneActivation() { NL(); +#if 0 Emit("void standalone_activation__CPP()"); StartBlock(); for ( auto& a : activations ) Emit(a); EndBlock(); +#endif NL(); Emit("void standalone_init__CPP()"); diff --git a/src/script_opt/CPP/InitsInfo.cc b/src/script_opt/CPP/InitsInfo.cc new file mode 100644 index 0000000000..4c2685afaf --- /dev/null +++ b/src/script_opt/CPP/InitsInfo.cc @@ -0,0 +1,577 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/Desc.h" +#include "zeek/RE.h" +#include "zeek/ZeekString.h" +#include "zeek/script_opt/CPP/Attrs.h" +#include "zeek/script_opt/CPP/Compile.h" + +using namespace std; + +namespace zeek::detail + { + +string CPP_InitsInfo::Name(int index) const + { + return base_name + "[" + Fmt(index) + "]"; + } + +void CPP_InitsInfo::AddInstance(shared_ptr g) + { + auto init_cohort = g->InitCohort(); + + if ( static_cast(instances.size()) <= init_cohort ) + instances.resize(init_cohort + 1); + + g->SetOffset(this, size++); + + instances[init_cohort].push_back(move(g)); + } + +string CPP_InitsInfo::Declare() const + { + return string("std::vector<") + CPPType() + "> " + base_name + ";"; + } + +void CPP_InitsInfo::GenerateInitializers(CPPCompile* c) + { + BuildOffsetSet(c); + + c->NL(); + + auto gt = InitsType(); + + // Declare the initializer. + c->Emit("%s %s = %s(%s, %s,", gt, InitializersName(), gt, base_name, Fmt(offset_set)); + + c->IndentUp(); + c->Emit("{"); + + // Add each cohort as a vector element. + for ( auto& cohort : instances ) + { + c->Emit("{"); + BuildCohort(c, cohort); + c->Emit("},"); + } + + c->Emit("}"); + c->IndentDown(); + c->Emit(");"); + } + +void CPP_InitsInfo::BuildOffsetSet(CPPCompile* c) + { + vector offsets_vec; + + for ( auto& cohort : instances ) + { + // Reduce the offsets used by this cohort to an + // offset into the managed vector-of-indices global. + vector offsets; + offsets.reserve(cohort.size()); + for ( auto& co : cohort ) + offsets.push_back(co->Offset()); + + offsets_vec.push_back(c->IndMgr().AddIndices(offsets)); + } + + // Now that we have all the offsets in a vector, reduce them, too, + // to an offset into the managed vector-of-indices global, + offset_set = c->IndMgr().AddIndices(offsets_vec); + } + +void CPP_InitsInfo::BuildCohort(CPPCompile* c, std::vector>& cohort) + { + for ( auto& co : cohort ) + { + vector ivs; + co->InitializerVals(ivs); + BuildCohortElement(c, co->InitializerType(), ivs); + } + } + +void CPP_InitsInfo::BuildCohortElement(CPPCompile* c, string init_type, vector& ivs) + { + string full_init; + bool did_one = false; + for ( auto& iv : ivs ) + { + if ( did_one ) + full_init += ", "; + else + did_one = true; + + full_init += iv; + } + + c->Emit("std::make_shared<%s>(%s),", init_type, full_init); + } + +void CPP_CompoundInitsInfo::BuildCohortElement(CPPCompile* c, string init_type, vector& ivs) + { + string init_line; + for ( auto& iv : ivs ) + init_line += iv + ", "; + + c->Emit("{ %s},", init_line); + } + +void CPP_BasicConstInitsInfo::BuildCohortElement(CPPCompile* c, string init_type, + vector& ivs) + { + ASSERT(ivs.size() == 1); + c->Emit(ivs[0] + ","); + } + +string CPP_InitInfo::ValElem(CPPCompile* c, ValPtr v) + { + string init_type; + string init_args; + + if ( v ) + { + int consts_offset; + auto gi = c->RegisterConstant(v, consts_offset); + init_cohort = max(init_cohort, gi->InitCohort() + 1); + return Fmt(consts_offset); + } + else + return Fmt(-1); + } + +DescConstInfo::DescConstInfo(CPPCompile* c, ValPtr v) : CPP_InitInfo() + { + ODesc d; + v->Describe(&d); + auto s = c->TrackString(d.Description()); + init = Fmt(s); + } + +EnumConstInfo::EnumConstInfo(CPPCompile* c, ValPtr v) + { + auto ev = v->AsEnumVal(); + auto& ev_t = ev->GetType(); + e_type = c->TypeOffset(ev_t); + init_cohort = c->TypeCohort(ev_t) + 1; + e_val = v->AsEnum(); + } + +StringConstInfo::StringConstInfo(CPPCompile* c, ValPtr v) : CPP_InitInfo() + { + auto s = v->AsString(); + const char* b = (const char*)(s->Bytes()); + + len = s->Len(); + chars = c->TrackString(CPPEscape(b, len)); + } + +PatternConstInfo::PatternConstInfo(CPPCompile* c, ValPtr v) : CPP_InitInfo() + { + auto re = v->AsPatternVal()->Get(); + pattern = c->TrackString(CPPEscape(re->OrigText())); + is_case_insensitive = re->IsCaseInsensitive(); + } + +CompoundItemInfo::CompoundItemInfo(CPPCompile* _c, ValPtr v) : CPP_InitInfo(), c(_c) + { + auto& t = v->GetType(); + type = c->TypeOffset(t); + init_cohort = c->TypeCohort(t) + 1; + } + +ListConstInfo::ListConstInfo(CPPCompile* _c, ValPtr v) : CompoundItemInfo(_c) + { + auto lv = cast_intrusive(v); + auto n = lv->Length(); + + for ( auto i = 0U; i < n; ++i ) + vals.emplace_back(ValElem(c, lv->Idx(i))); + } + +VectorConstInfo::VectorConstInfo(CPPCompile* c, ValPtr v) : CompoundItemInfo(c, v) + { + auto vv = cast_intrusive(v); + auto n = vv->Size(); + + for ( auto i = 0; i < n; ++i ) + vals.emplace_back(ValElem(c, vv->ValAt(i))); + } + +RecordConstInfo::RecordConstInfo(CPPCompile* c, ValPtr v) : CompoundItemInfo(c, v) + { + auto r = cast_intrusive(v); + auto n = r->NumFields(); + + type = c->TypeOffset(r->GetType()); + + for ( auto i = 0; i < n; ++i ) + vals.emplace_back(ValElem(c, r->GetField(i))); + } + +TableConstInfo::TableConstInfo(CPPCompile* c, ValPtr v) : CompoundItemInfo(c, v) + { + auto tv = cast_intrusive(v); + + for ( auto& tv_i : tv->ToMap() ) + { + vals.emplace_back(ValElem(c, tv_i.first)); // index + vals.emplace_back(ValElem(c, tv_i.second)); // value + } + } + +FileConstInfo::FileConstInfo(CPPCompile* c, ValPtr v) : CompoundItemInfo(c, v) + { + auto fv = cast_intrusive(v); + auto fname = c->TrackString(fv->Get()->Name()); + vals.emplace_back(Fmt(fname)); + } + +FuncConstInfo::FuncConstInfo(CPPCompile* _c, ValPtr v) : CompoundItemInfo(_c, v), fv(v->AsFuncVal()) + { + // This is slightly hacky. There's a chance that this constant + // depends on a lambda being registered. Here we use the knowledge + // that LambdaRegistrationInfo sets its cohort to 1 more than + // the function type, so we can ensure any possible lambda has + // been registered by setting ours to 2 more. CompoundItemInfo + // has already set our cohort to 1 more. + ++init_cohort; + } + +void FuncConstInfo::InitializerVals(std::vector& ivs) const + { + auto f = fv->AsFunc(); + const auto& fn = f->Name(); + + ivs.emplace_back(Fmt(type)); + ivs.emplace_back(Fmt(c->TrackString(fn))); + + string hashes; + + if ( ! c->NotFullyCompilable(fn) ) + { + const auto& bodies = f->GetBodies(); + + for ( const auto& b : bodies ) + { + auto h = c->BodyHash(b.stmts.get()); + auto h_o = c->TrackHash(h); + ivs.emplace_back(Fmt(h_o)); + } + } + } + +AttrInfo::AttrInfo(CPPCompile* _c, const AttrPtr& attr) : CompoundItemInfo(_c) + { + vals.emplace_back(Fmt(static_cast(attr->Tag()))); + auto a_e = attr->GetExpr(); + + if ( a_e ) + { + auto gi = c->RegisterType(a_e->GetType()); + init_cohort = max(init_cohort, gi->InitCohort() + 1); + + if ( ! CPPCompile::IsSimpleInitExpr(a_e) ) + { + gi = c->RegisterInitExpr(a_e); + init_cohort = max(init_cohort, gi->InitCohort() + 1); + + vals.emplace_back(Fmt(static_cast(AE_CALL))); + vals.emplace_back(Fmt(gi->Offset())); + } + + else if ( a_e->Tag() == EXPR_CONST ) + { + auto v = a_e->AsConstExpr()->ValuePtr(); + vals.emplace_back(Fmt(static_cast(AE_CONST))); + vals.emplace_back(ValElem(c, v)); + } + + else if ( a_e->Tag() == EXPR_NAME ) + { + auto g = a_e->AsNameExpr()->Id(); + auto gi = c->RegisterGlobal(g); + init_cohort = max(init_cohort, gi->InitCohort() + 1); + + vals.emplace_back(Fmt(static_cast(AE_NAME))); + vals.emplace_back(Fmt(c->TrackString(g->Name()))); + } + + else + { + ASSERT(a_e->Tag() == EXPR_RECORD_COERCE); + vals.emplace_back(Fmt(static_cast(AE_RECORD))); + vals.emplace_back(Fmt(gi->Offset())); + } + } + + else + vals.emplace_back(Fmt(static_cast(AE_NONE))); + } + +AttrsInfo::AttrsInfo(CPPCompile* _c, const AttributesPtr& _attrs) : CompoundItemInfo(_c) + { + for ( const auto& a : _attrs->GetAttrs() ) + { + ASSERT(c->ProcessedAttr().count(a.get()) > 0); + auto gi = c->ProcessedAttr()[a.get()]; + init_cohort = max(init_cohort, gi->InitCohort() + 1); + vals.emplace_back(Fmt(gi->Offset())); + } + } + +GlobalInitInfo::GlobalInitInfo(CPPCompile* c, const ID* g, string _CPP_name) + : CPP_InitInfo(), CPP_name(move(_CPP_name)) + { + Zeek_name = g->Name(); + + auto gi = c->RegisterType(g->GetType()); + init_cohort = max(init_cohort, gi->InitCohort() + 1); + type = gi->Offset(); + + gi = c->RegisterAttributes(g->GetAttrs()); + if ( gi ) + { + init_cohort = max(init_cohort, gi->InitCohort() + 1); + attrs = gi->Offset(); + } + else + attrs = -1; + + exported = g->IsExport(); + + val = ValElem(c, g->GetVal()); + } + +void GlobalInitInfo::InitializerVals(std::vector& ivs) const + { + ivs.push_back(CPP_name); + ivs.push_back(string("\"") + Zeek_name + "\""); + ivs.push_back(Fmt(type)); + ivs.push_back(Fmt(attrs)); + ivs.push_back(val); + ivs.push_back(Fmt(exported)); + } + +CallExprInitInfo::CallExprInitInfo(CPPCompile* c, ExprPtr _e, string _e_name, string _wrapper_class) + : e(move(_e)), e_name(move(_e_name)), wrapper_class(move(_wrapper_class)) + { + auto gi = c->RegisterType(e->GetType()); + init_cohort = max(init_cohort, gi->InitCohort() + 1); + } + +LambdaRegistrationInfo::LambdaRegistrationInfo(CPPCompile* c, string _name, FuncTypePtr ft, + string _wrapper_class, p_hash_type _h, + bool _has_captures) + : name(move(_name)), wrapper_class(move(_wrapper_class)), h(_h), has_captures(_has_captures) + { + auto gi = c->RegisterType(ft); + init_cohort = max(init_cohort, gi->InitCohort() + 1); + func_type = gi->Offset(); + } + +void LambdaRegistrationInfo::InitializerVals(std::vector& ivs) const + { + ivs.emplace_back(string("\"") + name + "\""); + ivs.emplace_back(Fmt(func_type)); + ivs.emplace_back(Fmt(h)); + ivs.emplace_back(has_captures ? "true" : "false"); + } + +void EnumTypeInfo::AddInitializerVals(std::vector& ivs) const + { + ivs.emplace_back(Fmt(c->TrackString(t->GetName()))); + + auto et = t->AsEnumType(); + + for ( const auto& name_pair : et->Names() ) + { + ivs.emplace_back(Fmt(c->TrackString(name_pair.first))); + ivs.emplace_back(Fmt(int(name_pair.second))); + } + } + +void OpaqueTypeInfo::AddInitializerVals(std::vector& ivs) const + { + ivs.emplace_back(Fmt(c->TrackString(t->GetName()))); + } + +TypeTypeInfo::TypeTypeInfo(CPPCompile* _c, TypePtr _t) : AbstractTypeInfo(_c, move(_t)) + { + tt = t->AsTypeType()->GetType(); + auto gi = c->RegisterType(tt); + if ( gi ) + init_cohort = gi->InitCohort(); + } + +void TypeTypeInfo::AddInitializerVals(std::vector& ivs) const + { + ivs.emplace_back(to_string(c->TypeOffset(tt))); + } + +VectorTypeInfo::VectorTypeInfo(CPPCompile* _c, TypePtr _t) : AbstractTypeInfo(_c, move(_t)) + { + yield = t->Yield(); + auto gi = c->RegisterType(yield); + if ( gi ) + init_cohort = gi->InitCohort(); + } + +void VectorTypeInfo::AddInitializerVals(std::vector& ivs) const + { + ivs.emplace_back(to_string(c->TypeOffset(yield))); + } + +ListTypeInfo::ListTypeInfo(CPPCompile* _c, TypePtr _t) + : AbstractTypeInfo(_c, move(_t)), types(t->AsTypeList()->GetTypes()) + { + for ( auto& tl_i : types ) + { + auto gi = c->RegisterType(tl_i); + if ( gi ) + init_cohort = max(init_cohort, gi->InitCohort()); + } + } + +void ListTypeInfo::AddInitializerVals(std::vector& ivs) const + { + string type_list; + for ( auto& t : types ) + ivs.emplace_back(Fmt(c->TypeOffset(t))); + } + +TableTypeInfo::TableTypeInfo(CPPCompile* _c, TypePtr _t) : AbstractTypeInfo(_c, move(_t)) + { + auto tbl = t->AsTableType(); + + auto gi = c->RegisterType(tbl->GetIndices()); + ASSERT(gi); + indices = gi->Offset(); + init_cohort = gi->InitCohort(); + + yield = tbl->Yield(); + + if ( yield ) + { + gi = c->RegisterType(yield); + if ( gi ) + init_cohort = max(init_cohort, gi->InitCohort()); + } + } + +void TableTypeInfo::AddInitializerVals(std::vector& ivs) const + { + ivs.emplace_back(Fmt(indices)); + ivs.emplace_back(Fmt(yield ? c->TypeOffset(yield) : -1)); + } + +FuncTypeInfo::FuncTypeInfo(CPPCompile* _c, TypePtr _t) : AbstractTypeInfo(_c, move(_t)) + { + auto f = t->AsFuncType(); + + flavor = f->Flavor(); + params = f->Params(); + yield = f->Yield(); + + auto gi = c->RegisterType(f->Params()); + if ( gi ) + init_cohort = gi->InitCohort(); + + if ( yield ) + { + gi = c->RegisterType(f->Yield()); + if ( gi ) + init_cohort = max(init_cohort, gi->InitCohort()); + } + } + +void FuncTypeInfo::AddInitializerVals(std::vector& ivs) const + { + ivs.emplace_back(Fmt(c->TypeOffset(params))); + ivs.emplace_back(Fmt(yield ? c->TypeOffset(yield) : -1)); + ivs.emplace_back(Fmt(static_cast(flavor))); + } + +RecordTypeInfo::RecordTypeInfo(CPPCompile* _c, TypePtr _t) : AbstractTypeInfo(_c, move(_t)) + { + auto r = t->AsRecordType()->Types(); + + if ( ! r ) + return; + + for ( const auto& r_i : *r ) + { + field_names.emplace_back(r_i->id); + + auto gi = c->RegisterType(r_i->type); + if ( gi ) + init_cohort = max(init_cohort, gi->InitCohort()); + // else it's a recursive type, no need to adjust cohort here + + field_types.push_back(r_i->type); + + if ( r_i->attrs ) + { + gi = c->RegisterAttributes(r_i->attrs); + init_cohort = max(init_cohort, gi->InitCohort() + 1); + field_attrs.push_back(gi->Offset()); + } + else + field_attrs.push_back(-1); + } + } + +void RecordTypeInfo::AddInitializerVals(std::vector& ivs) const + { + ivs.emplace_back(Fmt(c->TrackString(t->GetName()))); + + auto n = field_names.size(); + + for ( auto i = 0U; i < n; ++i ) + { + ivs.emplace_back(Fmt(c->TrackString(field_names[i]))); + + // Because RecordType's can be recursively defined, + // during construction we couldn't reliably access + // the field type's offsets. At this point, though, + // they should all be available. + ivs.emplace_back(Fmt(c->TypeOffset(field_types[i]))); + ivs.emplace_back(Fmt(field_attrs[i])); + } + } + +void IndicesManager::Generate(CPPCompile* c) + { + c->Emit("int CPP__Indices__init[] ="); + c->StartBlock(); + + int nset = 0; + for ( auto& is : indices_set ) + { + // Track the offsets into the raw vector, to make it + // easier to debug problems. + auto line = string("/* ") + to_string(nset++) + " */ "; + + // We first record the size, then the values. + line += to_string(is.size()) + ", "; + + auto n = 1; + for ( auto i : is ) + { + line += to_string(i) + ", "; + if ( ++n % 10 == 0 ) + { + c->Emit(line); + line.clear(); + } + } + + if ( line.size() > 0 ) + c->Emit(line); + } + + c->Emit("-1"); + c->EndBlock(true); + } + + } // zeek::detail diff --git a/src/script_opt/CPP/InitsInfo.h b/src/script_opt/CPP/InitsInfo.h new file mode 100644 index 0000000000..b8453b8135 --- /dev/null +++ b/src/script_opt/CPP/InitsInfo.h @@ -0,0 +1,693 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +// Classes for tracking information for initializing C++ values used by the +// generated code. + +// Initialization is probably the most complex part of the entire compiler, +// as there are a lot of considerations. There are two basic parts: (1) the +// generation of C++ code for doing run-time initialization, which is covered +// by the classes in this file, and (2) the execution of that code to do the +// actual initialization, which is covered by the classes in RuntimeInits.h. +// +// There are two fundamental types of initialization, those that create values +// (such as Zeek Type and Val objects) that will be used during the execution +// of compiled scripts, and those that perform actions such as registering +// the presence of a global or a lambda. In addition, for the former (values +// used at run-time), some are grouped together into vectors, with the compiled +// code using a hardwired index to get to a particular value; and some have +// standalone globals (for example, one for each BiF that a compiled script +// may call). +// +// For each of these types of initialization, our general approach is to a +// class that manages a single instance of that type, and an an object that +// manages all of those instances collectively. The latter object will, for +// example, attend to determining the offset into the run-time vector associated +// with a particular initialized value. +// +// An additional complexity is that often the initialization of a particular +// value will depend on *other* values having already been initialized. For +// example, a record type might have a field that is a table, and thus the +// type corresponding to the table needs to be available before we can create +// the record type. However, the table might have a set of attributes +// associated with it, which have to be initialized before we can create the +// table type, those in turn requiring the initialization of each of the +// individual attributes in the set. One of those attributes might specify +// a &default function for the table, requiring initializing *that* value +// (not just the type, but also a way to refer to the particular instance of +// the function) before initializing the attribute, etc. Worse, record types +// can be *indirectly recursive*, which requires first initializing a "stub" +// for the record type before doing the final initialization. +// +// The general strategy for dealing with all of these dependencies is to +// compute for each initialization its "cohort". An initialization that +// doesn't depend on any others is in cohort 0. An initialization X that +// depends on an initialization Y will have cohort(X) = cohort(Y) + 1; or, +// in general, one more than the highest cohort of any initialization it +// depends on. (We cut a corner in that, due to how initialization information +// is constructed, if X and Y are for the same type of object then we can +// safely use cohort(X) = cohort(Y).) We then execute run-time initialization +// in waves, one cohort at a time. +// +// Because C++ compilers can struggle when trying to optimize large quantities +// of code - clang in particular could take many CPU *hours* back when our +// compiler just generated C++ code snippets for each initialization - rather +// than producing code that directly executes each given initialization, we +// instead employ a table-driven approach. The C++ initializers for the +// tables contain simple values - often just vectors of integers - that compile +// quickly. At run-time we then spin through the elements of the tables (one +// cohort at a time) to obtain the information needed to initialize any given +// item. +// +// Many forms of initialization are specified in terms of indices into globals +// that hold items of various types. Thus, the most common initialization +// information is a vector of integers/indices. These data structures can +// be recursive, too, namely we sometimes associate an index with a vector +// of integers/indices and then we can track multiple such vectors using +// another vector of integers/indices. + +#include "zeek/File.h" +#include "zeek/Val.h" +#include "zeek/script_opt/ProfileFunc.h" + +#pragma once + +namespace zeek::detail + { + +class CPPCompile; + +// Abstract class for tracking information about a single initialization item. +class CPP_InitInfo; + +// Abstract class for tracking information about a collection of initialization +// items. +class CPP_InitsInfo + { +public: + CPP_InitsInfo(std::string _tag, std::string type) : tag(std::move(_tag)) + { + base_name = std::string("CPP__") + tag + "__"; + CPP_type = tag + type; + } + + virtual ~CPP_InitsInfo() { } + + // Returns the name of the C++ global that will hold the items' values + // at run-time, once initialized. These are all vectors, for which + // the generated code accesses a particular item by indexing the vector. + const std::string& InitsName() const { return base_name; } + + // Returns the name of the C++ global used to hold the table we employ + // for table-driven initialization. + std::string InitializersName() const { return base_name + "init"; } + + // Returns the "name" of the given element in the run-time vector + // associated with this collection of initialization items. It's not + // really a name but rather a vector index, so for example Name(12) + // might return "CPP__Pattern__[12]", but we use the term Name because + // the representation used to be individualized globals, such as + // "CPP__Pattern__12". + std::string Name(int index) const; + + // Returns the name that will correspond to the next item added to + // this set. + std::string NextName() const { return Name(size); } + + // The largest initialization cohort of any item in this collection. + int MaxCohort() const { return static_cast(instances.size()) - 1; } + + // Returns the number of initializations in this collection that below + // to the given cohort c. + int CohortSize(int c) const { return c > MaxCohort() ? 0 : instances[c].size(); } + + // Returns the C++ type associated with this collection's run-time vector. + // This might be, for example, "PatternVal" + const std::string& CPPType() const { return CPP_type; } + + // Sets the associated C++ type. + virtual void SetCPPType(std::string ct) { CPP_type = std::move(ct); } + + // Returns the type associated with the table used for initialization + // (i.e., this is the type of the global returned by InitializersName()). + std::string InitsType() const { return inits_type; } + + // Add a new initialization instance to the collection. + void AddInstance(std::shared_ptr g); + + // Emit code to populate the table used to initialize this collection. + void GenerateInitializers(CPPCompile* c); + +protected: + // Computes offset_set - see below. + void BuildOffsetSet(CPPCompile* c); + + // Returns a declaration suitable for the run-time vector that holds + // the initialized items in the collection. + std::string Declare() const; + + // For a given cohort, generates the associated table elements for + // creating it. + void BuildCohort(CPPCompile* c, std::vector>& cohort); + + // Given the initialization type and initializers for with a given + // cohort element, build the associated table element. + virtual void BuildCohortElement(CPPCompile* c, std::string init_type, + std::vector& ivs); + + // Total number of initializers. + int size = 0; + + // Each cohort is represented by a vector whose elements correspond + // to the initialization information for a single item. This variable + // holds a vector of cohorts, indexed by the number of the cohort. + // (Note, some cohorts may be empty.) + std::vector>> instances; + + // Each cohort has associated with it a vector of offsets, specifying + // positions in the run-time vector of the items in the cohort. + // + // We reduce each such vector to an index into the collection of + // such vectors (as managed by an IndicesManager - see below). + // + // Once we've done that reduction, we can represent each cohort + // using a single index, and thus all of the cohorts using a vector + // of indices. We then reduce *that* vector to a single index, + // again using the IndicesManager. We store that single index + // in the "offset_set" variable. + int offset_set = 0; + + // Tag used to distinguish a particular collection of constants. + std::string tag; + + // C++ name for this collection of constants. + std::string base_name; + + // C++ type associated with a single instance of these constants. + std::string CPP_type; + + // C++ type associated with the collection of initializers. + std::string inits_type; + }; + +// A class for a collection of initialization items for which each item +// has a "custom" initializer (that is, a bespoke C++ object, rather than +// a simple C++ type or a vector of indices). +class CPP_CustomInitsInfo : public CPP_InitsInfo + { +public: + CPP_CustomInitsInfo(std::string _tag, std::string _type) + : CPP_InitsInfo(std::move(_tag), std::move(_type)) + { + BuildInitType(); + } + + void SetCPPType(std::string ct) override + { + CPP_InitsInfo::SetCPPType(std::move(ct)); + BuildInitType(); + } + +private: + void BuildInitType() { inits_type = std::string("CPP_CustomInits<") + CPPType() + ">"; } + }; + +// A class for a collection of initialization items corresponding to "basic" +// constants, i.e., those that can be represented either directly as C++ +// constants, or as indices into a vector of C++ objects. +class CPP_BasicConstInitsInfo : public CPP_CustomInitsInfo + { +public: + // In the following, if "c_type" is non-empty then it specifes the + // C++ type used to directly represent the constant. If empty, it + // indicates that we instead use an index into a separate vector. + CPP_BasicConstInitsInfo(std::string _tag, std::string type, std::string c_type) + : CPP_CustomInitsInfo(std::move(_tag), std::move(type)) + { + if ( c_type.empty() ) + inits_type = std::string("CPP_") + tag + "Consts"; + else + inits_type = std::string("CPP_BasicConsts<") + CPP_type + ", " + c_type + ", " + tag + + "Val>"; + } + + void BuildCohortElement(CPPCompile* c, std::string init_type, + std::vector& ivs) override; + }; + +// A class for a collection of initialization items that are defined using +// other initialization items. +class CPP_CompoundInitsInfo : public CPP_InitsInfo + { +public: + CPP_CompoundInitsInfo(std::string _tag, std::string type) + : CPP_InitsInfo(std::move(_tag), std::move(type)) + { + if ( tag == "Type" ) + // These need a refined version of CPP_IndexedInits + // in order to build different types dynamically. + inits_type = "CPP_TypeInits"; + else + inits_type = std::string("CPP_IndexedInits<") + CPPType() + ">"; + } + + void BuildCohortElement(CPPCompile* c, std::string init_type, + std::vector& ivs) override; + }; + +// Abstract class for tracking information about a single initialization item. +class CPP_InitInfo + { +public: + // No constructor - basic initialization happens when the object is + // added via AddInstance() to a CPP_InitsInfo object, which in turn + // will lead to invocation of this object's SetOffset() method. + + virtual ~CPP_InitInfo() { } + + // Associates this item with an initialization collection and run-time + // vector offset. + void SetOffset(const CPP_InitsInfo* _inits_collection, int _offset) + { + inits_collection = _inits_collection; + offset = _offset; + } + + // Returns the offset for this item into the associated run-time vector. + int Offset() const { return offset; } + + // Returns the name that should be used for referring to this + // value in the generated code. + std::string Name() const { return inits_collection->Name(offset); } + + // Returns this item's initialization cohort. + int InitCohort() const { return init_cohort; } + + // Returns the type used for this initializer. + virtual std::string InitializerType() const { return ""; } + + // Returns values used for creating this value, one element per + // constructor parameter. + virtual void InitializerVals(std::vector& ivs) const = 0; + +protected: + // Returns an offset (into the run-time vector holding all Zeek + // constant values) corresponding to the given value. Registers + // the constant if needed. + std::string ValElem(CPPCompile* c, ValPtr v); + + // By default, values have no dependencies on other values + // being first initialized. Those that do must increase this + // value in their constructors. + int init_cohort = 0; + + // Tracks the collection to which this item belongs. + const CPP_InitsInfo* inits_collection = nullptr; + + // Offset of this item in the collection, or -1 if no association. + int offset = -1; + }; + +// Information associated with initializing a basic (non-compound) constant. +class BasicConstInfo : public CPP_InitInfo + { +public: + BasicConstInfo(std::string _val) : val(std::move(_val)) { } + + void InitializerVals(std::vector& ivs) const override { ivs.emplace_back(val); } + +private: + // All we need to track is the C++ representation of the constant. + std::string val; + }; + +// Information associated with initializing a constant whose Val constructor +// takes a string. +class DescConstInfo : public CPP_InitInfo + { +public: + DescConstInfo(CPPCompile* c, ValPtr v); + + void InitializerVals(std::vector& ivs) const override { ivs.emplace_back(init); } + +private: + std::string init; + }; + +class EnumConstInfo : public CPP_InitInfo + { +public: + EnumConstInfo(CPPCompile* c, ValPtr v); + + void InitializerVals(std::vector& ivs) const override + { + ivs.emplace_back(std::to_string(e_type)); + ivs.emplace_back(std::to_string(e_val)); + } + +private: + int e_type; // an index into the enum's Zeek type + int e_val; // integer value of the enum + }; + +class StringConstInfo : public CPP_InitInfo + { +public: + StringConstInfo(CPPCompile* c, ValPtr v); + + void InitializerVals(std::vector& ivs) const override + { + ivs.emplace_back(std::to_string(chars)); + ivs.emplace_back(std::to_string(len)); + } + +private: + int chars; // index into vector of char*'s + int len; // length of the string + }; + +class PatternConstInfo : public CPP_InitInfo + { +public: + PatternConstInfo(CPPCompile* c, ValPtr v); + + void InitializerVals(std::vector& ivs) const override + { + ivs.emplace_back(std::to_string(pattern)); + ivs.emplace_back(std::to_string(is_case_insensitive)); + } + +private: + int pattern; // index into string representation of pattern + int is_case_insensitive; // case-insensitivity flag, 0 or 1 + }; + +class PortConstInfo : public CPP_InitInfo + { +public: + PortConstInfo(ValPtr v) : p(static_cast(v->AsPortVal())->Get()) { } + + void InitializerVals(std::vector& ivs) const override + { + ivs.emplace_back(std::to_string(p) + "U"); + } + +private: + bro_uint_t p; + }; + +// Abstract class for compound items (those defined in terms of other items). +class CompoundItemInfo : public CPP_InitInfo + { +public: + // The first of these is used for items with custom Zeek types, + // the second when the type is generic/inapplicable. + CompoundItemInfo(CPPCompile* c, ValPtr v); + CompoundItemInfo(CPPCompile* _c) : c(_c) { type = -1; } + + void InitializerVals(std::vector& ivs) const override + { + if ( type >= 0 ) + ivs.emplace_back(std::to_string(type)); + + for ( auto& v : vals ) + ivs.push_back(v); + } + +protected: + CPPCompile* c; + int type; + std::vector vals; // initialization values + }; + +// This next set corresponds to compound Zeek constants of various types. +class ListConstInfo : public CompoundItemInfo + { +public: + ListConstInfo(CPPCompile* c, ValPtr v); + }; + +class VectorConstInfo : public CompoundItemInfo + { +public: + VectorConstInfo(CPPCompile* c, ValPtr v); + }; + +class RecordConstInfo : public CompoundItemInfo + { +public: + RecordConstInfo(CPPCompile* c, ValPtr v); + }; + +class TableConstInfo : public CompoundItemInfo + { +public: + TableConstInfo(CPPCompile* c, ValPtr v); + }; + +class FileConstInfo : public CompoundItemInfo + { +public: + FileConstInfo(CPPCompile* c, ValPtr v); + }; + +class FuncConstInfo : public CompoundItemInfo + { +public: + FuncConstInfo(CPPCompile* _c, ValPtr v); + + void InitializerVals(std::vector& ivs) const override; + +private: + FuncVal* fv; + }; + +// Initialization information for single attributes and sets of attributes. +class AttrInfo : public CompoundItemInfo + { +public: + AttrInfo(CPPCompile* c, const AttrPtr& attr); + }; + +class AttrsInfo : public CompoundItemInfo + { +public: + AttrsInfo(CPPCompile* c, const AttributesPtr& attrs); + }; + +// Information for initialization a Zeek global. +class GlobalInitInfo : public CPP_InitInfo + { +public: + GlobalInitInfo(CPPCompile* c, const ID* g, std::string CPP_name); + + std::string InitializerType() const override { return "CPP_GlobalInit"; } + void InitializerVals(std::vector& ivs) const override; + +protected: + std::string Zeek_name; + std::string CPP_name; + int type; + int attrs; + std::string val; + bool exported; + }; + +// Information for initializing an item corresponding to a Zeek function +// call, needed to associate complex expressions with attributes. +class CallExprInitInfo : public CPP_InitInfo + { +public: + CallExprInitInfo(CPPCompile* c, ExprPtr e, std::string e_name, std::string wrapper_class); + + std::string InitializerType() const override + { + return std::string("CPP_CallExprInit<") + wrapper_class + ">"; + } + void InitializerVals(std::vector& ivs) const override { ivs.emplace_back(e_name); } + + // Accessors, since code to initialize these is generated separately + // from that of most initialization collections. + const ExprPtr& GetExpr() const { return e; } + const std::string& Name() const { return e_name; } + const std::string& WrapperClass() const { return wrapper_class; } + +protected: + ExprPtr e; + std::string e_name; + std::string wrapper_class; + }; + +// Information for registering the class/function assocaited with a lambda. +class LambdaRegistrationInfo : public CPP_InitInfo + { +public: + LambdaRegistrationInfo(CPPCompile* c, std::string name, FuncTypePtr ft, + std::string wrapper_class, p_hash_type h, bool has_captures); + + std::string InitializerType() const override + { + return std::string("CPP_LambdaRegistration<") + wrapper_class + ">"; + } + void InitializerVals(std::vector& ivs) const override; + +protected: + std::string name; + int func_type; + std::string wrapper_class; + p_hash_type h; + bool has_captures; + }; + +// Abstract class for representing information for initializing a Zeek type. +class AbstractTypeInfo : public CPP_InitInfo + { +public: + AbstractTypeInfo(CPPCompile* _c, TypePtr _t) : c(_c), t(std::move(_t)) { } + + void InitializerVals(std::vector& ivs) const override + { + ivs.emplace_back(std::to_string(static_cast(t->Tag()))); + AddInitializerVals(ivs); + } + + virtual void AddInitializerVals(std::vector& ivs) const { } + +protected: + CPPCompile* c; + TypePtr t; // the type we're initializing + }; + +// The following capture information for different Zeek types. +class BaseTypeInfo : public AbstractTypeInfo + { +public: + BaseTypeInfo(CPPCompile* _c, TypePtr _t) : AbstractTypeInfo(_c, std::move(_t)) { } + }; + +class EnumTypeInfo : public AbstractTypeInfo + { +public: + EnumTypeInfo(CPPCompile* _c, TypePtr _t) : AbstractTypeInfo(_c, std::move(_t)) { } + + void AddInitializerVals(std::vector& ivs) const override; + }; + +class OpaqueTypeInfo : public AbstractTypeInfo + { +public: + OpaqueTypeInfo(CPPCompile* _c, TypePtr _t) : AbstractTypeInfo(_c, std::move(_t)) { } + + void AddInitializerVals(std::vector& ivs) const override; + }; + +class TypeTypeInfo : public AbstractTypeInfo + { +public: + TypeTypeInfo(CPPCompile* c, TypePtr _t); + + void AddInitializerVals(std::vector& ivs) const override; + +private: + TypePtr tt; // the type referred to by t + }; + +class VectorTypeInfo : public AbstractTypeInfo + { +public: + VectorTypeInfo(CPPCompile* c, TypePtr _t); + + void AddInitializerVals(std::vector& ivs) const override; + +private: + TypePtr yield; + }; + +class ListTypeInfo : public AbstractTypeInfo + { +public: + ListTypeInfo(CPPCompile* c, TypePtr _t); + + void AddInitializerVals(std::vector& ivs) const override; + +private: + const std::vector& types; + }; + +class TableTypeInfo : public AbstractTypeInfo + { +public: + TableTypeInfo(CPPCompile* c, TypePtr _t); + + void AddInitializerVals(std::vector& ivs) const override; + +private: + int indices; + TypePtr yield; + }; + +class FuncTypeInfo : public AbstractTypeInfo + { +public: + FuncTypeInfo(CPPCompile* c, TypePtr _t); + + void AddInitializerVals(std::vector& ivs) const override; + +private: + FunctionFlavor flavor; + TypePtr params; + TypePtr yield; + }; + +class RecordTypeInfo : public AbstractTypeInfo + { +public: + RecordTypeInfo(CPPCompile* c, TypePtr _t); + + void AddInitializerVals(std::vector& ivs) const override; + +private: + std::vector field_names; + std::vector field_types; + std::vector field_attrs; + }; + +// Much of the table-driven initialization is based on vectors of indices, +// which we represent as vectors of int's, where each int is used to index a +// global C++ vector. This class manages such vectors. In particular, it +// reduces a given vector-of-indices to a single value, itself an index, that +// can be used at run-time to retrieve a reference to the original vector. +// +// Note that the notion recurses: if we have several vector-of-indices, we can +// reduce each to an index, and then take the resulting vector-of-meta-indices +// and reduce it further to an index. Doing so allows us to concisely refer +// to a potentially large, deep set of indices using a single value - such as +// for CPP_InitsInfo's "offset_set" member variable. + +class IndicesManager + { +public: + IndicesManager() { } + + // Adds a new vector-of-indices to the collection we're tracking, + // returning the offset that will be associated with it at run-time. + int AddIndices(std::vector indices) + { + int n = indices_set.size(); + indices_set.emplace_back(std::move(indices)); + return n; + } + + // Generates the initializations used to construct the managed + // vectors at run-time. + void Generate(CPPCompile* c); + +private: + // Each vector-of-indices being tracked. We could obtain some + // space and time savings by recognizing duplicate vectors + // (for example, empty vectors are very common), but as long + // as the code compiles and executes without undue overhead, + // this doesn't appear necessary. + std::vector> indices_set; + }; + + } // zeek::detail diff --git a/src/script_opt/CPP/Runtime.h b/src/script_opt/CPP/Runtime.h index 258d24d673..d872c1d9e3 100644 --- a/src/script_opt/CPP/Runtime.h +++ b/src/script_opt/CPP/Runtime.h @@ -17,18 +17,21 @@ #include "zeek/ZeekString.h" #include "zeek/module_util.h" #include "zeek/script_opt/CPP/Func.h" -#include "zeek/script_opt/CPP/RuntimeInit.h" +#include "zeek/script_opt/CPP/RuntimeInitSupport.h" +#include "zeek/script_opt/CPP/RuntimeInits.h" #include "zeek/script_opt/CPP/RuntimeOps.h" #include "zeek/script_opt/CPP/RuntimeVec.h" #include "zeek/script_opt/ScriptOpt.h" -namespace zeek +namespace zeek::detail { using BoolValPtr = IntrusivePtr; +using IntValPtr = IntrusivePtr; using CountValPtr = IntrusivePtr; using DoubleValPtr = IntrusivePtr; using StringValPtr = IntrusivePtr; +using TimeValPtr = IntrusivePtr; using IntervalValPtr = IntrusivePtr; using PatternValPtr = IntrusivePtr; using FuncValPtr = IntrusivePtr; diff --git a/src/script_opt/CPP/RuntimeInit.cc b/src/script_opt/CPP/RuntimeInitSupport.cc similarity index 89% rename from src/script_opt/CPP/RuntimeInit.cc rename to src/script_opt/CPP/RuntimeInitSupport.cc index f98718dce0..87a7fb484e 100644 --- a/src/script_opt/CPP/RuntimeInit.cc +++ b/src/script_opt/CPP/RuntimeInitSupport.cc @@ -1,6 +1,6 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include "zeek/script_opt/CPP/RuntimeInit.h" +#include "zeek/script_opt/CPP/RuntimeInitSupport.h" #include "zeek/EventRegistry.h" #include "zeek/module_util.h" @@ -49,7 +49,7 @@ static int flag_init_CPP() static int dummy = flag_init_CPP(); -void register_type__CPP(TypePtr t, const std::string& name) +void register_type__CPP(TypePtr t, const string& name) { if ( t->GetName().size() > 0 ) // Already registered. @@ -113,8 +113,8 @@ void activate_bodies__CPP(const char* fn, const char* module, bool exported, Typ auto v = fg->GetVal(); if ( ! v ) { // Create it. - std::vector no_bodies; - std::vector no_priorities; + vector no_bodies; + vector no_priorities; auto sf = make_intrusive(fn, ft, no_bodies, no_priorities); v = make_intrusive(move(sf)); @@ -193,6 +193,28 @@ FuncValPtr lookup_func__CPP(string name, vector hashes, const TypeP { auto ft = cast_intrusive(t); + if ( hashes.empty() ) + { + // This happens for functions that have at least one + // uncompilable body. + auto gl = lookup_ID(name.c_str(), GLOBAL_MODULE_NAME, false, false, false); + if ( ! gl ) + { + reporter->CPPRuntimeError("non-compiled function %s missing", name.c_str()); + exit(1); + } + + auto v = gl->GetVal(); + if ( ! v || v->GetType()->Tag() != TYPE_FUNC ) + { + reporter->CPPRuntimeError("non-compiled function %s has an invalid value", + name.c_str()); + exit(1); + } + + return cast_intrusive(v); + } + vector bodies; vector priorities; diff --git a/src/script_opt/CPP/RuntimeInit.h b/src/script_opt/CPP/RuntimeInitSupport.h similarity index 98% rename from src/script_opt/CPP/RuntimeInit.h rename to src/script_opt/CPP/RuntimeInitSupport.h index fbc44a32dc..284290c95b 100644 --- a/src/script_opt/CPP/RuntimeInit.h +++ b/src/script_opt/CPP/RuntimeInitSupport.h @@ -5,6 +5,7 @@ #pragma once #include "zeek/Val.h" +#include "zeek/script_opt/CPP/Attrs.h" #include "zeek/script_opt/CPP/Func.h" namespace zeek diff --git a/src/script_opt/CPP/RuntimeInits.cc b/src/script_opt/CPP/RuntimeInits.cc new file mode 100644 index 0000000000..ef795f4942 --- /dev/null +++ b/src/script_opt/CPP/RuntimeInits.cc @@ -0,0 +1,528 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/script_opt/CPP/RunTimeInits.h" + +#include "zeek/Desc.h" +#include "zeek/File.h" +#include "zeek/RE.h" +#include "zeek/ZeekString.h" +#include "zeek/script_opt/CPP/RunTimeInitSupport.h" + +using namespace std; + +namespace zeek::detail + { + +template +void CPP_IndexedInits::InitializeCohortWithOffsets(InitsManager* im, int cohort, + const std::vector& cohort_offsets) + { + auto& co = this->inits[cohort]; + for ( auto i = 0U; i < co.size(); ++i ) + Generate(im, this->inits_vec, cohort_offsets[i], co[i]); + } + +template +void CPP_IndexedInits::Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) + { + auto& e_type = im->Types(init_vals[0]); + int val = init_vals[1]; + ivec[offset] = make_enum__CPP(e_type, val); + } + +template +void CPP_IndexedInits::Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) + { + auto chars = im->Strings(init_vals[0]); + int len = init_vals[1]; + ivec[offset] = make_intrusive(len, chars); + } + +template +void CPP_IndexedInits::Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) + { + auto re = new RE_Matcher(im->Strings(init_vals[0])); + if ( init_vals[1] ) + re->MakeCaseInsensitive(); + + re->Compile(); + + ivec[offset] = make_intrusive(re); + } + +template +void CPP_IndexedInits::Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const + { + auto n = init_vals.size(); + auto i = 0U; + + auto l = make_intrusive(TYPE_ANY); + + while ( i < n ) + l->Append(im->ConstVals(init_vals[i++])); + + ivec[offset] = l; + } + +template +void CPP_IndexedInits::Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const + { + auto n = init_vals.size(); + auto i = 0U; + auto t = init_vals[i++]; + + auto vt = cast_intrusive(im->Types(t)); + auto vv = make_intrusive(vt); + + while ( i < n ) + vv->Append(im->ConstVals(init_vals[i++])); + + ivec[offset] = vv; + } + +template +void CPP_IndexedInits::Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const + { + auto n = init_vals.size(); + auto i = 0U; + auto t = init_vals[i++]; + + auto rt = cast_intrusive(im->Types(t)); + auto rv = make_intrusive(rt); + + while ( i < n ) + { + auto v = init_vals[i]; + if ( v >= 0 ) + rv->Assign(i - 1, im->ConstVals(v)); + ++i; + } + + ivec[offset] = rv; + } + +template +void CPP_IndexedInits::Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const + { + auto n = init_vals.size(); + auto i = 0U; + auto t = init_vals[i++]; + + auto tt = cast_intrusive(im->Types(t)); + auto tv = make_intrusive(tt); + + while ( i < n ) + { + auto index = im->ConstVals(init_vals[i++]); + auto v = init_vals[i++]; + auto value = v >= 0 ? im->ConstVals(v) : nullptr; + tv->Assign(index, value); + } + + ivec[offset] = tv; + } + +template +void CPP_IndexedInits::Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const + { + auto n = init_vals.size(); + auto i = 0U; + auto t = init_vals[i++]; // not used + + auto fn = im->Strings(init_vals[i++]); + auto fv = make_intrusive(make_intrusive(fn, "w")); + + ivec[offset] = fv; + } + +template +void CPP_IndexedInits::Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const + { + auto n = init_vals.size(); + auto i = 0U; + auto t = init_vals[i++]; + + auto fn = im->Strings(init_vals[i++]); + + std::vector hashes; + + while ( i < n ) + hashes.push_back(im->Hashes(init_vals[i++])); + + ivec[offset] = lookup_func__CPP(fn, hashes, im->Types(t)); + } + +template +void CPP_IndexedInits::Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const + { + auto tag = static_cast(init_vals[0]); + auto ae_tag = static_cast(init_vals[1]); + + ExprPtr e; + auto e_arg = init_vals[2]; + + switch ( ae_tag ) + { + case AE_NONE: + break; + + case AE_CONST: + e = make_intrusive(im->ConstVals(e_arg)); + break; + + case AE_NAME: + { + auto name = im->Strings(e_arg); + auto gl = lookup_ID(name, GLOBAL_MODULE_NAME, false, false, false); + ASSERT(gl); + e = make_intrusive(gl); + break; + } + + case AE_RECORD: + { + auto t = im->Types(e_arg); + auto rt = cast_intrusive(t); + auto empty_vals = make_intrusive(); + auto construct = make_intrusive(empty_vals); + e = make_intrusive(construct, rt); + break; + } + + case AE_CALL: + e = im->CallExprs(e_arg); + break; + } + + ivec[offset] = make_intrusive(tag, e); + } + +template +void CPP_IndexedInits::Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const + { + auto n = init_vals.size(); + auto i = 0U; + + std::vector a_list; + while ( i < n ) + a_list.emplace_back(im->Attrs(init_vals[i++])); + + ivec[offset] = make_intrusive(a_list, nullptr, false, false); + } + +// Instantiate the templates we'll need. + +template class CPP_IndexedInits; +template class CPP_IndexedInits; +template class CPP_IndexedInits; +template class CPP_IndexedInits; +template class CPP_IndexedInits; +template class CPP_IndexedInits; +template class CPP_IndexedInits; +template class CPP_IndexedInits; +template class CPP_IndexedInits; +template class CPP_IndexedInits; +template class CPP_IndexedInits; +template class CPP_IndexedInits; + +void CPP_TypeInits::DoPreInits(InitsManager* im, const std::vector& offsets_vec) + { + for ( auto cohort = 0U; cohort < offsets_vec.size(); ++cohort ) + { + auto& co = inits[cohort]; + auto& cohort_offsets = im->Indices(offsets_vec[cohort]); + for ( auto i = 0U; i < co.size(); ++i ) + PreInit(im, cohort_offsets[i], co[i]); + } + } + +void CPP_TypeInits::PreInit(InitsManager* im, int offset, ValElemVec& init_vals) + { + auto tag = static_cast(init_vals[0]); + + if ( tag == TYPE_LIST ) + inits_vec[offset] = make_intrusive(); + + else if ( tag == TYPE_RECORD ) + { + auto name = im->Strings(init_vals[1]); + if ( name[0] ) + inits_vec[offset] = get_record_type__CPP(name); + else + inits_vec[offset] = get_record_type__CPP(nullptr); + } + + // else no pre-initialization needed + } + +void CPP_TypeInits::Generate(InitsManager* im, vector& ivec, int offset, + ValElemVec& init_vals) const + { + auto tag = static_cast(init_vals[0]); + TypePtr t; + switch ( tag ) + { + case TYPE_ADDR: + case TYPE_ANY: + case TYPE_BOOL: + case TYPE_COUNT: + case TYPE_DOUBLE: + case TYPE_ERROR: + case TYPE_INT: + case TYPE_INTERVAL: + case TYPE_PATTERN: + case TYPE_PORT: + case TYPE_STRING: + case TYPE_TIME: + case TYPE_TIMER: + case TYPE_VOID: + case TYPE_SUBNET: + case TYPE_FILE: + t = base_type(tag); + break; + + case TYPE_ENUM: + t = BuildEnumType(im, init_vals); + break; + + case TYPE_OPAQUE: + t = BuildOpaqueType(im, init_vals); + break; + + case TYPE_TYPE: + t = BuildTypeType(im, init_vals); + break; + + case TYPE_VECTOR: + t = BuildVectorType(im, init_vals); + break; + + case TYPE_LIST: + t = BuildTypeList(im, init_vals, offset); + break; + + case TYPE_TABLE: + t = BuildTableType(im, init_vals); + break; + + case TYPE_FUNC: + t = BuildFuncType(im, init_vals); + break; + + case TYPE_RECORD: + t = BuildRecordType(im, init_vals, offset); + break; + + default: + ASSERT(0); + } + + ivec[offset] = t; + } + +TypePtr CPP_TypeInits::BuildEnumType(InitsManager* im, ValElemVec& init_vals) const + { + auto name = im->Strings(init_vals[1]); + auto et = get_enum_type__CPP(name); + + if ( et->Names().empty() ) + { + auto n = init_vals.size(); + auto i = 2U; + + while ( i < n ) + { + auto e_name = im->Strings(init_vals[i++]); + auto e_val = init_vals[i++]; + et->AddNameInternal(e_name, e_val); + } + } + + return et; + } + +TypePtr CPP_TypeInits::BuildOpaqueType(InitsManager* im, ValElemVec& init_vals) const + { + auto name = im->Strings(init_vals[1]); + return make_intrusive(name); + } + +TypePtr CPP_TypeInits::BuildTypeType(InitsManager* im, ValElemVec& init_vals) const + { + auto& t = im->Types(init_vals[1]); + return make_intrusive(t); + } + +TypePtr CPP_TypeInits::BuildVectorType(InitsManager* im, ValElemVec& init_vals) const + { + auto& t = im->Types(init_vals[1]); + return make_intrusive(t); + } + +TypePtr CPP_TypeInits::BuildTypeList(InitsManager* im, ValElemVec& init_vals, int offset) const + { + const auto& tl = cast_intrusive(inits_vec[offset]); + + auto n = init_vals.size(); + auto i = 1U; + + while ( i < n ) + tl->Append(im->Types(init_vals[i++])); + + return tl; + } + +TypePtr CPP_TypeInits::BuildTableType(InitsManager* im, ValElemVec& init_vals) const + { + auto index = cast_intrusive(im->Types(init_vals[1])); + auto yield_i = init_vals[2]; + auto yield = yield_i >= 0 ? im->Types(yield_i) : nullptr; + + return make_intrusive(index, yield); + } + +TypePtr CPP_TypeInits::BuildFuncType(InitsManager* im, ValElemVec& init_vals) const + { + auto p = cast_intrusive(im->Types(init_vals[1])); + auto yield_i = init_vals[2]; + auto flavor = static_cast(init_vals[3]); + + TypePtr y; + + if ( yield_i >= 0 ) + y = im->Types(yield_i); + + else if ( flavor == FUNC_FLAVOR_FUNCTION || flavor == FUNC_FLAVOR_HOOK ) + y = base_type(TYPE_VOID); + + return make_intrusive(p, y, flavor); + } + +TypePtr CPP_TypeInits::BuildRecordType(InitsManager* im, ValElemVec& init_vals, int offset) const + { + auto r = cast_intrusive(inits_vec[offset]); + ASSERT(r); + + if ( r->NumFields() == 0 ) + { + type_decl_list tl; + + auto n = init_vals.size(); + auto i = 2U; + + while ( i < n ) + { + auto s = im->Strings(init_vals[i++]); + auto id = util::copy_string(s); + auto type = im->Types(init_vals[i++]); + auto attrs_i = init_vals[i++]; + + AttributesPtr attrs; + if ( attrs_i >= 0 ) + attrs = im->Attributes(attrs_i); + + tl.append(new TypeDecl(id, type, attrs)); + } + + r->AddFieldsDirectly(tl); + } + + return r; + } + +int CPP_FieldMapping::ComputeOffset(InitsManager* im) const + { + auto r = im->Types(rec)->AsRecordType(); + auto fm_offset = r->FieldOffset(field_name.c_str()); + + if ( fm_offset < 0 ) + { // field does not exist, create it + fm_offset = r->NumFields(); + + auto id = util::copy_string(field_name.c_str()); + auto type = im->Types(field_type); + + AttributesPtr attrs; + if ( field_attrs >= 0 ) + attrs = im->Attributes(field_attrs); + + type_decl_list tl; + tl.append(new TypeDecl(id, type, attrs)); + + r->AddFieldsDirectly(tl); + } + + return fm_offset; + } + +int CPP_EnumMapping::ComputeOffset(InitsManager* im) const + { + auto e = im->Types(e_type)->AsEnumType(); + + auto em_offset = e->Lookup(e_name); + if ( em_offset < 0 ) + { // enum constant does not exist, create it + em_offset = e->Names().size(); + if ( e->Lookup(em_offset) ) + reporter->InternalError("enum inconsistency while initializing compiled scripts"); + e->AddNameInternal(e_name, em_offset); + } + + return em_offset; + } + +void CPP_GlobalInit::Generate(InitsManager* im, std::vector& /* inits_vec */, + int /* offset */) const + { + global = lookup_global__CPP(name, im->Types(type), exported); + + if ( ! global->HasVal() && val >= 0 ) + { + global->SetVal(im->ConstVals(val)); + if ( attrs >= 0 ) + global->SetAttrs(im->Attributes(attrs)); + } + } + +void generate_indices_set(int* inits, std::vector>& indices_set) + { + // First figure out how many groups of indices there are, so we + // can pre-allocate the outer vector. + auto i_ptr = inits; + int num_inits = 0; + while ( *i_ptr >= 0 ) + { + ++num_inits; + int n = *i_ptr; + i_ptr += n + 1; // skip over vector elements + } + + indices_set.reserve(num_inits); + + i_ptr = inits; + while ( *i_ptr >= 0 ) + { + int n = *i_ptr; + ++i_ptr; + std::vector indices; + indices.reserve(n); + for ( int i = 0; i < n; ++i ) + indices.push_back(i_ptr[i]); + i_ptr += n; + + indices_set.emplace_back(move(indices)); + } + } + + } // zeek::detail diff --git a/src/script_opt/CPP/RuntimeInits.h b/src/script_opt/CPP/RuntimeInits.h new file mode 100644 index 0000000000..031208a1ce --- /dev/null +++ b/src/script_opt/CPP/RuntimeInits.h @@ -0,0 +1,542 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +// Classes for run-time initialization and management of C++ values used +// by the generated code. + +// See InitsInfo.h for a discussion of initialization issues and the +// associated strategies for dealing with them. + +#include "zeek/Expr.h" +#include "zeek/module_util.h" +#include "zeek/script_opt/CPP/RuntimeInitSupport.h" + +#pragma once + +namespace zeek::detail + { + +using FileValPtr = IntrusivePtr; +using FuncValPtr = IntrusivePtr; + +class InitsManager; + +// An abstract helper class used to access elements of an initialization vector. +// We need the abstraction because InitsManager below needs to be able to refer +// to any of a range of templated classes. +class CPP_AbstractInitAccessor + { +public: + virtual ~CPP_AbstractInitAccessor() { } + virtual ValPtr Get(int index) const { return nullptr; } + }; + +// Convenient way to refer to an offset associated with a particular Zeek type. +using CPP_ValElem = std::pair; + +// This class groups together all of the vectors needed for run-time +// initialization. We gather them together into a single object so as +// to avoid wiring in a set of globals that the various initialization +// methods have to know about. +class InitsManager + { +public: + InitsManager(std::vector& _const_vals, + std::map>& _consts, + std::vector>& _indices, std::vector& _strings, + std::vector& _hashes, std::vector& _types, + std::vector& _attributes, std::vector& _attrs, + std::vector& _call_exprs) + : const_vals(_const_vals), consts(_consts), indices(_indices), strings(_strings), + hashes(_hashes), types(_types), attributes(_attributes), attrs(_attrs), + call_exprs(_call_exprs) + { + } + + // Providse generic access to Zeek constant values based on a single + // index. + ValPtr ConstVals(int offset) const + { + auto& cv = const_vals[offset]; + return Consts(cv.first, cv.second); + } + + // Retrieves the Zeek constant value for a particular Zeek type. + ValPtr Consts(TypeTag tag, int index) const { return consts[tag]->Get(index); } + + // Accessors for the sundry initialization vectors, each retrieving + // a specific element identified by an index/offset. + const std::vector& Indices(int offset) const { return indices[offset]; } + const char* Strings(int offset) const { return strings[offset]; } + const p_hash_type Hashes(int offset) const { return hashes[offset]; } + const TypePtr& Types(int offset) const { return types[offset]; } + const AttributesPtr& Attributes(int offset) const { return attributes[offset]; } + const AttrPtr& Attrs(int offset) const { return attrs[offset]; } + const CallExprPtr& CallExprs(int offset) const { return call_exprs[offset]; } + +private: + std::vector& const_vals; + std::map>& consts; + std::vector>& indices; + std::vector& strings; + std::vector& hashes; + std::vector& types; + std::vector& attributes; + std::vector& attrs; + std::vector& call_exprs; + }; + +// Manages an initialization vector of the given type. +template class CPP_Init + { +public: + virtual ~CPP_Init() { } + + // Pre-initializes the given element of the vector, if necessary. + virtual void PreInit(InitsManager* im, std::vector& inits_vec, int offset) const { } + + // Initializes the given element of the vector. + virtual void Generate(InitsManager* im, std::vector& inits_vec, int offset) const { } + }; + +// Abstract class for creating a collection of initializers. T1 is +// the type of the generated vector, T2 the type of its initializers. +template class CPP_AbstractInits + { +public: + CPP_AbstractInits(std::vector& _inits_vec, int _offsets_set, std::vector _inits) + : inits_vec(_inits_vec), offsets_set(_offsets_set), inits(std::move(_inits)) + { + // Compute how big to make the vector. + int num_inits = 0; + + for ( const auto& cohort : inits ) + num_inits += cohort.size(); + + inits_vec.resize(num_inits); + } + + // Initialize the given cohort of elements. + void InitializeCohort(InitsManager* im, int cohort) + { + // Get this object's vector-of-vector-of-indices. + auto& offsets_vec = im->Indices(offsets_set); + + if ( cohort == 0 ) + DoPreInits(im, offsets_vec); + + // Get the vector-of-indices for this cohort. + auto& cohort_offsets = im->Indices(offsets_vec[cohort]); + + InitializeCohortWithOffsets(im, cohort, cohort_offsets); + } + +protected: + virtual void InitializeCohortWithOffsets(InitsManager* im, int cohort, + const std::vector& cohort_offsets) + { + } + + // Pre-initialize all elements requiring it. + virtual void DoPreInits(InitsManager* im, const std::vector& offsets_vec) { } + + // Generate a single element. + virtual void GenerateElement(InitsManager* im, T2& init, int offset) { } + + // The initialization vector in its entirety. + std::vector& inits_vec; + + // A meta-index for retrieving the vector-of-vector-of-indices. + int offsets_set; + + // Indexed by cohort. + std::vector inits; + }; + +// Manages an initialization vector that uses "custom" initializers +// (tailored ones rather than initializers based on indexing). +template using CPP_InitVec = std::vector>>; +template class CPP_CustomInits : public CPP_AbstractInits> + { +public: + CPP_CustomInits(std::vector& _inits_vec, int _offsets_set, + std::vector> _inits) + : CPP_AbstractInits>(_inits_vec, _offsets_set, std::move(_inits)) + { + } + +private: + void DoPreInits(InitsManager* im, const std::vector& offsets_vec) override + { + int cohort = 0; + for ( const auto& co : this->inits ) + { + auto& cohort_offsets = im->Indices(offsets_vec[cohort]); + for ( auto i = 0U; i < co.size(); ++i ) + co[i]->PreInit(im, this->inits_vec, cohort_offsets[i]); + ++cohort; + } + } + + void InitializeCohortWithOffsets(InitsManager* im, int cohort, + const std::vector& cohort_offsets) override + { + // Loop over the cohort's elements to initialize them. + auto& co = this->inits[cohort]; + for ( auto i = 0U; i < co.size(); ++i ) + co[i]->Generate(im, this->inits_vec, cohort_offsets[i]); + } + }; + +// Provides access to elements of an initialization vector of the given type. +template class CPP_InitAccessor : public CPP_AbstractInitAccessor + { +public: + CPP_InitAccessor(std::vector& _inits_vec) : inits_vec(_inits_vec) { } + + ValPtr Get(int index) const override { return inits_vec[index]; } + +private: + std::vector& inits_vec; + }; + +// A type used for initializations that are based on indices into +// initialization vectors. +using ValElemVec = std::vector; +using ValElemVecVec = std::vector; + +// Manages an initialization vector of the given type whose elements are +// built up from previously constructed values in other initialization vectors. +template class CPP_IndexedInits : public CPP_AbstractInits + { +public: + CPP_IndexedInits(std::vector& _inits_vec, int _offsets_set, + std::vector _inits) + : CPP_AbstractInits(_inits_vec, _offsets_set, std::move(_inits)) + { + } + +protected: + void InitializeCohortWithOffsets(InitsManager* im, int cohort, + const std::vector& cohort_offsets) override; + + // Note, in the following we pass in the inits_vec, even though + // the method will have direct access to it, because we want to + // use overloading to dispatch to custom generation for different + // types of values. + void Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals); + void Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals); + void Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals); + void Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const; + void Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const; + void Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const; + void Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const; + void Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const; + void Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const; + void Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const; + void Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const; + + // The TypePtr initialization vector requires special treatment, since + // it has to dispatch on subclasses of TypePtr. + virtual void Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const + { + ASSERT(0); + } + }; + +// A specialization of CPP_IndexedInits that supports initializing based +// on subclasses of TypePtr. +class CPP_TypeInits : public CPP_IndexedInits + { +public: + CPP_TypeInits(std::vector& _inits_vec, int _offsets_set, + std::vector> _inits) + : CPP_IndexedInits(_inits_vec, _offsets_set, _inits) + { + } + +protected: + void DoPreInits(InitsManager* im, const std::vector& offsets_vec) override; + void PreInit(InitsManager* im, int offset, ValElemVec& init_vals); + + void Generate(InitsManager* im, std::vector& ivec, int offset, + ValElemVec& init_vals) const override; + + TypePtr BuildEnumType(InitsManager* im, ValElemVec& init_vals) const; + TypePtr BuildOpaqueType(InitsManager* im, ValElemVec& init_vals) const; + TypePtr BuildTypeType(InitsManager* im, ValElemVec& init_vals) const; + TypePtr BuildVectorType(InitsManager* im, ValElemVec& init_vals) const; + TypePtr BuildTypeList(InitsManager* im, ValElemVec& init_vals, int offset) const; + TypePtr BuildTableType(InitsManager* im, ValElemVec& init_vals) const; + TypePtr BuildFuncType(InitsManager* im, ValElemVec& init_vals) const; + TypePtr BuildRecordType(InitsManager* im, ValElemVec& init_vals, int offset) const; + }; + +// Abstract class for initializing basic (non-compound) constants. T1 is +// the Zeek type for the constructed constant, T2 is the C++ type of its +// initializer. +// +// In principle we could derive this from CPP_AbstractInits, though to do so +// we'd need to convert the initializers to a vector-of-vector-of-T2, which +// would trade complexity here for complexity in InitsInfo. So we instead +// keep this class distinct, since at heart it's a simpler set of methods +// and that way we can keep them as such here. +template class CPP_AbstractBasicConsts + { +public: + CPP_AbstractBasicConsts(std::vector& _inits_vec, int _offsets_set, std::vector _inits) + : inits_vec(_inits_vec), offsets_set(_offsets_set), inits(std::move(_inits)) + { + inits_vec.resize(inits.size()); + } + + void InitializeCohort(InitsManager* im, int cohort) + { + ASSERT(cohort == 0); + auto& offsets_vec = im->Indices(offsets_set); + auto& cohort_offsets = im->Indices(offsets_vec[cohort]); + for ( auto i = 0U; i < inits.size(); ++i ) + InitElem(im, cohort_offsets[i], i); + } + +protected: + virtual void InitElem(InitsManager* im, int offset, int index) { ASSERT(0); } + +protected: + // See CPP_AbstractInits for the nature of these. + std::vector& inits_vec; + int offsets_set; + std::vector inits; + }; + +// Class for initializing a basic constant of Zeek type T1, using initializers +// of C++ type T2. T1 is an intrusive pointer to a T3 type; for example, if +// T1 is a BoolValPtr then T3 will be BoolVal. +template +class CPP_BasicConsts : public CPP_AbstractBasicConsts + { +public: + CPP_BasicConsts(std::vector& _inits_vec, int _offsets_set, std::vector _inits) + : CPP_AbstractBasicConsts(_inits_vec, _offsets_set, std::move(_inits)) + { + } + + void InitElem(InitsManager* /* im */, int offset, int index) override + { + this->inits_vec[offset] = make_intrusive(this->inits[index]); + } + }; + +// Specific classes for basic constants that use string-based constructors. +class CPP_AddrConsts : public CPP_AbstractBasicConsts + { +public: + CPP_AddrConsts(std::vector& _inits_vec, int _offsets_set, std::vector _inits) + : CPP_AbstractBasicConsts(_inits_vec, _offsets_set, std::move(_inits)) + { + } + + void InitElem(InitsManager* im, int offset, int index) override + { + auto s = im->Strings(this->inits[index]); + this->inits_vec[offset] = make_intrusive(s); + } + }; + +class CPP_SubNetConsts : public CPP_AbstractBasicConsts + { +public: + CPP_SubNetConsts(std::vector& _inits_vec, int _offsets_set, + std::vector _inits) + : CPP_AbstractBasicConsts(_inits_vec, _offsets_set, std::move(_inits)) + { + } + + void InitElem(InitsManager* im, int offset, int index) override + { + auto s = im->Strings(this->inits[index]); + this->inits_vec[offset] = make_intrusive(s); + } + }; + +// Class for initializing a Zeek global. These don't go into an initialization +// vector, so we use void* as the underlying type. +class CPP_GlobalInit : public CPP_Init + { +public: + CPP_GlobalInit(IDPtr& _global, const char* _name, int _type, int _attrs, int _val, + bool _exported) + : CPP_Init(), global(_global), name(_name), type(_type), attrs(_attrs), val(_val), + exported(_exported) + { + } + + void Generate(InitsManager* im, std::vector& /* inits_vec */, + int /* offset */) const override; + +protected: + IDPtr& global; + const char* name; + int type; + int attrs; + int val; + bool exported; + }; + +// Abstract class for constructing a CallExpr to evaluate a Zeek expression. +class CPP_AbstractCallExprInit : public CPP_Init + { +public: + CPP_AbstractCallExprInit() : CPP_Init() { } + }; + +// Constructs a CallExpr that calls a given CPPFunc subclass. +template class CPP_CallExprInit : public CPP_AbstractCallExprInit + { +public: + CPP_CallExprInit(CallExprPtr& _e_var) : CPP_AbstractCallExprInit(), e_var(_e_var) { } + + void Generate(InitsManager* /* im */, std::vector& inits_vec, + int offset) const override + { + auto wrapper_class = make_intrusive(); + auto func_val = make_intrusive(wrapper_class); + auto func_expr = make_intrusive(func_val); + auto empty_args = make_intrusive(); + + e_var = make_intrusive(func_expr, empty_args); + inits_vec[offset] = e_var; + } + +private: + // Where to store the expression once we've built it. + CallExprPtr& e_var; + }; + +// Abstract class for registering a lambda defined in terms of a CPPStmt. +class CPP_AbstractLambdaRegistration : public CPP_Init + { +public: + CPP_AbstractLambdaRegistration() : CPP_Init() { } + }; + +// Registers a lambda defined in terms of a given CPPStmt subclass. +template class CPP_LambdaRegistration : public CPP_AbstractLambdaRegistration + { +public: + CPP_LambdaRegistration(const char* _name, int _func_type, p_hash_type _h, bool _has_captures) + : CPP_AbstractLambdaRegistration(), name(_name), func_type(_func_type), h(_h), + has_captures(_has_captures) + { + } + + void Generate(InitsManager* im, std::vector& inits_vec, int offset) const override + { + auto l = make_intrusive(name); + auto& ft = im->Types(func_type); + register_lambda__CPP(l, h, name, ft, has_captures); + } + +protected: + const char* name; + int func_type; + p_hash_type h; + bool has_captures; + }; + +// Constructs at run-time a mapping between abstract record field offsets used +// when compiling a set of scripts to their concrete offsets (which might differ +// from those during compilation due to loading of other scripts that extend +// various records). +class CPP_FieldMapping + { +public: + CPP_FieldMapping(int _rec, std::string _field_name, int _field_type, int _field_attrs) + : rec(_rec), field_name(std::move(_field_name)), field_type(_field_type), + field_attrs(_field_attrs) + { + } + + int ComputeOffset(InitsManager* im) const; + +private: + int rec; // index to retrieve the record's type + std::string field_name; // which field this offset pertains to + int field_type; // the field's type, in case we have to construct it + int field_attrs; // the same for the field's attributes + }; + +// Constructs at run-time a mapping between abstract enum values used when +// compiling a set of scripts to their concrete values (which might differ +// from those during compilation due to loading of other scripts that extend +// the enum). +class CPP_EnumMapping + { +public: + CPP_EnumMapping(int _e_type, std::string _e_name) : e_type(_e_type), e_name(std::move(_e_name)) + { + } + + int ComputeOffset(InitsManager* im) const; + +private: + int e_type; // index to EnumType + std::string e_name; // which enum constant for that type + }; + +// Looks up a BiF of the given name, making it available to compiled +// code via a C++ global. +class CPP_LookupBiF + { +public: + CPP_LookupBiF(zeek::Func*& _bif_func, std::string _bif_name) + : bif_func(_bif_func), bif_name(std::move(_bif_name)) + { + } + + void ResolveBiF() const { bif_func = lookup_bif__CPP(bif_name.c_str()); } + +protected: + zeek::Func*& bif_func; // where to store the pointer to the BiF + std::string bif_name; // the BiF's name + }; + +// Information needed to register a compiled function body (which makes it +// available to substitute for the body's AST). The compiler generates +// code that loops over a vector of these to perform the registrations. +struct CPP_RegisterBody + { + CPP_RegisterBody(std::string _func_name, void* _func, int _type_signature, int _priority, + p_hash_type _h, std::vector _events) + : func_name(std::move(_func_name)), func(_func), type_signature(_type_signature), + priority(_priority), h(_h), events(std::move(_events)) + { + } + + std::string func_name; // name of the function + void* func; // pointer to C++ + int type_signature; + int priority; + p_hash_type h; + std::vector events; + }; + +// Helper function that takes a (large) array of int's and from them +// constructs the corresponding vector-of-vector-of-indices. Each +// vector-of-indices is represented first by an int specifying its +// size, and then that many int's for its values. We recognize the +// end of the array upon encountering a "size" entry of -1. +extern void generate_indices_set(int* inits, std::vector>& indices_set); + + } // zeek::detail diff --git a/src/script_opt/CPP/Stmts.cc b/src/script_opt/CPP/Stmts.cc index 47d8f713a9..f62e72b4cb 100644 --- a/src/script_opt/CPP/Stmts.cc +++ b/src/script_opt/CPP/Stmts.cc @@ -245,7 +245,7 @@ void CPPCompile::GenSwitchStmt(const SwitchStmt* sw) else sw_val = string("p_hash(") + GenExpr(e, GEN_VAL_PTR) + ")"; - Emit("switch ( %s ) {", sw_val.c_str()); + Emit("switch ( %s ) {", sw_val); ++break_level; diff --git a/src/script_opt/CPP/Tracker.cc b/src/script_opt/CPP/Tracker.cc index e82e5878b2..8eeb616e5e 100644 --- a/src/script_opt/CPP/Tracker.cc +++ b/src/script_opt/CPP/Tracker.cc @@ -51,13 +51,25 @@ template string CPPTracker::KeyName(const T* key) auto hash = map[key]; ASSERT(hash != 0); + auto rep = reps[hash]; + if ( gi_s.count(rep) > 0 ) + return gi_s[rep]->Name(); + auto index = map2[hash]; string scope; if ( IsInherited(hash) ) scope = scope_prefix(scope2[hash]); - return scope + string(base_name) + "_" + Fmt(index) + "__CPP"; + string ind = Fmt(index); + string full_name; + + if ( single_global ) + full_name = base_name + "__CPP[" + ind + "]"; + else + full_name = base_name + "_" + ind + "__CPP"; + + return scope + full_name; } template void CPPTracker::LogIfNew(IntrusivePtr key, int scope, FILE* log_file) diff --git a/src/script_opt/CPP/Tracker.h b/src/script_opt/CPP/Tracker.h index 175d48192e..6005247058 100644 --- a/src/script_opt/CPP/Tracker.h +++ b/src/script_opt/CPP/Tracker.h @@ -15,6 +15,7 @@ #pragma once #include "zeek/script_opt/CPP/HashMgr.h" +#include "zeek/script_opt/CPP/InitsInfo.h" namespace zeek::detail { @@ -24,11 +25,13 @@ namespace zeek::detail template class CPPTracker { public: - // The base name is used to construct key names. The mapper, - // if present, maps hash values to information about the previously - // generated scope in which the value appears. - CPPTracker(const char* _base_name, VarMapper* _mapper = nullptr) - : base_name(_base_name), mapper(_mapper) + // The base name is used to construct key names. "single_global", + // if true, specifies that the names should be constructed as + // indexes into a single global, rather than as distinct globals. + // The mapper, if present, maps hash values to information about + // the previously generated scope in which the value appears. + CPPTracker(const char* _base_name, bool _single_global, VarMapper* _mapper = nullptr) + : base_name(_base_name), single_global(_single_global), mapper(_mapper) { } @@ -40,6 +43,8 @@ public: // is provided, then refrains from computing it. void AddKey(IntrusivePtr key, p_hash_type h = 0); + void AddInitInfo(const T* rep, std::shared_ptr gi) { gi_s[rep] = std::move(gi); } + // Returns the (C++ variable) name associated with the given key. std::string KeyName(const T* key); std::string KeyName(IntrusivePtr key) { return KeyName(key.get()); } @@ -81,6 +86,8 @@ private: // Maps keys to internal representations (i.e., hashes). std::unordered_map map; + std::unordered_map> gi_s; + // Maps internal representations to distinct values. These // may-or-may-not be indices into an "inherited" namespace scope. std::unordered_map map2; @@ -98,6 +105,10 @@ private: // Used to construct key names. std::string base_name; + // Whether to base the names out of a single global, or distinct + // globals. + bool single_global; + // If non-nil, the mapper to consult for previous names. VarMapper* mapper; }; diff --git a/src/script_opt/CPP/Types.cc b/src/script_opt/CPP/Types.cc index 1e04a33206..c55a2287f8 100644 --- a/src/script_opt/CPP/Types.cc +++ b/src/script_opt/CPP/Types.cc @@ -91,170 +91,13 @@ string CPPCompile::GenericValPtrToGT(const string& expr, const TypePtr& t, GenTy return string("cast_intrusive<") + IntrusiveVal(t) + ">(" + expr + ")"; } -void CPPCompile::ExpandTypeVar(const TypePtr& t) - { - auto tn = GenTypeName(t); - - switch ( t->Tag() ) - { - case TYPE_LIST: - ExpandListTypeVar(t, tn); - break; - - case TYPE_RECORD: - ExpandRecordTypeVar(t, tn); - break; - - case TYPE_ENUM: - ExpandEnumTypeVar(t, tn); - break; - - case TYPE_TABLE: - ExpandTableTypeVar(t, tn); - break; - - case TYPE_FUNC: - ExpandFuncTypeVar(t, tn); - break; - - case TYPE_TYPE: - AddInit(t, tn, - string("make_intrusive(") + GenTypeName(t->AsTypeType()->GetType()) + - ")"); - break; - - case TYPE_VECTOR: - AddInit(t, tn, - string("make_intrusive(") + - GenTypeName(t->AsVectorType()->Yield()) + ")"); - break; - - default: - break; - } - - auto& script_type_name = t->GetName(); - if ( ! script_type_name.empty() ) - AddInit(t, "register_type__CPP(" + tn + ", \"" + script_type_name + "\");"); - - AddInit(t); - } - -void CPPCompile::ExpandListTypeVar(const TypePtr& t, string& tn) - { - const auto& tl = t->AsTypeList()->GetTypes(); - auto t_name = tn + "->AsTypeList()"; - - for ( const auto& tl_i : tl ) - AddInit(t, t_name + "->Append(" + GenTypeName(tl_i) + ");"); - } - -void CPPCompile::ExpandRecordTypeVar(const TypePtr& t, string& tn) - { - auto r = t->AsRecordType()->Types(); - - if ( ! r ) - return; - - auto t_name = tn + "->AsRecordType()"; - - AddInit(t, string("if ( ") + t_name + "->NumFields() == 0 )"); - - AddInit(t, "{"); - AddInit(t, "type_decl_list tl;"); - - for ( auto i = 0; i < r->length(); ++i ) - { - const auto& td = (*r)[i]; - AddInit(t, GenTypeDecl(td)); - } - - AddInit(t, t_name + "->AddFieldsDirectly(tl);"); - AddInit(t, "}"); - } - -void CPPCompile::ExpandEnumTypeVar(const TypePtr& t, string& tn) - { - auto e_name = tn + "->AsEnumType()"; - auto et = t->AsEnumType(); - auto names = et->Names(); - - AddInit(t, "{ auto et = " + e_name + ";"); - AddInit(t, "if ( et->Names().empty() ) {"); - - for ( const auto& name_pair : et->Names() ) - AddInit(t, string("\tet->AddNameInternal(\"") + name_pair.first + "\", " + - Fmt(int(name_pair.second)) + ");"); - - AddInit(t, "}}"); - } - -void CPPCompile::ExpandTableTypeVar(const TypePtr& t, string& tn) - { - auto tbl = t->AsTableType(); - - const auto& indices = tbl->GetIndices(); - const auto& yield = tbl->Yield(); - - if ( tbl->IsSet() ) - AddInit(t, tn, - string("make_intrusive(cast_intrusive(") + GenTypeName(indices) + - " ), nullptr)"); - else - AddInit(t, tn, - string("make_intrusive(cast_intrusive(") + - GenTypeName(indices) + "), " + GenTypeName(yield) + ")"); - } - -void CPPCompile::ExpandFuncTypeVar(const TypePtr& t, string& tn) - { - auto f = t->AsFuncType(); - - auto args_type_accessor = GenTypeName(f->Params()); - const auto& yt = f->Yield(); - - string yield_type_accessor; - - if ( yt ) - yield_type_accessor += GenTypeName(yt); - else - yield_type_accessor += "nullptr"; - - auto fl = f->Flavor(); - - string fl_name; - if ( fl == FUNC_FLAVOR_FUNCTION ) - fl_name = "FUNC_FLAVOR_FUNCTION"; - else if ( fl == FUNC_FLAVOR_EVENT ) - fl_name = "FUNC_FLAVOR_EVENT"; - else if ( fl == FUNC_FLAVOR_HOOK ) - fl_name = "FUNC_FLAVOR_HOOK"; - - auto type_init = string("make_intrusive(cast_intrusive(") + - args_type_accessor + "), " + yield_type_accessor + ", " + fl_name + ")"; - - AddInit(t, tn, type_init); - } - -string CPPCompile::GenTypeDecl(const TypeDecl* td) - { - auto type_accessor = GenTypeName(td->type); - - auto td_name = string("util::copy_string(\"") + td->id + "\")"; - - if ( td->attrs ) - return string("tl.append(new TypeDecl(") + td_name + ", " + type_accessor + ", " + - AttrsName(td->attrs) + "));"; - - return string("tl.append(new TypeDecl(") + td_name + ", " + type_accessor + "));"; - } - string CPPCompile::GenTypeName(const Type* t) { + ASSERT(processed_types.count(TypeRep(t)) > 0); return types.KeyName(TypeRep(t)); } -const char* CPPCompile::TypeTagName(TypeTag tag) const +const char* CPPCompile::TypeTagName(TypeTag tag) { switch ( tag ) { @@ -280,6 +123,8 @@ const char* CPPCompile::TypeTagName(TypeTag tag) const return "TYPE_INT"; case TYPE_INTERVAL: return "TYPE_INTERVAL"; + case TYPE_LIST: + return "TYPE_LIST"; case TYPE_OPAQUE: return "TYPE_OPAQUE"; case TYPE_PATTERN: @@ -431,16 +276,16 @@ const char* CPPCompile::TypeType(const TypePtr& t) } } -void CPPCompile::RegisterType(const TypePtr& tp) +shared_ptr CPPCompile::RegisterType(const TypePtr& tp) { auto t = TypeRep(tp); if ( processed_types.count(t) > 0 ) - return; + return processed_types[t]; - // Add the type before going further, to avoid loops due to types - // that reference each other. - processed_types.insert(t); + processed_types[t] = nullptr; + + shared_ptr gi; switch ( t->Tag() ) { @@ -449,7 +294,6 @@ void CPPCompile::RegisterType(const TypePtr& tp) case TYPE_BOOL: case TYPE_COUNT: case TYPE_DOUBLE: - case TYPE_ENUM: case TYPE_ERROR: case TYPE_INT: case TYPE_INTERVAL: @@ -459,119 +303,53 @@ void CPPCompile::RegisterType(const TypePtr& tp) case TYPE_TIME: case TYPE_TIMER: case TYPE_VOID: - case TYPE_OPAQUE: case TYPE_SUBNET: case TYPE_FILE: - // Nothing to do. + gi = make_shared(this, tp); + break; + + case TYPE_ENUM: + gi = make_shared(this, tp); + break; + + case TYPE_OPAQUE: + gi = make_shared(this, tp); break; case TYPE_TYPE: - { - const auto& tt = t->AsTypeType()->GetType(); - NoteNonRecordInitDependency(t, tt); - RegisterType(tt); - } + gi = make_shared(this, tp); break; case TYPE_VECTOR: - { - const auto& yield = t->AsVectorType()->Yield(); - NoteNonRecordInitDependency(t, yield); - RegisterType(yield); - } + gi = make_shared(this, tp); break; case TYPE_LIST: - RegisterListType(tp); + gi = make_shared(this, tp); break; case TYPE_TABLE: - RegisterTableType(tp); + gi = make_shared(this, tp); break; case TYPE_RECORD: - RegisterRecordType(tp); + gi = make_shared(this, tp); break; case TYPE_FUNC: - RegisterFuncType(tp); + gi = make_shared(this, tp); break; default: reporter->InternalError("bad type in CPPCompile::RegisterType"); } - AddInit(t); + type_info->AddInstance(gi); + processed_types[t] = gi; - if ( ! types.IsInherited(t) ) - { - auto t_rep = types.GetRep(t); - if ( t_rep == t ) - GenPreInit(t); - else - NoteInitDependency(t, t_rep); - } - } + types.AddInitInfo(t, gi); -void CPPCompile::RegisterListType(const TypePtr& t) - { - const auto& tl = t->AsTypeList()->GetTypes(); - - for ( auto& tl_i : tl ) - { - NoteNonRecordInitDependency(t, tl_i); - RegisterType(tl_i); - } - } - -void CPPCompile::RegisterTableType(const TypePtr& t) - { - auto tbl = t->AsTableType(); - const auto& indices = tbl->GetIndices(); - const auto& yield = tbl->Yield(); - - NoteNonRecordInitDependency(t, indices); - RegisterType(indices); - - if ( yield ) - { - NoteNonRecordInitDependency(t, yield); - RegisterType(yield); - } - } - -void CPPCompile::RegisterRecordType(const TypePtr& t) - { - auto r = t->AsRecordType()->Types(); - - if ( ! r ) - return; - - for ( const auto& r_i : *r ) - { - NoteNonRecordInitDependency(t, r_i->type); - RegisterType(r_i->type); - - if ( r_i->attrs ) - { - NoteInitDependency(t, r_i->attrs); - RegisterAttributes(r_i->attrs); - } - } - } - -void CPPCompile::RegisterFuncType(const TypePtr& t) - { - auto f = t->AsFuncType(); - - NoteInitDependency(t, TypeRep(f->Params())); - RegisterType(f->Params()); - - if ( f->Yield() ) - { - NoteNonRecordInitDependency(t, f->Yield()); - RegisterType(f->Yield()); - } + return gi; } const char* CPPCompile::NativeAccessor(const TypePtr& t) diff --git a/src/script_opt/CPP/Vars.cc b/src/script_opt/CPP/Vars.cc index 6d06ffd08d..432dce39a9 100644 --- a/src/script_opt/CPP/Vars.cc +++ b/src/script_opt/CPP/Vars.cc @@ -83,7 +83,7 @@ void CPPCompile::CreateGlobal(const ID* g) if ( pfs.Globals().count(g) == 0 ) { // Only used in the context of calls. If it's compilable, - // the we'll call it directly. + // then we'll call it directly. if ( compilable_funcs.count(gn) > 0 ) { AddGlobal(gn, "zf", true); @@ -102,18 +102,12 @@ void CPPCompile::CreateGlobal(const ID* g) Emit("IDPtr %s;", globals[gn]); if ( pfs.Events().count(gn) > 0 ) - // This is an event that's also used as - // a variable. + // This is an event that's also used as a variable. Emit("EventHandlerPtr %s_ev;", globals[gn]); - const auto& t = g->GetType(); - NoteInitDependency(g, TypeRep(t)); - - auto exported = g->IsExport() ? "true" : "false"; - - AddInit(g, globals[gn], - string("lookup_global__CPP(\"") + gn + "\", " + GenTypeName(t) + ", " + exported + - ")"); + auto gi = make_shared(this, g, globals[gn]); + global_id_info->AddInstance(gi); + global_gis[g] = gi; } if ( is_bif ) @@ -124,40 +118,22 @@ void CPPCompile::CreateGlobal(const ID* g) global_vars.emplace(g); } -void CPPCompile::UpdateGlobalHashes() +std::shared_ptr CPPCompile::RegisterGlobal(const ID* g) { - for ( auto& g : pfs.AllGlobals() ) + if ( global_gis.count(g) == 0 ) { - auto gn = g->Name(); + auto gn = string(g->Name()); - if ( hm.HasGlobal(gn) ) - // Not new to this compilation run. - continue; + if ( globals.count(gn) == 0 ) + // Create a name for it. + (void)IDNameStr(g); - auto ht = pfs.HashType(g->GetType()); - - p_hash_type hv = 0; - if ( g->GetVal() ) - hv = p_hash(g->GetVal()); - - fprintf(hm.HashFile(), "global\n%s\n", gn); - fprintf(hm.HashFile(), "%llu %llu\n", ht, hv); - - // Record location information in the hash file for - // diagnostic purposes. - auto loc = g->GetLocationInfo(); - fprintf(hm.HashFile(), "%s %d\n", loc->filename, loc->first_line); - - // Flag any named record/enum types. - if ( g->IsType() ) - { - const auto& t = g->GetType(); - if ( t->Tag() == TYPE_RECORD ) - fprintf(hm.HashFile(), "record\n%s\n", gn); - else if ( t->Tag() == TYPE_ENUM ) - fprintf(hm.HashFile(), "enum\n%s\n", gn); - } + auto gi = make_shared(this, g, globals[gn]); + global_id_info->AddInstance(gi); + global_gis[g] = gi; } + + return global_gis[g]; } void CPPCompile::AddBiF(const ID* b, bool is_var) @@ -170,12 +146,8 @@ void CPPCompile::AddBiF(const ID* b, bool is_var) if ( AddGlobal(n, "bif", true) ) Emit("Func* %s;", globals[n]); - auto lookup = string("lookup_bif__CPP(\"") + bn + "\")"; - - if ( standalone ) - AddActivation(globals[n] + " = " + lookup + ";"); - else - AddInit(b, globals[n], lookup); + ASSERT(BiFs.count(globals[n]) == 0); + BiFs[globals[n]] = bn; } bool CPPCompile::AddGlobal(const string& g, const char* suffix, bool track) @@ -189,13 +161,8 @@ bool CPPCompile::AddGlobal(const string& g, const char* suffix, bool track) if ( hm.HasGlobalVar(gn) ) gn = scope_prefix(hm.GlobalVarScope(gn)) + gn; else - { new_var = true; - if ( track && update ) - fprintf(hm.HashFile(), "global-var\n%s\n%d\n", gn.c_str(), addl_tag); - } - globals.emplace(g, gn); } @@ -207,18 +174,19 @@ void CPPCompile::RegisterEvent(string ev_name) body_events[body_name].emplace_back(move(ev_name)); } -const string& CPPCompile::IDNameStr(const ID* id) const +const string& CPPCompile::IDNameStr(const ID* id) { if ( id->IsGlobal() ) { auto g = string(id->Name()); - ASSERT(globals.count(g) > 0); - return ((CPPCompile*)(this))->globals[g]; + if ( globals.count(g) == 0 ) + CreateGlobal(id); + return globals[g]; } ASSERT(locals.count(id) > 0); - return ((CPPCompile*)(this))->locals[id]; + return locals[id]; } string CPPCompile::LocalName(const ID* l) const diff --git a/src/script_opt/ScriptOpt.cc b/src/script_opt/ScriptOpt.cc index 9f12373ce9..aa70a8ae9d 100644 --- a/src/script_opt/ScriptOpt.cc +++ b/src/script_opt/ScriptOpt.cc @@ -399,7 +399,7 @@ static void generate_CPP(std::unique_ptr& pfs) { const auto hash_name = hash_dir + "CPP-hashes"; - auto hm = std::make_unique(hash_name.c_str(), analysis_options.add_CPP); + auto hm = std::make_unique(hash_name.c_str()); if ( analysis_options.gen_CPP ) { @@ -413,26 +413,12 @@ static void generate_CPP(std::unique_ptr& pfs) } } } - else - { // doing add-C++ instead, so look for previous compilations - for ( auto& func : funcs ) - { - auto hash = func.Profile()->HashVal(); - if ( compiled_scripts.count(hash) > 0 || hm->HasHash(hash) ) - func.SetSkip(true); - } - - // Now that we've presumably marked a lot of functions - // as skippable, recompute the global profile. - pfs = std::make_unique(funcs, is_CPP_compilable, false); - } const auto gen_name = hash_dir + "CPP-gen.cc"; const auto addl_name = hash_dir + "CPP-gen-addl.h"; - CPPCompile cpp(funcs, *pfs, gen_name, addl_name, *hm, - analysis_options.gen_CPP || analysis_options.update_CPP, - analysis_options.gen_standalone_CPP, analysis_options.report_uncompilable); + CPPCompile cpp(funcs, *pfs, gen_name, addl_name, *hm, analysis_options.gen_standalone_CPP, + analysis_options.report_uncompilable); } static void find_when_funcs(std::unique_ptr& pfs, From bb618bae30712387ea53e8c5446f23dc2ca0156d Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Sun, 7 Nov 2021 17:02:18 -0800 Subject: [PATCH 041/210] remove -O add-C++ and -O update-C++ options --- src/Options.cc | 8 +------- src/script_opt/ScriptOpt.cc | 19 ------------------- src/script_opt/ScriptOpt.h | 12 ------------ 3 files changed, 1 insertion(+), 38 deletions(-) diff --git a/src/Options.cc b/src/Options.cc index 9bc235cbcc..5cba4fe246 100644 --- a/src/Options.cc +++ b/src/Options.cc @@ -202,8 +202,6 @@ static void print_analysis_help() fprintf(stderr, " report-uncompilable print names of functions that can't be compiled\n"); fprintf(stderr, " use-C++ use available C++ script bodies\n"); fprintf(stderr, "\n experimental options for incremental compilation:\n"); - fprintf(stderr, " add-C++ generate private C++ for any missing script bodies\n"); - fprintf(stderr, " update-C++ generate reusable C++ for any missing script bodies\n"); } static void set_analysis_option(const char* opt, Options& opts) @@ -223,9 +221,7 @@ static void set_analysis_option(const char* opt, Options& opts) exit(0); } - if ( util::streq(opt, "add-C++") ) - a_o.add_CPP = true; - else if ( util::streq(opt, "dump-uds") ) + if ( util::streq(opt, "dump-uds") ) a_o.activate = a_o.dump_uds = true; else if ( util::streq(opt, "dump-xform") ) a_o.activate = a_o.dump_xform = true; @@ -253,8 +249,6 @@ static void set_analysis_option(const char* opt, Options& opts) a_o.inliner = a_o.report_recursive = true; else if ( util::streq(opt, "report-uncompilable") ) a_o.report_uncompilable = true; - else if ( util::streq(opt, "update-C++") ) - a_o.update_CPP = true; else if ( util::streq(opt, "use-C++") ) a_o.use_CPP = true; else if ( util::streq(opt, "xform") ) diff --git a/src/script_opt/ScriptOpt.cc b/src/script_opt/ScriptOpt.cc index aa70a8ae9d..2940b1b5ec 100644 --- a/src/script_opt/ScriptOpt.cc +++ b/src/script_opt/ScriptOpt.cc @@ -221,8 +221,6 @@ static void init_options() check_env_opt("ZEEK_PROFILE", analysis_options.profile_ZAM); // Compile-to-C++-related options. - check_env_opt("ZEEK_ADD_CPP", analysis_options.add_CPP); - check_env_opt("ZEEK_UPDATE_CPP", analysis_options.update_CPP); check_env_opt("ZEEK_GEN_CPP", analysis_options.gen_CPP); check_env_opt("ZEEK_GEN_STANDALONE_CPP", analysis_options.gen_standalone_CPP); check_env_opt("ZEEK_COMPILE_ALL", analysis_options.compile_all); @@ -233,23 +231,6 @@ static void init_options() analysis_options.gen_CPP = true; if ( analysis_options.gen_CPP ) - { - if ( analysis_options.add_CPP ) - { - reporter->Warning("gen-C++ incompatible with add-C++"); - analysis_options.add_CPP = false; - } - - if ( analysis_options.update_CPP ) - { - reporter->Warning("gen-C++ incompatible with update-C++"); - analysis_options.update_CPP = false; - } - - generating_CPP = true; - } - - if ( analysis_options.update_CPP || analysis_options.add_CPP ) generating_CPP = true; if ( analysis_options.use_CPP && generating_CPP ) diff --git a/src/script_opt/ScriptOpt.h b/src/script_opt/ScriptOpt.h index 065a9bb2c9..8d15eadb23 100644 --- a/src/script_opt/ScriptOpt.h +++ b/src/script_opt/ScriptOpt.h @@ -96,18 +96,6 @@ struct AnalyOpt // of the corresponding script, and not activated by default). bool gen_standalone_CPP = false; - // If true, generate C++ for those script bodies that don't already - // have generated code, in a form that enables later compiles to - // take advantage of the newly-added elements. Only use for generating - // a zeek that will always include the associated scripts. - bool update_CPP = false; - - // If true, generate C++ for those script bodies that don't already - // have generated code. The added C++ is not made available for - // later generated code, and will work for a generated zeek that - // runs without including the associated scripts. - bool add_CPP = false; - // If true, use C++ bodies if available. bool use_CPP = false; From ab1a29184c8d5f454c1312fc27d123870371d665 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Sun, 7 Nov 2021 17:05:21 -0800 Subject: [PATCH 042/210] documentation updates --- src/script_opt/CPP/README.md | 43 ++++-------------------------------- 1 file changed, 4 insertions(+), 39 deletions(-) diff --git a/src/script_opt/CPP/README.md b/src/script_opt/CPP/README.md index e2790320fd..b9153cd9a5 100644 --- a/src/script_opt/CPP/README.md +++ b/src/script_opt/CPP/README.md @@ -114,40 +114,6 @@ There are additional workflows relating to running the test suite, which we document only briefly here as they're likely going to change or go away , as it's not clear they're actually needed. -First, `-O update-C++` will run using a Zeek instance that already includes -compiled scripts and, for any functions pulled in by the command-line scripts, -if they're not already compiled, will generate additional C++ code for -those that can be combined with the already-compiled code. The -additionally compiled code leverages the existing compiled-in functions -(and globals), which it learns about via the `CPP-hashes.dat` file mentioned -above. Any code compiled in this fashion must be _consistent_ with the -previously compiled code, meaning that globals and extensible types (enums, -records) have definitions that align with those previously used, and any -other code later compiled must also be consistent. - -In a similar vein, `-O add-C++` likewise uses a Zeek instance that already -includes compiled scripts. It generates additional C++ code that leverages -that existing compilation. However, this code is _not_ meant for use with -subsequently compiled code; later code also build with `add-C++` can have -inconsistencies with this code. (The utility of this mode is to support -compiling the entire test suite as one large incremental compilation, -rather than as hundreds of pointwise compilations.) - -Both of these _append_ to any existing `CPP-gen-addl.h` file, providing -a means for building it up to reflect a number of compilations. - -The `update-C++` and `add-C++` options help support different -ways of building the `btest` test suite. They were meant to enable doing so -without requiring per-test-suite-element recompilations. However, experiences -to date have found that trying to avoid pointwise compilations incurs -additional headaches, so it's better to just bite off the cost of a large -number of recompilations. Given that, it might make sense to remove these -options. - -Finally, with respect to workflow there are number of simple scripts in -`src/script_opt/CPP/` (which should ultimately be replaced) in support of -compiler maintenance: - * `non-embedded-build` Builds `zeek` without any embedded compiled-to-C++ scripts. * `bare-embedded-build` @@ -183,12 +149,11 @@ Known Issues Here we list various known issues with using the compiler:
-* Compilation of compiled code can be noticeably slow (if built using -`./configure --enable-debug`) or hugely slow (if not), with the latter -taking on the order of an hour on a beefy laptop. This slowness complicates +* Compilation of compiled code can be quite slow when the C++ compilation +includes optimization, +taking many minutes on a beefy laptop. This slowness complicates CI/CD approaches for always running compiled code against the test suite -when merging changes. It's not presently clear how feasible it is to -speed this up. +when merging changes. * Run-time error messages generally lack location information and information about associated expressions/statements, making them hard to puzzle out. From 0bdc268a00cf608c3db063fa6d7eb2aa584fa2be Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Sun, 7 Nov 2021 17:49:30 -0800 Subject: [PATCH 043/210] geez sometimes it's signed sometimes it's not --- src/script_opt/CPP/InitsInfo.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/script_opt/CPP/InitsInfo.cc b/src/script_opt/CPP/InitsInfo.cc index 4c2685afaf..4eb670de37 100644 --- a/src/script_opt/CPP/InitsInfo.cc +++ b/src/script_opt/CPP/InitsInfo.cc @@ -185,7 +185,7 @@ ListConstInfo::ListConstInfo(CPPCompile* _c, ValPtr v) : CompoundItemInfo(_c) auto lv = cast_intrusive(v); auto n = lv->Length(); - for ( auto i = 0U; i < n; ++i ) + for ( auto i = 0; i < n; ++i ) vals.emplace_back(ValElem(c, lv->Idx(i))); } @@ -194,7 +194,7 @@ VectorConstInfo::VectorConstInfo(CPPCompile* c, ValPtr v) : CompoundItemInfo(c, auto vv = cast_intrusive(v); auto n = vv->Size(); - for ( auto i = 0; i < n; ++i ) + for ( auto i = 0U; i < n; ++i ) vals.emplace_back(ValElem(c, vv->ValAt(i))); } @@ -205,7 +205,7 @@ RecordConstInfo::RecordConstInfo(CPPCompile* c, ValPtr v) : CompoundItemInfo(c, type = c->TypeOffset(r->GetType()); - for ( auto i = 0; i < n; ++i ) + for ( auto i = 0U; i < n; ++i ) vals.emplace_back(ValElem(c, r->GetField(i))); } From 3ea362be91a5c8d5218ee9f1397e474b49bf9ef8 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Sun, 7 Nov 2021 17:54:56 -0800 Subject: [PATCH 044/210] fix for case-sensitive file systems --- src/script_opt/CPP/RuntimeInits.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/script_opt/CPP/RuntimeInits.cc b/src/script_opt/CPP/RuntimeInits.cc index ef795f4942..739ed9a4d9 100644 --- a/src/script_opt/CPP/RuntimeInits.cc +++ b/src/script_opt/CPP/RuntimeInits.cc @@ -1,12 +1,12 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include "zeek/script_opt/CPP/RunTimeInits.h" +#include "zeek/script_opt/CPP/RuntimeInits.h" #include "zeek/Desc.h" #include "zeek/File.h" #include "zeek/RE.h" #include "zeek/ZeekString.h" -#include "zeek/script_opt/CPP/RunTimeInitSupport.h" +#include "zeek/script_opt/CPP/RuntimeInitSupport.h" using namespace std; From c190c85bf0131e5eeab7d0cada4210050bfe2bf5 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Mon, 8 Nov 2021 09:08:20 -0800 Subject: [PATCH 045/210] Update doc submodule [nomail] [skip ci] --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index 931cc50ce8..6bcc7c3e12 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 931cc50ce8fa21dbe8b2816136069b2d18716f06 +Subproject commit 6bcc7c3e125e0bef4352d44a57ded9e9e9fc3775 From e0b116154a19ad1d8b5d7b201eb35c1b5fbdb395 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 8 Nov 2021 15:19:57 -0700 Subject: [PATCH 046/210] GH-1819: Handle recursive types when describing type in binary mode --- src/Type.cc | 9 +++++++-- testing/btest/core/recursive-types.zeek | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 testing/btest/core/recursive-types.zeek diff --git a/src/Type.cc b/src/Type.cc index 80a6e9c678..9c0f22d868 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1326,10 +1326,15 @@ void RecordType::DescribeFields(ODesc* d) const d->AddCount(types->length()); for ( const auto& type : *types ) { - type->type->Describe(d); - d->SP(); d->Add(type->id); d->SP(); + + if ( d->FindType(type->type.get()) ) + d->Add(""); + else + type->type->Describe(d); + + d->SP(); } } } diff --git a/testing/btest/core/recursive-types.zeek b/testing/btest/core/recursive-types.zeek new file mode 100644 index 0000000000..c4f6f595e1 --- /dev/null +++ b/testing/btest/core/recursive-types.zeek @@ -0,0 +1,22 @@ +# @TEST-EXEC: zeek -b %INPUT + +# global_ids() here contains some types that are recursive, in that +# arguments to functions contain chained references to the type that +# defines the function. This tests that we don't crash when +# attempting to call Describe() on those types in binary-mode. +event zeek_init() + { + local sh: string = ""; + local gi = global_ids(); + for (myfunc in gi) + { + if(gi[myfunc]?$value) + { + if(strstr(myfunc,"lambda") > 0) + { + sh = sha256_hash(gi[myfunc]$value); + print(sh); + } + } + } + } From 72cbc7cd13b7c1bda98658104431c3b530ff68d6 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Fri, 5 Nov 2021 10:35:11 +0100 Subject: [PATCH 047/210] Move 3rdparty source files to `3rdparty/` This patch moves in-tree 3rdparty source files to `3rdparty/`. With that we can remove special treatment of these files for `run-clang-format`. --- .clang-format-ignore | 14 +- COPYING.3rdparty | 14 +- src/3rdparty | 2 +- src/CMakeLists.txt | 20 +- src/ConvertUTF.c | 755 ---------------------- src/ConvertUTF.h | 233 ------- src/DNS_Mgr.cc | 2 +- src/Debug.cc | 2 +- src/Desc.cc | 2 +- src/IPAddr.cc | 2 +- src/Options.cc | 2 +- src/PrefixTable.h | 2 +- src/RunState.cc | 2 +- src/binpac_zeek-lib.pac | 2 +- src/bro_inet_ntop.c | 189 ------ src/bro_inet_ntop.h | 16 - src/bsd-getopt-long.c | 524 --------------- src/bsd-getopt-long.h | 127 ---- src/in_cksum.cc | 141 ---- src/input/readers/raw/Raw.cc | 2 +- src/modp_numtoa.c | 494 -------------- src/modp_numtoa.h | 114 ---- src/nb_dns.c | 794 ----------------------- src/nb_dns.h | 41 -- src/patricia.c | 1174 ---------------------------------- src/patricia.h | 195 ------ src/setsignal.c | 52 -- src/setsignal.h | 7 - src/strsep.c | 93 --- src/supervisor/Supervisor.cc | 2 +- src/threading/Formatter.cc | 2 +- src/util.cc | 2 +- src/util.h | 2 +- src/zeek-setup.cc | 2 +- 34 files changed, 33 insertions(+), 4994 deletions(-) delete mode 100644 src/ConvertUTF.c delete mode 100644 src/ConvertUTF.h delete mode 100644 src/bro_inet_ntop.c delete mode 100644 src/bro_inet_ntop.h delete mode 100644 src/bsd-getopt-long.c delete mode 100644 src/bsd-getopt-long.h delete mode 100644 src/in_cksum.cc delete mode 100644 src/modp_numtoa.c delete mode 100644 src/modp_numtoa.h delete mode 100644 src/nb_dns.c delete mode 100644 src/nb_dns.h delete mode 100644 src/patricia.c delete mode 100644 src/patricia.h delete mode 100644 src/setsignal.c delete mode 100644 src/setsignal.h delete mode 100644 src/strsep.c diff --git a/.clang-format-ignore b/.clang-format-ignore index 50f6bce6a5..1300d06344 100644 --- a/.clang-format-ignore +++ b/.clang-format-ignore @@ -1,17 +1,5 @@ # Ignore everything 3rdparty src/3rdparty/* -# These are files that are technically sourced from other places but aren't in 3rdparty -# and shouldn't be reformatted. -src/ConvertUTF.* -src/bro_inet_ntop.* -src/bsd-getopt-long.* -src/in_cksum.* -src/nb_dns.* -src/modp_numtoa.* -src/patricia.* -src/strsep.c -src/setsignal.c - # These files are generated code -src/DebugCmdInfoConstants.* \ No newline at end of file +src/DebugCmdInfoConstants.* diff --git a/COPYING.3rdparty b/COPYING.3rdparty index 4b21b90ab5..ffbb0dee0b 100644 --- a/COPYING.3rdparty +++ b/COPYING.3rdparty @@ -250,7 +250,7 @@ PROJECT (https://github.com/zeek) UNDER BSD LICENCE. ============================================================================== -%%% in_cksum.cc +%%% 3rdparty/in_cksum.cc ============================================================================== @@ -283,7 +283,7 @@ SUCH DAMAGE. ============================================================================== -%%% Patricia.c +%%% 3rdparty/patricia.c ============================================================================== @@ -328,7 +328,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ============================================================================== -%%% strsep.c +%%% 3rdparty/strsep.c ============================================================================== @@ -365,7 +365,7 @@ SUCH DAMAGE. ============================================================================== -%%% ConvertUTF.c +%%% 3rdparty/ConvertUTF.c ============================================================================== @@ -479,7 +479,7 @@ SUCH DAMAGE. ============================================================================== -%%% bsd-getopt-long.c +%%% 3rdparty/bsd-getopt-long.c ============================================================================== @@ -555,7 +555,7 @@ limitations under the License. ============================================================================== -%%% bro_inet_ntop.c +%%% 3rdparty/bro_inet_ntop.c ============================================================================== @@ -578,7 +578,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ============================================================================== -%%% modp_numtoa.h +%%% 3rdparty/modp_numtoa.h ============================================================================== diff --git a/src/3rdparty b/src/3rdparty index d31b51e6a0..4a8d0d86a8 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit d31b51e6a06ad4c71db81981920eb753954abbf8 +Subproject commit 4a8d0d86a8b38b40e19a47586abf9fe9b5ec9323 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c0f520fbeb..9f7eaffbab 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -280,7 +280,7 @@ add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ZAM-AssignFlavorsDefs.h WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) -set_source_files_properties(nb_dns.c PROPERTIES COMPILE_FLAGS +set_source_files_properties(3rdparty/nb_dns.c PROPERTIES COMPILE_FLAGS -fno-strict-aliasing) set(MAIN_SRCS @@ -297,7 +297,6 @@ set(MAIN_SRCS CCL.cc CompHash.cc Conn.cc - ConvertUTF.c DFA.cc DbgBreakpoint.cc DbgHelp.cc @@ -367,13 +366,6 @@ set(MAIN_SRCS ZeekArgs.cc ZeekString.cc ZVal.cc - bsd-getopt-long.c - bro_inet_ntop.c - in_cksum.cc - patricia.c - setsignal.c - strsep.c - modp_numtoa.c supervisor/Supervisor.cc @@ -437,12 +429,20 @@ set(MAIN_SRCS script_opt/ZAM/ZInst.cc script_opt/ZAM/ZOp.cc - nb_dns.c digest.h ) set(THIRD_PARTY_SRCS + 3rdparty/bro_inet_ntop.c + 3rdparty/bsd-getopt-long.c + 3rdparty/ConvertUTF.c + 3rdparty/in_cksum.cc + 3rdparty/modp_numtoa.c + 3rdparty/nb_dns.c + 3rdparty/patricia.c + 3rdparty/setsignal.c 3rdparty/sqlite3.c + 3rdparty/strsep.c ) set(GEN_ZAM_SRCS diff --git a/src/ConvertUTF.c b/src/ConvertUTF.c deleted file mode 100644 index b8acb69d27..0000000000 --- a/src/ConvertUTF.c +++ /dev/null @@ -1,755 +0,0 @@ -/*===--- ConvertUTF.c - Universal Character Names conversions ---------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is distributed under the University of Illinois Open Source - * License: - * - * University of Illinois/NCSA - * Open Source License - * - * Copyright (c) 2003-2014 University of Illinois at Urbana-Champaign. - * All rights reserved. - * - * Developed by: - * - * LLVM Team - * - * University of Illinois at Urbana-Champaign - * - * http://llvm.org - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal with the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * * Redistributions of source code must retain the above - * copyright notice, this list of conditions and the - * following disclaimers. - * - * * Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and - * the following disclaimers in the documentation and/or - * other materials provided with the distribution. - * - * * Neither the names of the LLVM Team, University of - * Illinois at Urbana-Champaign, nor the names of its - * contributors may be used to endorse or promote - * products derived from this Software without specific - * prior written permission. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE CONTRIBUTORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE - * USE OR OTHER DEALINGS WITH THE SOFTWARE. - * - *===------------------------------------------------------------------------=*/ -/* - * Copyright 2001-2004 Unicode, Inc. - * - * Disclaimer - * - * This source code is provided as is by Unicode, Inc. No claims are - * made as to fitness for any particular purpose. No warranties of any - * kind are expressed or implied. The recipient agrees to determine - * applicability of information provided. If this file has been - * purchased on magnetic or optical media from Unicode, Inc., the - * sole remedy for any claim will be exchange of defective media - * within 90 days of receipt. - * - * Limitations on Rights to Redistribute This Code - * - * Unicode, Inc. hereby grants the right to freely use the information - * supplied in this file in the creation of products supporting the - * Unicode Standard, and to make copies of this file in any form - * for internal or external distribution as long as this notice - * remains attached. - */ - -/* --------------------------------------------------------------------- - - Conversions between UTF32, UTF-16, and UTF-8. Source code file. - Author: Mark E. Davis, 1994. - Rev History: Rick McGowan, fixes & updates May 2001. - Sept 2001: fixed const & error conditions per - mods suggested by S. Parent & A. Lillich. - June 2002: Tim Dodd added detection and handling of incomplete - source sequences, enhanced error detection, added casts - to eliminate compiler warnings. - July 2003: slight mods to back out aggressive FFFE detection. - Jan 2004: updated switches in from-UTF8 conversions. - Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions. - - See the header file "ConvertUTF.h" for complete documentation. - ------------------------------------------------------------------------- */ - - -#include "zeek/ConvertUTF.h" -#ifdef CVTUTF_DEBUG -#include -#endif -#include - -static const int halfShift = 10; /* used for shifting by 10 bits */ - -static const UTF32 halfBase = 0x0010000UL; -static const UTF32 halfMask = 0x3FFUL; - -#define UNI_SUR_HIGH_START (UTF32)0xD800 -#define UNI_SUR_HIGH_END (UTF32)0xDBFF -#define UNI_SUR_LOW_START (UTF32)0xDC00 -#define UNI_SUR_LOW_END (UTF32)0xDFFF -#define false 0 -#define true 1 - -/* --------------------------------------------------------------------- */ - -/* - * Index into the table below with the first byte of a UTF-8 sequence to - * get the number of trailing bytes that are supposed to follow it. - * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is - * left as-is for anyone who may want to do such conversion, which was - * allowed in earlier algorithms. - */ -static const char trailingBytesForUTF8[256] = { - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 -}; - -/* - * Magic values subtracted from a buffer value during UTF8 conversion. - * This table contains as many values as there might be trailing bytes - * in a UTF-8 sequence. - */ -static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, - 0x03C82080UL, 0xFA082080UL, 0x82082080UL }; - -/* - * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed - * into the first byte, depending on how many bytes follow. There are - * as many entries in this table as there are UTF-8 sequence types. - * (I.e., one byte sequence, two byte... etc.). Remember that sequencs - * for *legal* UTF-8 will be 4 or fewer bytes total. - */ -static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; - -/* --------------------------------------------------------------------- */ - -/* The interface converts a whole buffer to avoid function-call overhead. - * Constants have been gathered. Loops & conditionals have been removed as - * much as possible for efficiency, in favor of drop-through switches. - * (See "Note A" at the bottom of the file for equivalent code.) - * If your compiler supports it, the "isLegalUTF8" call can be turned - * into an inline function. - */ - - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF32toUTF16 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF32* source = *sourceStart; - UTF16* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch; - if (target >= targetEnd) { - result = targetExhausted; break; - } - ch = *source++; - if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ - /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - if (flags == strictConversion) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - *target++ = (UTF16)ch; /* normal case */ - } - } else if (ch > UNI_MAX_LEGAL_UTF32) { - if (flags == strictConversion) { - result = sourceIllegal; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - /* target is a character in range 0xFFFF - 0x10FFFF. */ - if (target + 1 >= targetEnd) { - --source; /* Back up source pointer! */ - result = targetExhausted; break; - } - ch -= halfBase; - *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); - *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); - } - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF16toUTF32 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF16* source = *sourceStart; - UTF32* target = *targetStart; - UTF32 ch, ch2; - while (source < sourceEnd) { - const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ - ch = *source++; - /* If we have a surrogate pair, convert to UTF32 first. */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { - /* If the 16 bits following the high surrogate are in the source buffer... */ - if (source < sourceEnd) { - ch2 = *source; - /* If it's a low surrogate, convert to UTF32. */ - if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { - ch = ((ch - UNI_SUR_HIGH_START) << halfShift) - + (ch2 - UNI_SUR_LOW_START) + halfBase; - ++source; - } else if (flags == strictConversion) { /* it's an unpaired high surrogate */ - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } else { /* We don't have the 16 bits following the high surrogate. */ - --source; /* return to the high surrogate */ - result = sourceExhausted; - break; - } - } else if (flags == strictConversion) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - if (target >= targetEnd) { - source = oldSource; /* Back up source pointer! */ - result = targetExhausted; break; - } - *target++ = ch; - } - *sourceStart = source; - *targetStart = target; -#ifdef CVTUTF_DEBUG -if (result == sourceIllegal) { - fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2); - fflush(stderr); -} -#endif - return result; -} -ConversionResult ConvertUTF16toUTF8 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF16* source = *sourceStart; - UTF8* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch; - unsigned short bytesToWrite = 0; - const UTF32 byteMask = 0xBF; - const UTF32 byteMark = 0x80; - const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ - ch = *source++; - /* If we have a surrogate pair, convert to UTF32 first. */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { - /* If the 16 bits following the high surrogate are in the source buffer... */ - if (source < sourceEnd) { - UTF32 ch2 = *source; - /* If it's a low surrogate, convert to UTF32. */ - if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { - ch = ((ch - UNI_SUR_HIGH_START) << halfShift) - + (ch2 - UNI_SUR_LOW_START) + halfBase; - ++source; - } else if (flags == strictConversion) { /* it's an unpaired high surrogate */ - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } else { /* We don't have the 16 bits following the high surrogate. */ - --source; /* return to the high surrogate */ - result = sourceExhausted; - break; - } - } else if (flags == strictConversion) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - /* Figure out how many bytes the result will require */ - if (ch < (UTF32)0x80) { bytesToWrite = 1; - } else if (ch < (UTF32)0x800) { bytesToWrite = 2; - } else if (ch < (UTF32)0x10000) { bytesToWrite = 3; - } else if (ch < (UTF32)0x110000) { bytesToWrite = 4; - } else { bytesToWrite = 3; - ch = UNI_REPLACEMENT_CHAR; - } - - target += bytesToWrite; - if (target > targetEnd) { - source = oldSource; /* Back up source pointer! */ - target -= bytesToWrite; result = targetExhausted; break; - } - switch (bytesToWrite) { /* note: everything falls through. */ - case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 1: *--target = (UTF8)(ch | firstByteMark[bytesToWrite]); - } - target += bytesToWrite; - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF32toUTF8 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF32* source = *sourceStart; - UTF8* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch; - unsigned short bytesToWrite = 0; - const UTF32 byteMask = 0xBF; - const UTF32 byteMark = 0x80; - ch = *source++; - if (flags == strictConversion ) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - /* - * Figure out how many bytes the result will require. Turn any - * illegally large UTF32 things (> Plane 17) into replacement chars. - */ - if (ch < (UTF32)0x80) { bytesToWrite = 1; - } else if (ch < (UTF32)0x800) { bytesToWrite = 2; - } else if (ch < (UTF32)0x10000) { bytesToWrite = 3; - } else if (ch <= UNI_MAX_LEGAL_UTF32) { bytesToWrite = 4; - } else { bytesToWrite = 3; - ch = UNI_REPLACEMENT_CHAR; - result = sourceIllegal; - } - - target += bytesToWrite; - if (target > targetEnd) { - --source; /* Back up source pointer! */ - target -= bytesToWrite; result = targetExhausted; break; - } - switch (bytesToWrite) { /* note: everything falls through. */ - case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]); - } - target += bytesToWrite; - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -/* - * Utility routine to tell whether a sequence of bytes is legal UTF-8. - * This must be called with the length pre-determined by the first byte. - * If not calling this from ConvertUTF8to*, then the length can be set by: - * length = trailingBytesForUTF8[*source]+1; - * and the sequence is illegal right away if there aren't that many bytes - * available. - * If presented with a length > 4, this returns false. The Unicode - * definition of UTF-8 goes up to 4-byte sequences. - */ - -static Boolean isLegalUTF8(const UTF8 *source, int length) { - UTF8 a; - const UTF8 *srcptr = source+length; - switch (length) { - default: return false; - /* Everything else falls through when "true"... */ - case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; - case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; - case 2: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; - - switch (*source) { - /* no fall-through in this inner switch */ - case 0xE0: if (a < 0xA0) return false; break; - case 0xED: if (a > 0x9F) return false; break; - case 0xF0: if (a < 0x90) return false; break; - case 0xF4: if (a > 0x8F) return false; break; - default: if (a < 0x80) return false; - } - - case 1: if (*source >= 0x80 && *source < 0xC2) return false; - } - if (*source > 0xF4) return false; - return true; -} - -/* --------------------------------------------------------------------- */ - -/* - * Exported function to return whether a UTF-8 sequence is legal or not. - * This is not used here; it's just exported. - */ -Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) { - int length = trailingBytesForUTF8[*source]+1; - if (length > sourceEnd - source) { - return false; - } - return isLegalUTF8(source, length); -} - -/* --------------------------------------------------------------------- */ - -static unsigned -findMaximalSubpartOfIllFormedUTF8Sequence(const UTF8 *source, - const UTF8 *sourceEnd) { - UTF8 b1, b2, b3; - - assert(!isLegalUTF8Sequence(source, sourceEnd)); - - /* - * Unicode 6.3.0, D93b: - * - * Maximal subpart of an ill-formed subsequence: The longest code unit - * subsequence starting at an unconvertible offset that is either: - * a. the initial subsequence of a well-formed code unit sequence, or - * b. a subsequence of length one. - */ - - if (source == sourceEnd) - return 0; - - /* - * Perform case analysis. See Unicode 6.3.0, Table 3-7. Well-Formed UTF-8 - * Byte Sequences. - */ - - b1 = *source; - ++source; - if (b1 >= 0xC2 && b1 <= 0xDF) { - /* - * First byte is valid, but we know that this code unit sequence is - * invalid, so the maximal subpart has to end after the first byte. - */ - return 1; - } - - if (source == sourceEnd) - return 1; - - b2 = *source; - ++source; - - if (b1 == 0xE0) { - return (b2 >= 0xA0 && b2 <= 0xBF) ? 2 : 1; - } - if (b1 >= 0xE1 && b1 <= 0xEC) { - return (b2 >= 0x80 && b2 <= 0xBF) ? 2 : 1; - } - if (b1 == 0xED) { - return (b2 >= 0x80 && b2 <= 0x9F) ? 2 : 1; - } - if (b1 >= 0xEE && b1 <= 0xEF) { - return (b2 >= 0x80 && b2 <= 0xBF) ? 2 : 1; - } - if (b1 == 0xF0) { - if (b2 >= 0x90 && b2 <= 0xBF) { - if (source == sourceEnd) - return 2; - - b3 = *source; - return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2; - } - return 1; - } - if (b1 >= 0xF1 && b1 <= 0xF3) { - if (b2 >= 0x80 && b2 <= 0xBF) { - if (source == sourceEnd) - return 2; - - b3 = *source; - return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2; - } - return 1; - } - if (b1 == 0xF4) { - if (b2 >= 0x80 && b2 <= 0x8F) { - if (source == sourceEnd) - return 2; - - b3 = *source; - return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2; - } - return 1; - } - - assert((b1 >= 0x80 && b1 <= 0xC1) || b1 >= 0xF5); - /* - * There are no valid sequences that start with these bytes. Maximal subpart - * is defined to have length 1 in these cases. - */ - return 1; -} - -/* --------------------------------------------------------------------- */ - -/* - * Exported function to return the total number of bytes in a codepoint - * represented in UTF-8, given the value of the first byte. - */ -unsigned getNumBytesForUTF8(UTF8 first) { - return trailingBytesForUTF8[first] + 1; -} - -/* --------------------------------------------------------------------- */ - -/* - * Exported function to return whether a UTF-8 string is legal or not. - * This is not used here; it's just exported. - */ -Boolean isLegalUTF8String(const UTF8 **source, const UTF8 *sourceEnd) { - while (*source != sourceEnd) { - int length = trailingBytesForUTF8[**source] + 1; - if (length > sourceEnd - *source || !isLegalUTF8(*source, length)) - return false; - *source += length; - } - return true; -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF8toUTF16 ( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF8* source = *sourceStart; - UTF16* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch = 0; - unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; - if (extraBytesToRead >= sourceEnd - source) { - result = sourceExhausted; break; - } - /* Do this check whether lenient or strict */ - if (!isLegalUTF8(source, extraBytesToRead+1)) { - result = sourceIllegal; - break; - } - /* - * The cases all fall through. See "Note A" below. - */ - switch (extraBytesToRead) { - case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ - case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ - case 3: ch += *source++; ch <<= 6; - case 2: ch += *source++; ch <<= 6; - case 1: ch += *source++; ch <<= 6; - case 0: ch += *source++; - } - ch -= offsetsFromUTF8[extraBytesToRead]; - - if (target >= targetEnd) { - source -= (extraBytesToRead+1); /* Back up source pointer! */ - result = targetExhausted; break; - } - if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - if (flags == strictConversion) { - source -= (extraBytesToRead+1); /* return to the illegal value itself */ - result = sourceIllegal; - break; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - *target++ = (UTF16)ch; /* normal case */ - } - } else if (ch > UNI_MAX_UTF16) { - if (flags == strictConversion) { - result = sourceIllegal; - source -= (extraBytesToRead+1); /* return to the start */ - break; /* Bail out; shouldn't continue */ - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - /* target is a character in range 0xFFFF - 0x10FFFF. */ - if (target + 1 >= targetEnd) { - source -= (extraBytesToRead+1); /* Back up source pointer! */ - result = targetExhausted; break; - } - ch -= halfBase; - *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); - *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); - } - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -static ConversionResult ConvertUTF8toUTF32Impl( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags, - Boolean InputIsPartial) { - ConversionResult result = conversionOK; - const UTF8* source = *sourceStart; - UTF32* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch = 0; - unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; - if (extraBytesToRead >= sourceEnd - source) { - if (flags == strictConversion || InputIsPartial) { - result = sourceExhausted; - break; - } else { - result = sourceIllegal; - - /* - * Replace the maximal subpart of ill-formed sequence with - * replacement character. - */ - source += findMaximalSubpartOfIllFormedUTF8Sequence(source, - sourceEnd); - *target++ = UNI_REPLACEMENT_CHAR; - continue; - } - } - if (target >= targetEnd) { - result = targetExhausted; break; - } - - /* Do this check whether lenient or strict */ - if (!isLegalUTF8(source, extraBytesToRead+1)) { - result = sourceIllegal; - if (flags == strictConversion) { - /* Abort conversion. */ - break; - } else { - /* - * Replace the maximal subpart of ill-formed sequence with - * replacement character. - */ - source += findMaximalSubpartOfIllFormedUTF8Sequence(source, - sourceEnd); - *target++ = UNI_REPLACEMENT_CHAR; - continue; - } - } - /* - * The cases all fall through. See "Note A" below. - */ - switch (extraBytesToRead) { - case 5: ch += *source++; ch <<= 6; - case 4: ch += *source++; ch <<= 6; - case 3: ch += *source++; ch <<= 6; - case 2: ch += *source++; ch <<= 6; - case 1: ch += *source++; ch <<= 6; - case 0: ch += *source++; - } - ch -= offsetsFromUTF8[extraBytesToRead]; - - if (ch <= UNI_MAX_LEGAL_UTF32) { - /* - * UTF-16 surrogate values are illegal in UTF-32, and anything - * over Plane 17 (> 0x10FFFF) is illegal. - */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - if (flags == strictConversion) { - source -= (extraBytesToRead+1); /* return to the illegal value itself */ - result = sourceIllegal; - break; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - *target++ = ch; - } - } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */ - result = sourceIllegal; - *target++ = UNI_REPLACEMENT_CHAR; - } - } - *sourceStart = source; - *targetStart = target; - return result; -} - -ConversionResult ConvertUTF8toUTF32Partial(const UTF8 **sourceStart, - const UTF8 *sourceEnd, - UTF32 **targetStart, - UTF32 *targetEnd, - ConversionFlags flags) { - return ConvertUTF8toUTF32Impl(sourceStart, sourceEnd, targetStart, targetEnd, - flags, /*InputIsPartial=*/true); -} - -ConversionResult ConvertUTF8toUTF32(const UTF8 **sourceStart, - const UTF8 *sourceEnd, UTF32 **targetStart, - UTF32 *targetEnd, ConversionFlags flags) { - return ConvertUTF8toUTF32Impl(sourceStart, sourceEnd, targetStart, targetEnd, - flags, /*InputIsPartial=*/false); -} - -/* --------------------------------------------------------------------- - - Note A. - The fall-through switches in UTF-8 reading code save a - temp variable, some decrements & conditionals. The switches - are equivalent to the following loop: - { - int tmpBytesToRead = extraBytesToRead+1; - do { - ch += *source++; - --tmpBytesToRead; - if (tmpBytesToRead) ch <<= 6; - } while (tmpBytesToRead > 0); - } - In UTF-8 writing code, the switches on "bytesToWrite" are - similarly unrolled loops. - - --------------------------------------------------------------------- */ diff --git a/src/ConvertUTF.h b/src/ConvertUTF.h deleted file mode 100644 index fe7939914e..0000000000 --- a/src/ConvertUTF.h +++ /dev/null @@ -1,233 +0,0 @@ -/*===--- ConvertUTF.h - Universal Character Names conversions ---------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is distributed under the University of Illinois Open Source - * License: - * - * University of Illinois/NCSA - * Open Source License - * - * Copyright (c) 2003-2014 University of Illinois at Urbana-Champaign. - * All rights reserved. - * - * Developed by: - * - * LLVM Team - * - * University of Illinois at Urbana-Champaign - * - * http://llvm.org - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal with the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * * Redistributions of source code must retain the above - * copyright notice, this list of conditions and the - * following disclaimers. - * - * * Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and - * the following disclaimers in the documentation and/or - * other materials provided with the distribution. - * - * * Neither the names of the LLVM Team, University of - * Illinois at Urbana-Champaign, nor the names of its - * contributors may be used to endorse or promote - * products derived from this Software without specific - * prior written permission. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE CONTRIBUTORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE - * USE OR OTHER DEALINGS WITH THE SOFTWARE. - * - *==------------------------------------------------------------------------==*/ -/* - * Copyright 2001-2004 Unicode, Inc. - * - * Disclaimer - * - * This source code is provided as is by Unicode, Inc. No claims are - * made as to fitness for any particular purpose. No warranties of any - * kind are expressed or implied. The recipient agrees to determine - * applicability of information provided. If this file has been - * purchased on magnetic or optical media from Unicode, Inc., the - * sole remedy for any claim will be exchange of defective media - * within 90 days of receipt. - * - * Limitations on Rights to Redistribute This Code - * - * Unicode, Inc. hereby grants the right to freely use the information - * supplied in this file in the creation of products supporting the - * Unicode Standard, and to make copies of this file in any form - * for internal or external distribution as long as this notice - * remains attached. - */ - -/* --------------------------------------------------------------------- - - Conversions between UTF32, UTF-16, and UTF-8. Header file. - - Several funtions are included here, forming a complete set of - conversions between the three formats. UTF-7 is not included - here, but is handled in a separate source file. - - Each of these routines takes pointers to input buffers and output - buffers. The input buffers are const. - - Each routine converts the text between *sourceStart and sourceEnd, - putting the result into the buffer between *targetStart and - targetEnd. Note: the end pointers are *after* the last item: e.g. - *(sourceEnd - 1) is the last item. - - !!! NOTE: The source and end pointers must be aligned properly !!! - - The return result indicates whether the conversion was successful, - and if not, whether the problem was in the source or target buffers. - (Only the first encountered problem is indicated.) - - After the conversion, *sourceStart and *targetStart are both - updated to point to the end of last text successfully converted in - the respective buffers. - - Input parameters: - sourceStart - pointer to a pointer to the source buffer. - The contents of this are modified on return so that - it points at the next thing to be converted. - targetStart - similarly, pointer to pointer to the target buffer. - sourceEnd, targetEnd - respectively pointers to the ends of the - two buffers, for overflow checking only. - - These conversion functions take a ConversionFlags argument. When this - flag is set to strict, both irregular sequences and isolated surrogates - will cause an error. When the flag is set to lenient, both irregular - sequences and isolated surrogates are converted. - - Whether the flag is strict or lenient, all illegal sequences will cause - an error return. This includes sequences such as: , , - or in UTF-8, and values above 0x10FFFF in UTF-32. Conformant code - must check for illegal sequences. - - When the flag is set to lenient, characters over 0x10FFFF are converted - to the replacement character; otherwise (when the flag is set to strict) - they constitute an error. - - Output parameters: - The value "sourceIllegal" is returned from some routines if the input - sequence is malformed. When "sourceIllegal" is returned, the source - value will point to the illegal value that caused the problem. E.g., - in UTF-8 when a sequence is malformed, it points to the start of the - malformed sequence. - - Author: Mark E. Davis, 1994. - Rev History: Rick McGowan, fixes & updates May 2001. - Fixes & updates, Sept 2001. - ------------------------------------------------------------------------- */ - -#pragma once - -/* --------------------------------------------------------------------- - The following 4 definitions are compiler-specific. - The C standard does not guarantee that wchar_t has at least - 16 bits, so wchar_t is no less portable than unsigned short! - All should be unsigned values to avoid sign extension during - bit mask & shift operations. ------------------------------------------------------------------------- */ - -typedef unsigned int UTF32; /* at least 32 bits */ -typedef unsigned short UTF16; /* at least 16 bits */ -typedef unsigned char UTF8; /* typically 8 bits */ -typedef unsigned char Boolean; /* 0 or 1 */ - -/* Some fundamental constants */ -#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD -#define UNI_MAX_BMP (UTF32)0x0000FFFF -#define UNI_MAX_UTF16 (UTF32)0x0010FFFF -#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF -#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF - -#define UNI_MAX_UTF8_BYTES_PER_CODE_POINT 4 - -#define UNI_UTF16_BYTE_ORDER_MARK_NATIVE 0xFEFF -#define UNI_UTF16_BYTE_ORDER_MARK_SWAPPED 0xFFFE - -typedef enum { - conversionOK, /* conversion successful */ - sourceExhausted, /* partial character in source, but hit end */ - targetExhausted, /* insuff. room in target for conversion */ - sourceIllegal /* source sequence is illegal/malformed */ -} ConversionResult; - -typedef enum { - strictConversion = 0, - lenientConversion -} ConversionFlags; - -/* This is for C++ and does no harm in C */ -#ifdef __cplusplus -extern "C" { -#endif - -ConversionResult ConvertUTF8toUTF16 ( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags); - -/** - * Convert a partial UTF8 sequence to UTF32. If the sequence ends in an - * incomplete code unit sequence, returns \c sourceExhausted. - */ -ConversionResult ConvertUTF8toUTF32Partial( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags); - -/** - * Convert a partial UTF8 sequence to UTF32. If the sequence ends in an - * incomplete code unit sequence, returns \c sourceIllegal. - */ -ConversionResult ConvertUTF8toUTF32( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags); - -/* NOTE: The source and end pointers must be aligned properly. */ -ConversionResult ConvertUTF16toUTF8 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags); - -/* NOTE: The source and end pointers must be aligned properly. */ -ConversionResult ConvertUTF32toUTF8 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags); - -/* NOTE: The source and end pointers must be aligned properly. */ -ConversionResult ConvertUTF16toUTF32 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags); - -/* NOTE: The source and end pointers must be aligned properly. */ -ConversionResult ConvertUTF32toUTF16 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags); - -Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd); - -Boolean isLegalUTF8String(const UTF8 **source, const UTF8 *sourceEnd); - -unsigned getNumBytesForUTF8(UTF8 firstByte); - -#ifdef __cplusplus -} -#endif - -/* --------------------------------------------------------------------- */ diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 827f17ecfe..8d776b7351 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -46,7 +46,7 @@ extern "C" #include -#include "zeek/nb_dns.h" +#include "zeek/3rdparty/nb_dns.h" } using namespace std; diff --git a/src/Debug.cc b/src/Debug.cc index eb08b54207..469d016f2d 100644 --- a/src/Debug.cc +++ b/src/Debug.cc @@ -34,7 +34,7 @@ extern "C" { -#include "zeek/setsignal.h" +#include "zeek/3rdparty/setsignal.h" } using namespace std; diff --git a/src/Desc.cc b/src/Desc.cc index 7eac71491d..1712cea05c 100644 --- a/src/Desc.cc +++ b/src/Desc.cc @@ -9,7 +9,7 @@ #include #include -#include "zeek/ConvertUTF.h" +#include "zeek/3rdparty/ConvertUTF.h" #include "zeek/File.h" #include "zeek/IPAddr.h" #include "zeek/Reporter.h" diff --git a/src/IPAddr.cc b/src/IPAddr.cc index 7c88b82ed9..a957ce4d10 100644 --- a/src/IPAddr.cc +++ b/src/IPAddr.cc @@ -6,12 +6,12 @@ #include #include +#include "zeek/3rdparty/bro_inet_ntop.h" #include "zeek/Conn.h" #include "zeek/Hash.h" #include "zeek/Reporter.h" #include "zeek/ZeekString.h" #include "zeek/analyzer/Manager.h" -#include "zeek/bro_inet_ntop.h" namespace zeek { diff --git a/src/Options.cc b/src/Options.cc index 9bc235cbcc..374ae84efe 100644 --- a/src/Options.cc +++ b/src/Options.cc @@ -17,7 +17,7 @@ #include #include -#include "zeek/bsd-getopt-long.h" +#include "zeek/3rdparty/bsd-getopt-long.h" #include "zeek/logging/writers/ascii/Ascii.h" namespace zeek diff --git a/src/PrefixTable.h b/src/PrefixTable.h index 3322e9a67d..3404041930 100644 --- a/src/PrefixTable.h +++ b/src/PrefixTable.h @@ -2,7 +2,7 @@ extern "C" { -#include "zeek/patricia.h" +#include "zeek/3rdparty/patricia.h" } #include diff --git a/src/RunState.cc b/src/RunState.cc index 9852fc0744..2161e7ba30 100644 --- a/src/RunState.cc +++ b/src/RunState.cc @@ -22,7 +22,7 @@ extern "C" { -#include "zeek/setsignal.h" +#include "zeek/3rdparty/setsignal.h" }; #include "zeek/Anon.h" diff --git a/src/binpac_zeek-lib.pac b/src/binpac_zeek-lib.pac index d4ed3ba96d..a4748d45ea 100644 --- a/src/binpac_zeek-lib.pac +++ b/src/binpac_zeek-lib.pac @@ -1,9 +1,9 @@ %extern{ +#include "zeek/3rdparty/ConvertUTF.h" #include "zeek/binpac_zeek.h" #include "zeek/util.h" #include "zeek/Reporter.h" #include "zeek/Val.h" -#include "zeek/ConvertUTF.h" #include "zeek/RunState.h" %} diff --git a/src/bro_inet_ntop.c b/src/bro_inet_ntop.c deleted file mode 100644 index 40ec26ca25..0000000000 --- a/src/bro_inet_ntop.c +++ /dev/null @@ -1,189 +0,0 @@ -/* Taken/adapted from FreeBSD 9.0.0 inet_ntop.c (CVS revision 1.3.16.1.2.1) */ -/* - * Copyright (c) 2004 by Internet Systems Consortium, Inc. ("ISC") - * Copyright (c) 1996-1999 by Internet Software Consortium. - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT - * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#include "zeek/bro_inet_ntop.h" - -#include -#include -#include - -#include -#include -#include - -#include -#include -#include - -/*% - * WARNING: Don't even consider trying to compile this on a system where - * sizeof(int) < 4. sizeof(int) > 4 is fine; all the world's not a VAX. - */ - -static const char *bro_inet_ntop4(const u_char *src, char *dst, socklen_t size); -static const char *bro_inet_ntop6(const u_char *src, char *dst, socklen_t size); - -/* char * - * bro_inet_ntop(af, src, dst, size) - * convert a network format address to presentation format. - * return: - * pointer to presentation format address (`dst'), or NULL (see errno). - * author: - * Paul Vixie, 1996. - */ -const char * -bro_inet_ntop(int af, const void * __restrict src, char * __restrict dst, - socklen_t size) -{ - switch (af) { - case AF_INET: - return (bro_inet_ntop4(src, dst, size)); - case AF_INET6: - return (bro_inet_ntop6(src, dst, size)); - default: - errno = EAFNOSUPPORT; - return (NULL); - } - /* NOTREACHED */ -} - -/* const char * - * bro_inet_ntop4(src, dst, size) - * format an IPv4 address - * return: - * `dst' (as a const) - * notes: - * (1) uses no statics - * (2) takes a u_char* not an in_addr as input - * author: - * Paul Vixie, 1996. Modified by Jon Siwek, 2012, to replace strlcpy - */ -static const char * -bro_inet_ntop4(const u_char *src, char *dst, socklen_t size) -{ - static const char fmt[] = "%u.%u.%u.%u"; - char tmp[sizeof "255.255.255.255"]; - int l; - - l = snprintf(tmp, sizeof(tmp), fmt, src[0], src[1], src[2], src[3]); - if (l <= 0 || (socklen_t) l >= size) { - errno = ENOSPC; - return (NULL); - } - strncpy(dst, tmp, size - 1); - dst[size - 1] = 0; - return (dst); -} - -/* const char * - * bro_inet_ntop6(src, dst, size) - * convert IPv6 binary address into presentation (printable) format - * author: - * Paul Vixie, 1996. Modified by Jon Siwek, 2012, for IPv4-translated format - */ -static const char * -bro_inet_ntop6(const u_char *src, char *dst, socklen_t size) -{ - /* - * Note that int32_t and int16_t need only be "at least" large enough - * to contain a value of the specified size. On some systems, like - * Crays, there is no such thing as an integer variable with 16 bits. - * Keep this in mind if you think this function should have been coded - * to use pointer overlays. All the world's not a VAX. - */ - char tmp[sizeof "ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255"], *tp; - struct { int base, len; } best, cur; - u_int words[NS_IN6ADDRSZ / NS_INT16SZ]; - int i; - - /* - * Preprocess: - * Copy the input (bytewise) array into a wordwise array. - * Find the longest run of 0x00's in src[] for :: shorthanding. - */ - memset(words, '\0', sizeof words); - for (i = 0; i < NS_IN6ADDRSZ; i++) - words[i / 2] |= (src[i] << ((1 - (i % 2)) << 3)); - best.base = -1; - best.len = 0; - cur.base = -1; - cur.len = 0; - for (i = 0; i < (NS_IN6ADDRSZ / NS_INT16SZ); i++) { - if (words[i] == 0) { - if (cur.base == -1) - cur.base = i, cur.len = 1; - else - cur.len++; - } else { - if (cur.base != -1) { - if (best.base == -1 || cur.len > best.len) - best = cur; - cur.base = -1; - } - } - } - if (cur.base != -1) { - if (best.base == -1 || cur.len > best.len) - best = cur; - } - if (best.base != -1 && best.len < 2) - best.base = -1; - - /* - * Format the result. - */ - tp = tmp; - for (i = 0; i < (NS_IN6ADDRSZ / NS_INT16SZ); i++) { - /* Are we inside the best run of 0x00's? */ - if (best.base != -1 && i >= best.base && - i < (best.base + best.len)) { - if (i == best.base) - *tp++ = ':'; - continue; - } - /* Are we following an initial run of 0x00s or any real hex? */ - if (i != 0) - *tp++ = ':'; - /* Is this address an encapsulated IPv4? */ - if (i == 6 && best.base == 0 && (best.len == 6 || - (best.len == 7 && words[7] != 0x0001) || - (best.len == 5 && words[5] == 0xffff) || - (best.len == 4 && words[4] == 0xffff && words[5] == 0))) { - if (!bro_inet_ntop4(src+12, tp, sizeof tmp - (tp - tmp))) - return (NULL); - tp += strlen(tp); - break; - } - tp += sprintf(tp, "%x", words[i]); - } - /* Was it a trailing run of 0x00's? */ - if (best.base != -1 && (best.base + best.len) == - (NS_IN6ADDRSZ / NS_INT16SZ)) - *tp++ = ':'; - *tp++ = '\0'; - - /* - * Check for overflow, copy, and we're done. - */ - if ((socklen_t)(tp - tmp) > size) { - errno = ENOSPC; - return (NULL); - } - strcpy(dst, tmp); - return (dst); -} diff --git a/src/bro_inet_ntop.h b/src/bro_inet_ntop.h deleted file mode 100644 index 61d1df7635..0000000000 --- a/src/bro_inet_ntop.h +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include - -const char * -bro_inet_ntop(int af, const void * __restrict src, char * __restrict dst, - socklen_t size); - -#ifdef __cplusplus -} -#endif diff --git a/src/bsd-getopt-long.c b/src/bsd-getopt-long.c deleted file mode 100644 index 6097010857..0000000000 --- a/src/bsd-getopt-long.c +++ /dev/null @@ -1,524 +0,0 @@ -/* $OpenBSD: getopt_long.c,v 1.17 2004/06/03 18:46:52 millert Exp $ */ -/* $NetBSD: getopt_long.c,v 1.15 2002/01/31 22:43:40 tv Exp $ */ - -/* - * Copyright (c) 2002 Todd C. Miller - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND TODD C. MILLER DISCLAIMS ALL - * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL TODD C. MILLER BE LIABLE - * FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION - * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN - * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ -/*- - * Copyright (c) 2000 The NetBSD Foundation, Inc. - * All rights reserved. - * - * This code is derived from software contributed to The NetBSD Foundation - * by Dieter Baron and Thomas Klausner. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the NetBSD - * Foundation, Inc. and its contributors. - * 4. Neither the name of The NetBSD Foundation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#define IN_GETOPT_LONG_C 1 - -#include -#include -#include -#include -#include - -#ifndef HAVE_GETOPT_LONG - -# include "bsd-getopt-long.h" - -# ifdef WITH_DMALLOC -# include -# endif - -int pure_opterr = 1; /* if error message should be printed */ -int pure_optind = 1; /* index into parent argv vector */ -int pure_optopt = '?'; /* character checked for validity */ -int pure_optreset; /* reset getopt */ -const char *pure_optarg; /* argument associated with option */ - -# define PRINT_ERROR ((pure_opterr) && (*options != ':')) - -# define FLAG_PERMUTE 0x01 /* permute non-options to the end of argv */ -# define FLAG_ALLARGS 0x02 /* treat non-options as args to option "-1" */ -# define FLAG_LONGONLY 0x04 /* operate as pure_getopt_long_only */ - -/* return values */ -# define BADCH (int)'?' -# define BADARG ((*options == ':') ? (int)':' : (int)'?') -# define INORDER (int)1 - -# define EMSG "" - -static int pure_getopt_internal(int, char * const *, const char *, - const struct pure_option *, int *, int); -static int pure_parse_long_options(char * const *, const char *, - const struct pure_option *, int *, int); -static int pure_gcd(int, int); -static void pure_permute_args(int, int, int, char * const *); - -static const char *pure_place = EMSG; /* option letter processing */ - -/* XXX: set pure_optreset to 1 rather than these two */ -static int nonopt_start = -1; /* first non option argument (for permute) */ -static int nonopt_end = -1; /* first option after non options (for permute) */ - -/* Error messages */ -static const char *recargchar = "option requires an argument -- %c\n"; -static const char *recargstring = "option requires an argument -- %s\n"; -static const char *ambig = "ambiguous option -- %.*s\n"; -static const char *noarg = "option doesn't take an argument -- %.*s\n"; -static const char *illoptchar = "unknown option -- %c\n"; -static const char *illoptstring = "unknown option -- %s\n"; - -/* - * Compute the greatest common divisor of a and b. - */ -static int pure_gcd(int a, int b) -{ - int c; - - c = a % b; - while (c != 0) { - a = b; - b = c; - c = a % b; - } - return b; -} - -/* - * Exchange the block from nonopt_start to nonopt_end with the block - * from nonopt_end to opt_end (keeping the same order of arguments - * in each block). - */ -static void pure_permute_args(int panonopt_start, int panonopt_end, - int opt_end, char * const *nargv) -{ - int cstart, cyclelen, i, j, ncycle, nnonopts, nopts, pos; - char *swap; - - /* - * compute lengths of blocks and number and size of cycles - */ - nnonopts = panonopt_end - panonopt_start; - nopts = opt_end - panonopt_end; - ncycle = pure_gcd(nnonopts, nopts); - cyclelen = (opt_end - panonopt_start) / ncycle; - - for (i = 0; i < ncycle; i++) { - cstart = panonopt_end+i; - pos = cstart; - for (j = 0; j < cyclelen; j++) { - if (pos >= panonopt_end) - pos -= nnonopts; - else - pos += nopts; - swap = nargv[pos]; - /* LINTED const cast */ - ((char **) nargv)[pos] = nargv[cstart]; - /* LINTED const cast */ - ((char **)nargv)[cstart] = swap; - } - } -} - -/* - * pure_parse_long_options -- - * Parse long options in argc/argv argument vector. - * Returns -1 if short_too is set and the option does not match long_options. - */ -static int pure_parse_long_options(char * const *nargv, const char *options, - const struct pure_option *long_options, - int *idx, int short_too) -{ - const char *current_argv, *has_equal; - size_t current_argv_len; - int i, match; - - current_argv = pure_place; - match = -1; - - pure_optind++; - - if ((has_equal = strchr(current_argv, '=')) != NULL) { - /* argument found (--option=arg) */ - current_argv_len = has_equal - current_argv; - has_equal++; - } else - current_argv_len = strlen(current_argv); - - for (i = 0; long_options[i].name; i++) { - /* find matching long option */ - if (strncmp(current_argv, long_options[i].name, - current_argv_len)) - continue; - - if (strlen(long_options[i].name) == current_argv_len) { - /* exact match */ - match = i; - break; - } - /* - * If this is a known short option, don't allow - * a partial match of a single character. - */ - if (short_too && current_argv_len == 1) - continue; - - if (match == -1) /* partial match */ - match = i; - else { - /* ambiguous abbreviation */ - if (PRINT_ERROR) - fprintf(stderr, ambig, (int)current_argv_len, - current_argv); - pure_optopt = 0; - return BADCH; - } - } - if (match != -1) { /* option found */ - if (long_options[match].has_arg == no_argument - && has_equal) { - if (PRINT_ERROR) - fprintf(stderr, noarg, (int)current_argv_len, - current_argv); - /* - * XXX: GNU sets pure_optopt to val regardless of flag - */ - if (long_options[match].flag == NULL) - pure_optopt = long_options[match].val; - else - pure_optopt = 0; - return BADARG; - } - if (long_options[match].has_arg == required_argument || - long_options[match].has_arg == optional_argument) { - if (has_equal) - pure_optarg = has_equal; - else if (long_options[match].has_arg == - required_argument) { - /* - * optional argument doesn't use next nargv - */ - pure_optarg = nargv[pure_optind++]; - } - } - if ((long_options[match].has_arg == required_argument) - && (pure_optarg == NULL)) { - /* - * Missing argument; leading ':' indicates no error - * should be generated. - */ - if (PRINT_ERROR) - fprintf(stderr, recargstring, - current_argv); - /* - * XXX: GNU sets pure_optopt to val regardless of flag - */ - if (long_options[match].flag == NULL) - pure_optopt = long_options[match].val; - else - pure_optopt = 0; - --pure_optind; - return BADARG; - } - } else { /* unknown option */ - if (short_too) { - --pure_optind; - return -1; - } - if (PRINT_ERROR) - fprintf(stderr, illoptstring, current_argv); - pure_optopt = 0; - return BADCH; - } - if (idx) - *idx = match; - if (long_options[match].flag) { - *long_options[match].flag = long_options[match].val; - return 0; - } else - return long_options[match].val; -} - -/* - * getopt_internal -- - * Parse argc/argv argument vector. Called by user level routines. - */ -static int pure_getopt_internal(int nargc, char * const *nargv, - const char *options, - const struct pure_option *long_options, - int *idx, int flags) -{ - char *oli; /* option letter list index */ - int optchar, short_too; - static int posixly_correct = -1; - - if (options == NULL) - return -1; - - /* - * Disable GNU extensions if POSIXLY_CORRECT is set or options - * string begins with a '+'. - */ - if (posixly_correct == -1) - posixly_correct = (getenv("POSIXLY_CORRECT") != NULL); - if (posixly_correct || *options == '+') - flags &= ~FLAG_PERMUTE; - else if (*options == '-') - flags |= FLAG_ALLARGS; - if (*options == '+' || *options == '-') - options++; - - /* - * XXX Some GNU programs (like cvs) set pure_optind to 0 instead of - * XXX using pure_optreset. Work around this braindamage. - */ - if (pure_optind == 0) - pure_optind = pure_optreset = 1; - - pure_optarg = NULL; - if (pure_optreset) - nonopt_start = nonopt_end = -1; - start: - if (pure_optreset || !*pure_place) { /* update scanning pointer */ - pure_optreset = 0; - if (pure_optind >= nargc) { /* end of argument vector */ - pure_place = EMSG; - if (nonopt_end != -1) { - /* do permutation, if we have to */ - pure_permute_args(nonopt_start, nonopt_end, - pure_optind, nargv); - pure_optind -= nonopt_end - nonopt_start; - } - else if (nonopt_start != -1) { - /* - * If we skipped non-options, set pure_optind - * to the first of them. - */ - pure_optind = nonopt_start; - } - nonopt_start = nonopt_end = -1; - return -1; - } - if (*(pure_place = nargv[pure_optind]) != '-' || - (pure_place[1] == '\0' && strchr(options, '-') == NULL)) { - pure_place = EMSG; /* found non-option */ - if (flags & FLAG_ALLARGS) { - /* - * GNU extension: - * return non-option as argument to option 1 - */ - pure_optarg = nargv[pure_optind++]; - return INORDER; - } - if (!(flags & FLAG_PERMUTE)) { - /* - * If no permutation wanted, stop parsing - * at first non-option. - */ - return -1; - } - /* do permutation */ - if (nonopt_start == -1) - nonopt_start = pure_optind; - else if (nonopt_end != -1) { - pure_permute_args(nonopt_start, nonopt_end, - pure_optind, nargv); - nonopt_start = pure_optind - - (nonopt_end - nonopt_start); - nonopt_end = -1; - } - pure_optind++; - /* process next argument */ - goto start; - } - if (nonopt_start != -1 && nonopt_end == -1) - nonopt_end = pure_optind; - - /* - * Check for "--" or "--foo" with no long options - * but if pure_place is simply "-" leave it unmolested. - */ - - if (pure_place[1] != '\0' && *++pure_place == '-' && - (pure_place[1] == '\0' || long_options == NULL)) { - pure_optind++; - pure_place = EMSG; - /* - * We found an option (--), so if we skipped - * non-options, we have to permute. - */ - if (nonopt_end != -1) { - pure_permute_args(nonopt_start, nonopt_end, - pure_optind, nargv); - pure_optind -= nonopt_end - nonopt_start; - } - nonopt_start = nonopt_end = -1; - return -1; - } - } - - /* - * Check long options if: - * 1) we were passed some - * 2) the arg is not just "-" - * 3) either the arg starts with -- we are pure_getopt_long_only() - */ - if (long_options != NULL && pure_place != nargv[pure_optind] && - (*pure_place == '-' || (flags & FLAG_LONGONLY))) { - short_too = 0; - if (*pure_place == '-') - pure_place++; /* --foo long option */ - else if (*pure_place != ':' && strchr(options, *pure_place) != NULL) - short_too = 1; /* could be short option too */ - - optchar = pure_parse_long_options(nargv, options, long_options, - idx, short_too); - if (optchar != -1) { - pure_place = EMSG; - return optchar; - } - } - - if ((optchar = (int) *pure_place++) == ':' || - (optchar == '-' && *pure_place != '\0') || - (oli = strchr(options, optchar)) == NULL) { - /* - * If the user specified "-" and '-' isn't listed in - * options, return -1 (non-option) as per POSIX. - * Otherwise, it is an unknown option character (or :'). - */ - if (optchar == '-' && *pure_place == '\0') - return -1; - if (!*pure_place) - ++pure_optind; - if (PRINT_ERROR) - fprintf(stderr, illoptchar, optchar); - pure_optopt = optchar; - return BADCH; - } - if (long_options != NULL && optchar == 'W' && oli[1] == ';') { - /* -W long-option */ - if (*pure_place) /* no space */ - /* NOTHING */; - else if (++pure_optind >= nargc) { /* no arg */ - pure_place = EMSG; - if (PRINT_ERROR) - fprintf(stderr, recargchar, optchar); - pure_optopt = optchar; - return BADARG; - } else /* white space */ - pure_place = nargv[pure_optind]; - optchar = pure_parse_long_options(nargv, options, long_options, - idx, 0); - pure_place = EMSG; - return optchar; - } - if (*++oli != ':') { /* doesn't take argument */ - if (!*pure_place) - ++pure_optind; - } else { /* takes (optional) argument */ - pure_optarg = NULL; - if (*pure_place) /* no white space */ - pure_optarg = pure_place; - /* XXX: disable test for :: if PC? (GNU doesn't) */ - else if (oli[1] != ':') { /* arg not optional */ - if (++pure_optind >= nargc) { /* no arg */ - pure_place = EMSG; - if (PRINT_ERROR) - fprintf(stderr, recargchar, optchar); - pure_optopt = optchar; - return BADARG; - } else { - pure_optarg = nargv[pure_optind]; - } - } - pure_place = EMSG; - ++pure_optind; - } - /* dump back option letter */ - return optchar; -} - -/* - * getopt -- - * Parse argc/argv argument vector. - */ -int pure_getopt(int nargc, char * const *nargv, const char *options) -{ - - /* - * We dont' pass FLAG_PERMUTE to pure_getopt_internal() since - * the BSD getopt(3) (unlike GNU) has never done this. - * - * Furthermore, since many privileged programs call getopt() - * before dropping privileges it makes sense to keep things - * as simple (and bug-free) as possible. - */ - return pure_getopt_internal(nargc, nargv, options, NULL, NULL, 0); -} - -/* - * pure_getopt_long -- - * Parse argc/argv argument vector. - */ -int pure_getopt_long(int nargc, char * const *nargv, const char *options, - const struct pure_option *long_options, int *idx) -{ - return pure_getopt_internal(nargc, nargv, options, long_options, idx, - FLAG_PERMUTE); -} - -/* - * pure_getopt_long_only -- - * Parse argc/argv argument vector. - */ -int pure_getopt_long_only(int nargc, char * const *nargv, - const char *options, - const struct pure_option *long_options, - int *idx) -{ - return pure_getopt_internal(nargc, nargv, options, long_options, idx, - FLAG_PERMUTE|FLAG_LONGONLY); -} - -#endif diff --git a/src/bsd-getopt-long.h b/src/bsd-getopt-long.h deleted file mode 100644 index 9a1a9cd85a..0000000000 --- a/src/bsd-getopt-long.h +++ /dev/null @@ -1,127 +0,0 @@ -/* $OpenBSD: getopt_long.c,v 1.13 2003/06/03 01:52:40 millert Exp $ */ -/* $NetBSD: getopt_long.c,v 1.15 2002/01/31 22:43:40 tv Exp $ */ - -/* - * Copyright (c) 2002 Todd C. Miller - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND TODD C. MILLER DISCLAIMS ALL - * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL TODD C. MILLER BE LIABLE - * FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION - * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN - * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ -/*- - * Copyright (c) 2000 The NetBSD Foundation, Inc. - * All rights reserved. - * - * This code is derived from software contributed to The NetBSD Foundation - * by Dieter Baron and Thomas Klausner. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the NetBSD - * Foundation, Inc. and its contributors. - * 4. Neither the name of The NetBSD Foundation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#pragma once - -#ifndef HAVE_GETOPT_LONG - -/* - * GNU-like getopt_long() and 4.4BSD getsubopt()/optreset extensions - */ -# ifndef no_argument -# define no_argument 0 -# endif -# ifndef required_argument -# define required_argument 1 -# endif -# ifndef optional_argument -# define optional_argument 2 -# endif - -struct pure_option { - /* name of long option */ - const char *name; - /* - * one of no_argument, required_argument, and optional_argument: - * whether option takes an argument - */ - int has_arg; - /* if not NULL, set *flag to val when option found */ - int *flag; - /* if flag not NULL, value to set *flag to; else return value */ - int val; -}; - -#ifdef __cplusplus -extern "C" { -#endif - -int pure_getopt_long(int nargc, char * const *nargv, const char *options, - const struct pure_option *long_options, int *idx); - -int pure_getopt_long_only(int nargc, char * const *nargv, - const char *options, - const struct pure_option *long_options, - int *idx); - -int pure_getopt(int nargc, char * const *nargv, const char *options); - -#ifdef __cplusplus -} -#endif - -/* prefix+macros just to avoid clashes with existing getopt() implementations */ - -# ifndef IN_GETOPT_LONG_C -# undef option -# define option pure_option -# undef getopt_long -# define getopt_long(A, B, C, D, E) pure_getopt_long(A, B, C, D, E) -# undef getopt_long_only -# define getopt_long_only(A, B, C, D, E) pure_getopt_long_only(A, B, C, D, E) -# undef getopt -# define getopt(A, B, C) pure_getopt(A, B, C) -# undef optarg -# define optarg pure_optarg -# undef opterr -# define opterr pure_opterr -# undef optind -# define optind pure_optind -# undef optopt -# define optopt pure_optopt -# undef optreset -# define optreset pure_optreset -# endif - -#endif diff --git a/src/in_cksum.cc b/src/in_cksum.cc deleted file mode 100644 index 40f00d9134..0000000000 --- a/src/in_cksum.cc +++ /dev/null @@ -1,141 +0,0 @@ -// Modified from tcpdump v4.9.3's in_cksum.c (which itself was a modified -// version of FreeBSD's in_cksum.c). - -/* in_cksum.c - * 4.4-Lite-2 Internet checksum routine, modified to take a vector of - * pointers/lengths giving the pieces to be checksummed. Also using - * Tahoe/CGI version of ADDCARRY(x) macro instead of from portable version. - */ - -/* - * Copyright (c) 1988, 1992, 1993 - * The Regents of the University of California. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * @(#)in_cksum.c 8.1 (Berkeley) 6/10/93 - */ - -#include "zeek/net_util.h" - -namespace zeek::detail { - -#define ADDCARRY(x) {if ((x) > 65535) (x) -= 65535;} -#define REDUCE {l_util.l = sum; sum = l_util.s[0] + l_util.s[1]; ADDCARRY(sum);} - -uint16_t in_cksum(const struct checksum_block *vec, int veclen) -{ - const uint16_t *w; - int sum = 0; - int mlen = 0; - int byte_swapped = 0; - - union { - uint8_t c[2]; - uint16_t s; - } s_util; - union { - uint16_t s[2]; - uint32_t l; - } l_util; - - for (; veclen != 0; vec++, veclen--) { - if (vec->len == 0) - continue; - w = (const uint16_t *)(const void *)vec->block; - if (mlen == -1) { - /* - * The first byte of this chunk is the continuation - * of a word spanning between this chunk and the - * last chunk. - * - * s_util.c[0] is already saved when scanning previous - * chunk. - */ - s_util.c[1] = *(const uint8_t *)w; - sum += s_util.s; - w = (const uint16_t *)(const void *)((const uint8_t *)w + 1); - mlen = vec->len - 1; - } else - mlen = vec->len; - /* - * Force to even boundary. - */ - if ((1 & (uintptr_t) w) && (mlen > 0)) { - REDUCE; - sum <<= 8; - s_util.c[0] = *(const uint8_t *)w; - w = (const uint16_t *)(const void *)((const uint8_t *)w + 1); - mlen--; - byte_swapped = 1; - } - /* - * Unroll the loop to make overhead from - * branches &c small. - */ - while ((mlen -= 32) >= 0) { - sum += w[0]; sum += w[1]; sum += w[2]; sum += w[3]; - sum += w[4]; sum += w[5]; sum += w[6]; sum += w[7]; - sum += w[8]; sum += w[9]; sum += w[10]; sum += w[11]; - sum += w[12]; sum += w[13]; sum += w[14]; sum += w[15]; - w += 16; - } - mlen += 32; - while ((mlen -= 8) >= 0) { - sum += w[0]; sum += w[1]; sum += w[2]; sum += w[3]; - w += 4; - } - mlen += 8; - if (mlen == 0 && byte_swapped == 0) - continue; - REDUCE; - while ((mlen -= 2) >= 0) { - sum += *w++; - } - if (byte_swapped) { - REDUCE; - sum <<= 8; - byte_swapped = 0; - if (mlen == -1) { - s_util.c[1] = *(const uint8_t *)w; - sum += s_util.s; - mlen = 0; - } else - mlen = -1; - } else if (mlen == -1) - s_util.c[0] = *(const uint8_t *)w; - } - if (mlen == -1) { - /* The last mbuf has odd # of bytes. Follow the - standard (the odd byte may be shifted left by 8 bits - or not as determined by endian-ness of the machine) */ - s_util.c[1] = 0; - sum += s_util.s; - } - REDUCE; - return sum; -} - -} // namespace zeek diff --git a/src/input/readers/raw/Raw.cc b/src/input/readers/raw/Raw.cc index 463f4d6a2d..a8ba5df1c8 100644 --- a/src/input/readers/raw/Raw.cc +++ b/src/input/readers/raw/Raw.cc @@ -18,7 +18,7 @@ extern "C" { -#include "zeek/setsignal.h" +#include "zeek/3rdparty/setsignal.h" } using zeek::threading::Field; diff --git a/src/modp_numtoa.c b/src/modp_numtoa.c deleted file mode 100644 index f6b7059d8a..0000000000 --- a/src/modp_numtoa.c +++ /dev/null @@ -1,494 +0,0 @@ -/* -*- mode: c; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 4 -*- */ -/* vi: set expandtab shiftwidth=4 tabstop=4: */ - -#include "zeek/modp_numtoa.h" - -#include -#include -#include -#include -#include - -// other interesting references on num to string convesion -// http://www.jb.man.ac.uk/~slowe/cpp/itoa.html -// and http://www.ddj.com/dept/cpp/184401596?pgno=6 - -// Version 19-Nov-2007 -// Fixed round-to-even rules to match printf -// thanks to Johannes Otepka - -/** - * Powers of 10 - * 10^0 to 10^9 - */ -static const double _pow10[] = {1, 10, 100, 1000, 10000, 100000, 1000000, - 10000000, 100000000, 1000000000}; -static const double _pow10r[] = {1, .1, .01, .001, .0001, .00001, .000001, - .0000001, .00000001, .000000001}; - -static void strreverse(char* begin, char* end) -{ - char aux; - while (end > begin) - aux = *end, *end-- = *begin, *begin++ = aux; -} - -// Expects 'str' to have been made using "%e" scientific notation format string -static void sn_strip_trailing_zeros(char* str) - { - char* frac = 0; - - for ( ; ; ) - { - if ( *str == '.' ) - { - frac = str + 1; - break; - } - - if ( *str == 0 ) - break; - - ++str; - } - - if ( ! frac ) - return; - - char* start_dec = frac; - char* exp = 0; - char* trailing_zeros = 0; - - for ( ; ; ) - { - if ( *frac == 0 ) - break; - - if ( *frac == 'e' ) - { - exp = frac; - break; - } - - if ( *frac == '0' ) - { - if ( ! trailing_zeros ) - trailing_zeros = frac; - } - else - trailing_zeros = 0; - - ++frac; - } - - if ( trailing_zeros == start_dec ) - --trailing_zeros; - - if ( trailing_zeros && exp ) - { - for ( ; ; ) - { - *trailing_zeros = *exp; - - if ( *exp == 0 ) - break; - - ++trailing_zeros; - ++exp; - } - } - } - -void modp_itoa10(int32_t value, char* str) -{ - char* wstr=str; - // Take care of sign - unsigned int uvalue = (value < 0) ? -value : value; - // Conversion. Number is reversed. - do *wstr++ = (char)(48 + (uvalue % 10)); while(uvalue /= 10); - if (value < 0) *wstr++ = '-'; - *wstr='\0'; - - // Reverse string - strreverse(str,wstr-1); -} - -void modp_uitoa10(uint32_t value, char* str) -{ - char* wstr=str; - // Conversion. Number is reversed. - do *wstr++ = (char)(48 + (value % 10)); while (value /= 10); - *wstr='\0'; - // Reverse string - strreverse(str, wstr-1); -} - -void modp_litoa10(int64_t value, char* str) -{ - char* wstr=str; - uint64_t uvalue = (value < 0) ? (value == INT64_MIN ? (uint64_t)(INT64_MAX) + 1 : -value) : value; - - // Conversion. Number is reversed. - do *wstr++ = (char)(48 + (uvalue % 10)); while(uvalue /= 10); - if (value < 0) *wstr++ = '-'; - *wstr='\0'; - - // Reverse string - strreverse(str,wstr-1); -} - -void modp_ulitoa10(uint64_t value, char* str) -{ - char* wstr=str; - // Conversion. Number is reversed. - do *wstr++ = (char)(48 + (value % 10)); while (value /= 10); - *wstr='\0'; - // Reverse string - strreverse(str, wstr-1); -} - -void modp_dtoa(double value, char* str, int prec) -{ - /* Hacky test for NaN - * under -fast-math this won't work, but then you also won't - * have correct nan values anyways. The alternative is - * to link with libmath (bad) or hack IEEE double bits (bad) - */ - if (! (value == value)) { - str[0] = 'n'; str[1] = 'a'; str[2] = 'n'; str[3] = '\0'; - return; - } - - /* we'll work in positive values and deal with the - negative sign issue later */ - int neg = 0; - if (value < 0) { - neg = 1; - value = -value; - } - - /* if input is larger than thres_max, revert to exponential */ - const double thres_max = (double)(INT_MAX); - - /* for very large numbers switch back to native sprintf for exponentials. - anyone want to write code to replace this? */ - /* - normal printf behavior is to print EVERY whole number digit - which can be 100s of characters overflowing your buffers == bad - */ - if (value >= thres_max) { - sprintf(str, "%.*e", DBL_DECIMAL_DIG - 1, neg ? -value : value); - sn_strip_trailing_zeros(str); - return; - } - - double diff = 0.0; - char* wstr = str; - - if (prec < 0) { - prec = 0; - } else if (prec > 9) { - /* precision of >= 10 can lead to overflow errors */ - prec = 9; - } - - int whole = (int) value; - double tmp = (value - whole) * _pow10[prec]; - uint32_t frac = (uint32_t)(tmp); - diff = tmp - frac; - - if (diff > 0.5) { - ++frac; - /* handle rollover, e.g. case 0.99 with prec 1 is 1.0 */ - if (frac >= _pow10[prec]) { - frac = 0; - ++whole; - } - } else if (diff == 0.5 && ((frac == 0) || (frac & 1))) { - /* if halfway, round up if odd, OR - if last digit is 0. That last part is strange */ - ++frac; - } - - if (prec == 0) { - diff = value - whole; - if (diff > 0.5) { - /* greater than 0.5, round up, e.g. 1.6 -> 2 */ - ++whole; - } else if (diff == 0.5 && (whole & 1)) { - /* exactly 0.5 and ODD, then round up */ - /* 1.5 -> 2, but 2.5 -> 2 */ - ++whole; - } - } else { - int count = prec; - // now do fractional part, as an unsigned number - do { - --count; - *wstr++ = (char)(48 + (frac % 10)); - } while (frac /= 10); - // add extra 0s - while (count-- > 0) *wstr++ = '0'; - // add decimal - *wstr++ = '.'; - } - - // do whole part - // Take care of sign - // Conversion. Number is reversed. - do *wstr++ = (char)(48 + (whole % 10)); while (whole /= 10); - if (neg) { - *wstr++ = '-'; - } - *wstr='\0'; - strreverse(str, wstr-1); -} - - -// This is near identical to modp_dtoa above -// The differnce is noted below -void modp_dtoa2(double value, char* str, int prec) -{ - /* Hacky test for NaN - * under -fast-math this won't work, but then you also won't - * have correct nan values anyways. The alternative is - * to link with libmath (bad) or hack IEEE double bits (bad) - */ - if (! (value == value)) { - str[0] = 'n'; str[1] = 'a'; str[2] = 'n'; str[3] = '\0'; - return; - } - - /* we'll work in positive values and deal with the - negative sign issue later */ - int neg = 0; - if (value < 0) { - neg = 1; - value = -value; - } - - /* if input is larger than thres_max, revert to exponential */ - const double thres_max = (double)(INT_MAX); - - /* for very large numbers switch back to native sprintf for exponentials. - anyone want to write code to replace this? */ - /* - normal printf behavior is to print EVERY whole number digit - which can be 100s of characters overflowing your buffers == bad - */ - if (value >= thres_max) { - sprintf(str, "%.*e", DBL_DECIMAL_DIG - 1, neg ? -value : value); - sn_strip_trailing_zeros(str); - return; - } - - int count; - double diff = 0.0; - char* wstr = str; - - if (prec < 0) { - prec = 0; - } else if (prec > 9) { - /* precision of >= 10 can lead to overflow errors */ - prec = 9; - } - - double smallest = _pow10r[prec]; - - if (value != 0.0 && value < smallest) { - sprintf(str, "%.*e", DBL_DECIMAL_DIG - 1, neg ? -value : value); - sn_strip_trailing_zeros(str); - return; - } - - int whole = (int) value; - double tmp = (value - whole) * _pow10[prec]; - uint32_t frac = (uint32_t)(tmp); - diff = tmp - frac; - - if (diff > 0.5) { - ++frac; - /* handle rollover, e.g. case 0.99 with prec 1 is 1.0 */ - if (frac >= _pow10[prec]) { - frac = 0; - ++whole; - } - } else if (diff == 0.5 && ((frac == 0) || (frac & 1))) { - /* if halfway, round up if odd, OR - if last digit is 0. That last part is strange */ - ++frac; - } - - if (prec == 0) { - diff = value - whole; - if (diff > 0.5) { - /* greater than 0.5, round up, e.g. 1.6 -> 2 */ - ++whole; - } else if (diff == 0.5 && (whole & 1)) { - /* exactly 0.5 and ODD, then round up */ - /* 1.5 -> 2, but 2.5 -> 2 */ - ++whole; - } - - //vvvvvvvvvvvvvvvvvvv Diff from modp_dto2 - } else if (frac) { - count = prec; - // now do fractional part, as an unsigned number - // we know it is not 0 but we can have leading zeros, these - // should be removed - while (!(frac % 10)) { - --count; - frac /= 10; - } - //^^^^^^^^^^^^^^^^^^^ Diff from modp_dto2 - - // now do fractional part, as an unsigned number - do { - --count; - *wstr++ = (char)(48 + (frac % 10)); - } while (frac /= 10); - // add extra 0s - while (count-- > 0) *wstr++ = '0'; - // add decimal - *wstr++ = '.'; - } - - // do whole part - // Take care of sign - // Conversion. Number is reversed. - do *wstr++ = (char)(48 + (whole % 10)); while (whole /= 10); - if (neg) { - *wstr++ = '-'; - } - *wstr='\0'; - strreverse(str, wstr-1); -} - -// This is near identical to modp_dtoa2 above, excep that it never uses -// exponential notation and requires a buffer length. -void modp_dtoa3(double value, char* str, int n, int prec) -{ - /* Hacky test for NaN - * under -fast-math this won't work, but then you also won't - * have correct nan values anyways. The alternative is - * to link with libmath (bad) or hack IEEE double bits (bad) - */ - if (! (value == value)) { - str[0] = 'n'; str[1] = 'a'; str[2] = 'n'; str[3] = '\0'; - return; - } - - /* we'll work in positive values and deal with the - negative sign issue later */ - int neg = 0; - if (value < 0) { - neg = 1; - value = -value; - } - - if (prec < 0) { - prec = 0; - } else if (prec > 9) { - /* precision of >= 10 can lead to overflow errors */ - prec = 9; - } - - /* if input is larger than thres_max, revert to exponential */ - const double thres_max = (double)(INT_MAX); - - /* for very large numbers switch back to native sprintf for exponentials. - anyone want to write code to replace this? */ - /* - normal printf behavior is to print EVERY whole number digit - which can be 100s of characters overflowing your buffers == bad - */ - if (value >= thres_max) { - /* ---- Modified part, compared to modp_dtoa3. */ - int i = snprintf(str, n, "%.*f", prec, neg ? -value : value); - - if ( i < 0 || i >= n ) { - // Error or truncated output. - snprintf(str, n, "NAN"); - return; - } - - /* Remove trailing zeros. */ - - char* p; - for ( p = str + i - 1; p >= str && *p == '0'; --p ); - - if ( p >= str && *p == '.' ) - --p; - - *++p = '\0'; - return; - - /* ---- End of modified part.. */ - } - - int count; - double diff = 0.0; - char* wstr = str; - - int whole = (int) value; - double tmp = (value - whole) * _pow10[prec]; - uint32_t frac = (uint32_t)(tmp); - diff = tmp - frac; - - if (diff > 0.5) { - ++frac; - /* handle rollover, e.g. case 0.99 with prec 1 is 1.0 */ - if (frac >= _pow10[prec]) { - frac = 0; - ++whole; - } - } else if (diff == 0.5 && ((frac == 0) || (frac & 1))) { - /* if halfway, round up if odd, OR - if last digit is 0. That last part is strange */ - ++frac; - } - - if (prec == 0) { - diff = value - whole; - if (diff > 0.5) { - /* greater than 0.5, round up, e.g. 1.6 -> 2 */ - ++whole; - } else if (diff == 0.5 && (whole & 1)) { - /* exactly 0.5 and ODD, then round up */ - /* 1.5 -> 2, but 2.5 -> 2 */ - ++whole; - } - - //vvvvvvvvvvvvvvvvvvv Diff from modp_dto2 - } else if (frac) { - count = prec; - // now do fractional part, as an unsigned number - // we know it is not 0 but we can have leading zeros, these - // should be removed - while (!(frac % 10)) { - --count; - frac /= 10; - } - //^^^^^^^^^^^^^^^^^^^ Diff from modp_dto2 - - // now do fractional part, as an unsigned number - do { - --count; - *wstr++ = (char)(48 + (frac % 10)); - } while (frac /= 10); - // add extra 0s - while (count-- > 0) *wstr++ = '0'; - // add decimal - *wstr++ = '.'; - } - - // do whole part - // Take care of sign - // Conversion. Number is reversed. - do *wstr++ = (char)(48 + (whole % 10)); while (whole /= 10); - if (neg) { - *wstr++ = '-'; - } - *wstr='\0'; - strreverse(str, wstr-1); -} diff --git a/src/modp_numtoa.h b/src/modp_numtoa.h deleted file mode 100644 index c8c46827c8..0000000000 --- a/src/modp_numtoa.h +++ /dev/null @@ -1,114 +0,0 @@ -/* -*- mode: c++; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 4 -*- */ -/* vi: set expandtab shiftwidth=4 tabstop=4: */ - -/** - * \file - * - *
- * Copyright © 2007, Nick Galbreath -- nickg [at] modp [dot] com
- * All rights reserved.
- * http://code.google.com/p/stringencoders/
- * Released under the bsd license.
- * 
- * - * This defines signed/unsigned integer, and 'double' to char buffer - * converters. The standard way of doing this is with "sprintf", however - * these functions are - * * guarenteed maximum size output - * * 5-20x faster! - * * core-dump safe - * - * - */ - -#pragma once - -#ifdef __cplusplus -#define BEGIN_C extern "C" { -#define END_C } -#else -#define BEGIN_C -#define END_C -#endif - -BEGIN_C - -#include - -/** \brief convert an signed integer to char buffer - * - * \param[in] value - * \param[out] buf the output buffer. Should be 16 chars or more. - */ -void modp_itoa10(int32_t value, char* buf); - -/** \brief convert an unsigned integer to char buffer - * - * \param[in] value - * \param[out] buf The output buffer, should be 16 chars or more. - */ -void modp_uitoa10(uint32_t value, char* buf); - -/** \brief convert an signed long integer to char buffer - * - * \param[in] value - * \param[out] buf the output buffer. Should be 24 chars or more. - */ -void modp_litoa10(int64_t value, char* buf); - -/** \brief convert an unsigned long integer to char buffer - * - * \param[in] value - * \param[out] buf The output buffer, should be 24 chars or more. - */ -void modp_ulitoa10(uint64_t value, char* buf); - -/** \brief convert a floating point number to char buffer with - * fixed-precision format - * - * This is similar to "%.[0-9]f" in the printf style. It will include - * trailing zeros - * - * If the input value is greater than 1<<31, then the output format - * will be switched exponential format and include as many precision digits - * as needed to preserve information. - * - * \param[in] value - * \param[out] buf The allocated output buffer. Should be 32 chars or more. - * \param[in] precision Number of digits to the right of the decimal point. - * Can only be 0-9. - */ -void modp_dtoa(double value, char* buf, int precision); - -/** \brief convert a floating point number to char buffer with a - * variable-precision format, and no trailing zeros - * - * This is similar to "%.[0-9]f" in the printf style, except it will - * NOT include trailing zeros after the decimal point. This type - * of format oddly does not exists with printf. - * - * If the input value is greater than 1<<31, then the output format - * will be switched exponential format and include as many precision digits - * as needed to preserve information. - * - * If a non-zero input value is less than 10^(-precision), the output format - * will be switched exponential format and include as many precision digits - * as needed to preserve information. - * - * \param[in] value - * \param[out] buf The allocated output buffer. Should be 32 chars or more. - * \param[in] precision Number of digits to the right of the decimal point. - * Can only be 0-9. - */ -void modp_dtoa2(double value, char* buf, int precision); - -/** \brief convert a floating point number to char buffer with a - * variable-precision format, no trailing zeros, and no - * scientific notation. - * - * Other than avoiding scientific notation, this is the same as mop_dtoa2. It does however - * require the max buffer length. The buffer will always be null-terminated. - */ -void modp_dtoa3(double value, char* buf, int n, int precision); - -END_C diff --git a/src/nb_dns.c b/src/nb_dns.c deleted file mode 100644 index d02ed35be2..0000000000 --- a/src/nb_dns.c +++ /dev/null @@ -1,794 +0,0 @@ -/* - * See the file "COPYING" in the main distribution directory for copyright. - */ -/* - * nb_dns - non-blocking dns routines - * - * This version works with BIND 9 - * - * Note: The code here is way more complicated than it should be but - * although the interface to send requests is public, the routine to - * crack reply buffers is private. - */ - -#include "zeek/zeek-config.h" /* must appear before first ifdef */ -#include "zeek/nb_dns.h" - -#include -#include - -#include - -#include -#include -#ifdef NEED_NAMESER_COMPAT_H -#include -#endif - -#include -#ifdef HAVE_MEMORY_H -#include -#endif -#include -#include -#include -#include -#include -#include - -#ifdef notdef -#include "gnuc.h" -#ifdef HAVE_OS_PROTO_H -#include "os-proto.h" -#endif -#endif - -#if PACKETSZ > 1024 -#define MAXPACKET PACKETSZ -#else -#define MAXPACKET 1024 -#endif - -#ifdef DO_SOCK_DECL -extern int socket(int, int, int); -extern int connect(int, const struct sockaddr *, int); -extern int send(int, const void *, int, int); -extern int recvfrom(int, void *, int, int, struct sockaddr *, int *); -#endif - -/* Private data */ -struct nb_dns_entry { - struct nb_dns_entry *next; - char name[NS_MAXDNAME + 1]; - u_short id; - int qtype; /* query type */ - int atype; /* address family */ - int asize; /* address size */ - void *cookie; -}; - -#ifndef MAXALIASES -#define MAXALIASES 35 -#endif -#ifndef MAXADDRS -#define MAXADDRS 35 -#endif - -struct nb_dns_hostent { - struct hostent hostent; - int numaliases; - int numaddrs; - char *host_aliases[MAXALIASES + 1]; - char *h_addr_ptrs[MAXADDRS + 1]; - char hostbuf[8 * 1024]; -}; - -struct nb_dns_info { - int s; /* Resolver file descriptor */ - struct sockaddr_storage server; /* server address to bind to */ - struct nb_dns_entry *list; /* outstanding requests */ - struct nb_dns_hostent dns_hostent; -}; - -/* Forwards */ -static int _nb_dns_mkquery(struct nb_dns_info *, const char *, int, int, - void *, char *); -static int _nb_dns_cmpsockaddr(struct sockaddr *, struct sockaddr *, char *); - -static char * -my_strerror(int errnum) -{ -#ifdef HAVE_STRERROR - extern char *strerror(int); - return strerror(errnum); -#else - static char errnum_buf[32]; - snprintf(errnum_buf, sizeof(errnum_buf), "errno %d", errnum); - return errnum_buf; -#endif -} - -static const char* sa_ntop(struct sockaddr* sa, char* buf, int len) - { - if ( sa->sa_family == AF_INET ) - return inet_ntop(sa->sa_family, - &(((struct sockaddr_in*)sa)->sin_addr), - buf, len); - else - return inet_ntop(sa->sa_family, - &(((struct sockaddr_in6*)sa)->sin6_addr), - buf, len); - } - -struct nb_dns_info * -nb_dns_init(char *errstr) -{ - register struct nb_dns_info *nd; - - nd = (struct nb_dns_info *)malloc(sizeof(*nd)); - if (nd == NULL) { - snprintf(errstr, NB_DNS_ERRSIZE, "nb_dns_init: malloc(): %s", - my_strerror(errno)); - return (NULL); - } - memset(nd, 0, sizeof(*nd)); - nd->s = -1; - - /* XXX should be able to init static hostent struct some other way */ - (void)gethostbyname("localhost"); - - if ((_res.options & RES_INIT) == 0 && res_init() == -1) { - snprintf(errstr, NB_DNS_ERRSIZE, "res_init() failed"); - free(nd); - return (NULL); - } - - if ( _res.nscount == 0 ) - { - // Really? Let's try parsing resolv.conf ourselves to see what's - // there. (e.g. musl libc has res_init() that doesn't actually - // parse the config file). - const char* config_file_path = "/etc/resolv.conf"; - -#ifdef _PATH_RESCONF - config_file_path = _PATH_RESCONF; -#endif - - FILE* config_file = fopen(config_file_path, "r"); - - if ( config_file ) - { - char line[128]; - char* ns; - - while ( fgets(line, sizeof(line), config_file) ) - { - ns = strtok(line, " \t\n"); - - if ( ! ns || strcmp(ns, "nameserver") ) - continue; - - ns = strtok(0, " \t\n"); - - if ( ! ns ) - continue; - - /* XXX support IPv6 */ - struct sockaddr_in a; - memset(&a, 0, sizeof(a)); - a.sin_family = AF_INET; - a.sin_port = htons(53); - - if ( inet_pton(AF_INET, ns, &a.sin_addr) == 1 ) - { - memcpy(&nd->server, &a, sizeof(a)); - nd->s = socket(nd->server.ss_family, SOCK_DGRAM, 0); - - if ( nd->s < 0 ) - { - snprintf(errstr, NB_DNS_ERRSIZE, "socket(): %s", - my_strerror(errno)); - fclose(config_file); - free(nd); - return (NULL); - } - - if ( connect(nd->s, (struct sockaddr *)&nd->server, - nd->server.ss_family == AF_INET ? - sizeof(struct sockaddr_in) : - sizeof(struct sockaddr_in6)) < 0 ) - { - char s[INET6_ADDRSTRLEN]; - sa_ntop((struct sockaddr*)&nd->server, s, INET6_ADDRSTRLEN); - snprintf(errstr, NB_DNS_ERRSIZE, "connect(%s): %s", s, - my_strerror(errno)); - fclose(config_file); - close(nd->s); - free(nd); - return (NULL); - } - - fclose(config_file); - return (nd); - } - } - - fclose(config_file); - snprintf(errstr, NB_DNS_ERRSIZE, "no valid nameserver found in %s", - config_file_path); - free(nd); - return (NULL); - } - - snprintf(errstr, NB_DNS_ERRSIZE, "resolver config file not located"); - free(nd); - return (NULL); - } - - int i; - - for ( i = 0; i < _res.nscount; ++i ) - { - memcpy(&nd->server, &_res.nsaddr_list[i], sizeof(struct sockaddr_in)); - /* XXX support IPv6 */ - if ( nd->server.ss_family != AF_INET ) - continue; - - nd->s = socket(nd->server.ss_family, SOCK_DGRAM, 0); - - if ( nd->s < 0 ) - { - snprintf(errstr, NB_DNS_ERRSIZE, "socket(): %s", - my_strerror(errno)); - free(nd); - return (NULL); - } - - if ( connect(nd->s, (struct sockaddr *)&nd->server, - nd->server.ss_family == AF_INET ? - sizeof(struct sockaddr_in) : - sizeof(struct sockaddr_in6)) < 0 ) - { - char s[INET6_ADDRSTRLEN]; - sa_ntop((struct sockaddr*)&nd->server, s, INET6_ADDRSTRLEN); - snprintf(errstr, NB_DNS_ERRSIZE, "connect(%s): %s", s, - my_strerror(errno)); - close(nd->s); - free(nd); - return (NULL); - } - - return (nd); - } - - snprintf(errstr, NB_DNS_ERRSIZE, "no valid nameservers in resolver config"); - free(nd); - return (NULL); -} - -struct nb_dns_info * -nb_dns_init2(char *errstr, struct sockaddr* sa) -{ - register struct nb_dns_info *nd; - - nd = (struct nb_dns_info *)malloc(sizeof(*nd)); - if (nd == NULL) { - snprintf(errstr, NB_DNS_ERRSIZE, "nb_dns_init: malloc(): %s", - my_strerror(errno)); - return (NULL); - } - memset(nd, 0, sizeof(*nd)); - nd->s = -1; - - if ( sa->sa_family == AF_INET ) - { - memcpy(&nd->server, sa, sizeof(struct sockaddr_in)); - ((struct sockaddr_in*)&nd->server)->sin_port = htons(53); - } - else - { - memcpy(&nd->server, sa, sizeof(struct sockaddr_in6)); - ((struct sockaddr_in6*)&nd->server)->sin6_port = htons(53); - } - - nd->s = socket(nd->server.ss_family, SOCK_DGRAM, 0); - - if ( nd->s < 0 ) - { - snprintf(errstr, NB_DNS_ERRSIZE, "socket(): %s", - my_strerror(errno)); - free(nd); - return (NULL); - } - - if ( connect(nd->s, (struct sockaddr *)&nd->server, - nd->server.ss_family == AF_INET ? - sizeof(struct sockaddr_in) : - sizeof(struct sockaddr_in6)) < 0 ) - { - char s[INET6_ADDRSTRLEN]; - sa_ntop((struct sockaddr*)&nd->server, s, INET6_ADDRSTRLEN); - snprintf(errstr, NB_DNS_ERRSIZE, "connect(%s): %s", s, - my_strerror(errno)); - close(nd->s); - free(nd); - return (NULL); - } - - return (nd); -} - -void -nb_dns_finish(struct nb_dns_info *nd) -{ - register struct nb_dns_entry *ne, *ne2; - - ne = nd->list; - while (ne != NULL) { - ne2 = ne; - ne = ne->next; - free(ne2); - } - close(nd->s); - free(nd); -} - -int -nb_dns_fd(struct nb_dns_info *nd) -{ - - return (nd->s); -} - -static int -_nb_dns_cmpsockaddr(register struct sockaddr *sa1, - register struct sockaddr *sa2, register char *errstr) -{ - register struct sockaddr_in *sin1, *sin2; -#ifdef AF_INET6 - register struct sockaddr_in6 *sin6a, *sin6b; -#endif - static const char serr[] = "answer from wrong nameserver (%d)"; - - if (sa1->sa_family != sa2->sa_family) { - snprintf(errstr, NB_DNS_ERRSIZE, serr, 1); - return (-1); - } - switch (sa1->sa_family) { - - case AF_INET: - sin1 = (struct sockaddr_in *)sa1; - sin2 = (struct sockaddr_in *)sa2; - if (sin1->sin_port != sin2->sin_port) { - snprintf(errstr, NB_DNS_ERRSIZE, serr, 2); - return (-1); - } - if (sin1->sin_addr.s_addr != sin2->sin_addr.s_addr) { - snprintf(errstr, NB_DNS_ERRSIZE, serr, 3); - return (-1); - } - break; - -#ifdef AF_INET6 - case AF_INET6: - sin6a = (struct sockaddr_in6 *)sa1; - sin6b = (struct sockaddr_in6 *)sa2; - if (sin6a->sin6_port != sin6b->sin6_port) { - snprintf(errstr, NB_DNS_ERRSIZE, serr, 62); - return (-1); - } - if (memcmp(&sin6a->sin6_addr, &sin6b->sin6_addr, - sizeof(sin6a->sin6_addr)) != 0) { - snprintf(errstr, NB_DNS_ERRSIZE, serr, 63); - return (-1); - } - break; -#endif - - default: - snprintf(errstr, NB_DNS_ERRSIZE, serr, 4); - return (-1); - } - return (0); -} - -static int -_nb_dns_mkquery(register struct nb_dns_info *nd, register const char *name, - register int atype, register int qtype, register void * cookie, - register char *errstr) -{ - register struct nb_dns_entry *ne; - register HEADER *hp; - register int n; - u_long msg[MAXPACKET / sizeof(u_long)]; - - /* Allocate an entry */ - ne = (struct nb_dns_entry *)malloc(sizeof(*ne)); - if (ne == NULL) { - snprintf(errstr, NB_DNS_ERRSIZE, "malloc(): %s", - my_strerror(errno)); - return (-1); - } - memset(ne, 0, sizeof(*ne)); - strncpy(ne->name, name, sizeof(ne->name) - 1); - ne->name[sizeof(ne->name) - 1] = '\0'; - ne->qtype = qtype; - ne->atype = atype; - switch (atype) { - - case AF_INET: - ne->asize = NS_INADDRSZ; - break; - -#ifdef AF_INET6 - case AF_INET6: - ne->asize = NS_IN6ADDRSZ; - break; -#endif - - default: - snprintf(errstr, NB_DNS_ERRSIZE, - "_nb_dns_mkquery: bad family %d", atype); - free(ne); - return (-1); - } - - /* Build the request */ - n = res_mkquery( - ns_o_query, /* op code (query) */ - name, /* domain name */ - ns_c_in, /* query class (internet) */ - qtype, /* query type */ - NULL, /* data */ - 0, /* length of data */ - NULL, /* new rr */ - (u_char *)msg, /* buffer */ - sizeof(msg)); /* size of buffer */ - if (n < 0) { - snprintf(errstr, NB_DNS_ERRSIZE, "res_mkquery() failed"); - free(ne); - return (-1); - } - - hp = (HEADER *)msg; - ne->id = htons(hp->id); - - if (send(nd->s, (char *)msg, n, 0) != n) { - snprintf(errstr, NB_DNS_ERRSIZE, "send(): %s", - my_strerror(errno)); - free(ne); - return (-1); - } - - ne->next = nd->list; - ne->cookie = cookie; - nd->list = ne; - - return(0); -} - -int -nb_dns_host_request(register struct nb_dns_info *nd, register const char *name, - register void *cookie, register char *errstr) -{ - - return (nb_dns_host_request2(nd, name, AF_INET, 0, cookie, errstr)); -} - -int -nb_dns_host_request2(register struct nb_dns_info *nd, register const char *name, - register int af, register int qtype, register void *cookie, register char *errstr) -{ - if (qtype != 16) { - - switch (af) { - - case AF_INET: - qtype = T_A; - break; - -#ifdef AF_INET6 - case AF_INET6: - qtype = T_AAAA; - break; -#endif - - default: - snprintf(errstr, NB_DNS_ERRSIZE, - "nb_dns_host_request2(): unknown address family %d", af); - return (-1); - } - } - return (_nb_dns_mkquery(nd, name, af, qtype, cookie, errstr)); -} - -int -nb_dns_addr_request(register struct nb_dns_info *nd, nb_uint32_t addr, - register void *cookie, register char *errstr) -{ - - return (nb_dns_addr_request2(nd, (char *)&addr, AF_INET, - cookie, errstr)); -} - -int -nb_dns_addr_request2(register struct nb_dns_info *nd, char *addrp, - register int af, register void *cookie, register char *errstr) -{ -#ifdef AF_INET6 - register char *cp; - register int n, i; - register size_t size; -#endif - register u_char *uaddr; - char name[NS_MAXDNAME + 1]; - - switch (af) { - - case AF_INET: - uaddr = (u_char *)addrp; - snprintf(name, sizeof(name), "%u.%u.%u.%u.in-addr.arpa", - (uaddr[3] & 0xff), - (uaddr[2] & 0xff), - (uaddr[1] & 0xff), - (uaddr[0] & 0xff)); - break; - -#ifdef AF_INET6 - case AF_INET6: - uaddr = (u_char *)addrp; - cp = name; - size = sizeof(name); - for (n = NS_IN6ADDRSZ - 1; n >= 0; --n) { - snprintf(cp, size, "%x.%x.", - (uaddr[n] & 0xf), - (uaddr[n] >> 4) & 0xf); - i = strlen(cp); - size -= i; - cp += i; - } - snprintf(cp, size, "ip6.arpa"); - break; -#endif - - default: - snprintf(errstr, NB_DNS_ERRSIZE, - "nb_dns_addr_request2(): unknown address family %d", af); - return (-1); - } - - return (_nb_dns_mkquery(nd, name, af, T_PTR, cookie, errstr)); -} - -int -nb_dns_abort_request(struct nb_dns_info *nd, void *cookie) -{ - register struct nb_dns_entry *ne, *lastne; - - /* Try to find this request on the outstanding request list */ - lastne = NULL; - for (ne = nd->list; ne != NULL; ne = ne->next) { - if (ne->cookie == cookie) - break; - lastne = ne; - } - - /* Not a currently pending request */ - if (ne == NULL) - return (-1); - - /* Unlink this entry */ - if (lastne == NULL) - nd->list = ne->next; - else - lastne->next = ne->next; - ne->next = NULL; - - return (0); -} - -/* Returns 1 with an answer, 0 when reply was old, -1 on fatal errors */ -int -nb_dns_activity(struct nb_dns_info *nd, struct nb_dns_result *nr, char *errstr) -{ - register int msglen, qtype, atype, n, i; - register struct nb_dns_entry *ne, *lastne; - socklen_t fromlen; - struct sockaddr_storage from; - u_long msg[MAXPACKET / sizeof(u_long)]; - register char *bp, *ep; - register char **ap, **hap; - register u_int16_t id; - register const u_char *rdata; - register u_int32_t rttl = 0; // make compiler happy. - register struct hostent *he; - register size_t rdlen; - ns_msg handle; - ns_rr rr; - - /* This comes from the second half of do_query() */ - fromlen = sizeof(from); - msglen = recvfrom(nd->s, (char *)msg, sizeof(msg), 0, - (struct sockaddr*)&from, &fromlen); - if (msglen <= 0) { - snprintf(errstr, NB_DNS_ERRSIZE, "recvfrom(): %s", - my_strerror(errno)); - return (-1); - } - if (msglen < HFIXEDSZ) { - snprintf(errstr, NB_DNS_ERRSIZE, "recvfrom(): undersized: %d", - msglen); - return (-1); - } - if (ns_initparse((u_char *)msg, msglen, &handle) < 0) { - snprintf(errstr, NB_DNS_ERRSIZE, "ns_initparse(): %s", - my_strerror(errno)); - nr->host_errno = NO_RECOVERY; - return (-1); - } - - /* RES_INSECURE1 style check */ - if (_nb_dns_cmpsockaddr((struct sockaddr*)&nd->server, - (struct sockaddr*)&from, errstr) < 0) { - nr->host_errno = NO_RECOVERY; - return (-1); - } - - /* Search for this request */ - lastne = NULL; - id = ns_msg_id(handle); - for (ne = nd->list; ne != NULL; ne = ne->next) { - if (ne->id == id) - break; - lastne = ne; - } - - /* Not an answer to a question we care about anymore */ - if (ne == NULL) - return (0); - - /* Unlink this entry */ - if (lastne == NULL) - nd->list = ne->next; - else - lastne->next = ne->next; - ne->next = NULL; - - /* RES_INSECURE2 style check */ - /* XXX not implemented */ - - /* Initialize result struct */ - memset(nr, 0, sizeof(*nr)); - nr->cookie = ne->cookie; - qtype = ne->qtype; - - /* Deal with various errors */ - switch (ns_msg_getflag(handle, ns_f_rcode)) { - - case ns_r_nxdomain: - nr->host_errno = HOST_NOT_FOUND; - free(ne); - return (1); - - case ns_r_servfail: - nr->host_errno = TRY_AGAIN; - free(ne); - return (1); - - case ns_r_noerror: - break; - - case ns_r_formerr: - case ns_r_notimpl: - case ns_r_refused: - default: - nr->host_errno = NO_RECOVERY; - free(ne); - return (1); - } - - /* Loop through records in packet */ - memset(&rr, 0, sizeof(rr)); - memset(&nd->dns_hostent, 0, sizeof(nd->dns_hostent)); - he = &nd->dns_hostent.hostent; - /* XXX no support for aliases */ - he->h_aliases = nd->dns_hostent.host_aliases; - he->h_addr_list = nd->dns_hostent.h_addr_ptrs; - he->h_addrtype = ne->atype; - he->h_length = ne->asize; - free(ne); - - bp = nd->dns_hostent.hostbuf; - ep = bp + sizeof(nd->dns_hostent.hostbuf); - hap = he->h_addr_list; - ap = he->h_aliases; - - for (i = 0; i < ns_msg_count(handle, ns_s_an); i++) { - /* Parse next record */ - if (ns_parserr(&handle, ns_s_an, i, &rr) < 0) { - if (errno != ENODEV) { - nr->host_errno = NO_RECOVERY; - return (1); - } - /* All done */ - break; - } - - /* Ignore records that don't answer our query (e.g. CNAMEs) */ - atype = ns_rr_type(rr); - if (atype != qtype) - continue; - - rdata = ns_rr_rdata(rr); - rdlen = ns_rr_rdlen(rr); - rttl = ns_rr_ttl(rr); - switch (atype) { - - case T_A: - case T_AAAA: - if (rdlen != (unsigned int) he->h_length) { - snprintf(errstr, NB_DNS_ERRSIZE, - "nb_dns_activity(): bad rdlen %d", - (int) rdlen); - nr->host_errno = NO_RECOVERY; - return (-1); - } - - if (bp + rdlen >= ep) { - snprintf(errstr, NB_DNS_ERRSIZE, - "nb_dns_activity(): overflow 1"); - nr->host_errno = NO_RECOVERY; - return (-1); - } - if (nd->dns_hostent.numaddrs + 1 >= MAXADDRS) { - snprintf(errstr, NB_DNS_ERRSIZE, - "nb_dns_activity(): overflow 2"); - nr->host_errno = NO_RECOVERY; - return (-1); - } - - memcpy(bp, rdata, rdlen); - *hap++ = bp; - bp += rdlen; - ++nd->dns_hostent.numaddrs; - - /* Keep looking for more A records */ - break; - - case T_TXT: - if (bp + rdlen >= ep) { - snprintf(errstr, NB_DNS_ERRSIZE, - "nb_dns_activity(): overflow 1 for txt"); - nr->host_errno = NO_RECOVERY; - return (-1); - } - - memcpy(bp, rdata, rdlen); - he->h_name = bp+1; /* First char is a control character. */ - nr->hostent = he; - nr->ttl = rttl; - return (1); - - case T_PTR: - n = dn_expand((const u_char *)msg, - (const u_char *)msg + msglen, rdata, bp, ep - bp); - if (n < 0) { - /* XXX return -1 here ??? */ - nr->host_errno = NO_RECOVERY; - return (1); - } - he->h_name = bp; - /* XXX check for overflow */ - bp += n; /* returned len includes EOS */ - - /* "Find first satisfactory answer" */ - nr->hostent = he; - nr->ttl = rttl; - return (1); - } - } - - nr->hostent = he; - nr->ttl = rttl; - return (1); -} diff --git a/src/nb_dns.h b/src/nb_dns.h deleted file mode 100644 index 91ea1d8e94..0000000000 --- a/src/nb_dns.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * See the file "COPYING" in the main distribution directory for copyright. - */ - -#pragma once - -#include -#include - -/* Private data */ -struct nb_dns_info; - -/* Public data */ -struct nb_dns_result { - void *cookie; - int host_errno; - uint32_t ttl; - struct hostent *hostent; -}; - -typedef unsigned int nb_uint32_t; - -/* Public routines */ -struct nb_dns_info *nb_dns_init(char *); -struct nb_dns_info *nb_dns_init2(char *, struct sockaddr*); -void nb_dns_finish(struct nb_dns_info *); - -int nb_dns_fd(struct nb_dns_info *); - -int nb_dns_host_request(struct nb_dns_info *, const char *, void *, char *); -int nb_dns_host_request2(struct nb_dns_info *, const char *, int, int, - void *, char *); - -int nb_dns_addr_request(struct nb_dns_info *, nb_uint32_t, void *, char *); -int nb_dns_addr_request2(struct nb_dns_info *, char *, int, void *, char *); - -int nb_dns_abort_request(struct nb_dns_info *, void *); - -int nb_dns_activity(struct nb_dns_info *, struct nb_dns_result *, char *); - -#define NB_DNS_ERRSIZE 256 diff --git a/src/patricia.c b/src/patricia.c deleted file mode 100644 index 7aa265c0fb..0000000000 --- a/src/patricia.c +++ /dev/null @@ -1,1174 +0,0 @@ -/* - * This code originates from Dave Plonka's Net::Security perl module. An adaptation - * of it in C is kept at https://github.com/CAIDA/cc-common/tree/master/libpatricia. - * That repository is considered the upstream version for Zeek's fork. We make some - * custom changes to this upstream: - * - Replaces void_fn_t with data_fn_t and prefix_data_fn_t - * - Adds patricia_search_all method - * - One commented-out portion of an if statement that breaks one of our tests - * - * The current version is based on commit 4a2c61374f507a420d28bd9084c976142d279605 - * from that repo. - */ - -/* - * Johanna Amann - * - * Added patricia_search_all function. - */ -/* - * Dave Plonka - * - * This product includes software developed by the University of Michigan, - * Merit Network, Inc., and their contributors. - * - * This file had been called "radix.c" in the MRT sources. - * - * I renamed it to "patricia.c" since it's not an implementation of a general - * radix trie. Also I pulled in various requirements from "prefix.c" and - * "demo.c" so that it could be used as a standalone API. - */ - -/* From copyright.txt: - * - * Copyright (c) 1997, 1998, 1999 - * - * - * The Regents of the University of Michigan ("The Regents") and Merit Network, - * Inc. All rights reserved. - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * 1. Redistributions of source code must retain the above - * copyright notice, this list of conditions and the - * following disclaimer. - * 2. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * 3. All advertising materials mentioning features or use of - * this software must display the following acknowledgement: - * This product includes software developed by the University of Michigan, Merit - * Network, Inc., and their contributors. - * 4. Neither the name of the University, Merit Network, nor the - * names of their contributors may be used to endorse or - * promote products derived from this software without - * specific prior written permission. - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef UNUSED -# if __GNUC__ >= 3 -# define UNUSED __attribute__((unused)) -# else -# define UNUSED -# endif -#endif - -UNUSED static char copyright[] = -"This product includes software developed by the University of Michigan, Merit" -"Network, Inc., and their contributors."; - -#include /* assert */ -#include /* isdigit */ -#include /* errno */ -#include /* sin */ -#include /* NULL */ -#include /* sprintf, fprintf, stderr */ -#include /* free, atol, calloc */ -#include /* memcpy, strchr, strlen */ -#include /* BSD: for inet_addr */ -#include /* BSD, Linux: for inet_addr */ -#include /* BSD, Linux: for inet_addr */ -#include /* BSD, Linux, Solaris: for inet_addr */ -#include - -#include "zeek/patricia.h" - -#define Delete free - -// From Zeek for reporting memory exhaustion. -extern void out_of_memory(const char* where); - -/* { from prefix.c */ - -/* prefix_tochar - * convert prefix information to bytes - */ -u_char * -prefix_tochar (prefix_t * prefix) -{ - if (prefix == NULL) - return (NULL); - - return ((u_char *) & prefix->add.sin); -} - -int -comp_with_mask (void *addr, void *dest, u_int mask) -{ - - if ( /* mask/8 == 0 || */ memcmp (addr, dest, mask / 8) == 0) { - int n = mask / 8; - int m = -(1 << (8 - (mask % 8))); - - if (mask % 8 == 0 || (((u_char *)addr)[n] & m) == (((u_char *)dest)[n] & m)) - return (1); - } - return (0); -} - -/* this allows imcomplete prefix */ -int -my_inet_pton (int af, const char *src, void *dst) -{ - if (af == AF_INET) { - int i, c, val; - u_char xp[sizeof(struct in_addr)] = {0, 0, 0, 0}; - - for (i = 0; ; i++) { - c = *src++; - if (!isdigit (c)) - return (-1); - val = 0; - do { - val = val * 10 + c - '0'; - if (val > 255) - return (0); - c = *src++; - } while (c && isdigit (c)); - xp[i] = val; - if (c == '\0') - break; - if (c != '.') - return (0); - if (i >= 3) - return (0); - } - memcpy (dst, xp, sizeof(struct in_addr)); - return (1); - } else if (af == AF_INET6) { - return (inet_pton (af, src, dst)); - } else { -#ifndef NT - errno = EAFNOSUPPORT; -#endif /* NT */ - return -1; - } -} - -#define PATRICIA_MAX_THREADS 16 - -/* - * convert prefix information to ascii string with length - * thread safe and (almost) re-entrant implementation - */ -char * -prefix_toa2x (prefix_t *prefix, char *buff, int with_len) -{ - if (prefix == NULL) - return ("(Null)"); - assert (prefix->ref_count >= 0); - if (buff == NULL) { - - struct buffer { - char buffs[PATRICIA_MAX_THREADS][48+5]; - u_int i; - } *buffp; - -# if 0 - THREAD_SPECIFIC_DATA (struct buffer, buffp, 1); -# else - { /* for scope only */ - static struct buffer local_buff; - buffp = &local_buff; - } -# endif - if (buffp == NULL) { - /* XXX should we report an error? */ - return (NULL); - } - - buff = buffp->buffs[buffp->i++%PATRICIA_MAX_THREADS]; - } - if (prefix->family == AF_INET) { - u_char *a; - assert (prefix->bitlen <= sizeof(struct in_addr) * 8); - a = prefix_touchar (prefix); - if (with_len) { - sprintf (buff, "%d.%d.%d.%d/%d", a[0], a[1], a[2], a[3], - prefix->bitlen); - } - else { - sprintf (buff, "%d.%d.%d.%d", a[0], a[1], a[2], a[3]); - } - return (buff); - } - else if (prefix->family == AF_INET6) { - char *r; - r = (char *) inet_ntop (AF_INET6, &prefix->add.sin6, buff, 48 /* a guess value */ ); - if (r && with_len) { - assert (prefix->bitlen <= sizeof(struct in6_addr) * 8); - sprintf (buff + strlen (buff), "/%d", prefix->bitlen); - } - return (buff); - } - else - return (NULL); -} - -/* prefix_toa2 - * convert prefix information to ascii string - */ -char * -prefix_toa2 (prefix_t *prefix, char *buff) -{ - return (prefix_toa2x (prefix, buff, 0)); -} - -/* prefix_toa - */ -char * -prefix_toa (prefix_t * prefix) -{ - return (prefix_toa2 (prefix, (char *) NULL)); -} - -prefix_t * -New_Prefix2 (int family, void *dest, int bitlen, prefix_t *prefix) -{ - int dynamic_allocated = 0; - int default_bitlen = sizeof(struct in_addr) * 8; - - if (family == AF_INET6) { - default_bitlen = sizeof(struct in6_addr) * 8; - if (prefix == NULL) { - prefix = calloc(1, sizeof (prefix_t)); - if (prefix == NULL) - out_of_memory("patricia/new_prefix2: unable to allocate memory"); - - dynamic_allocated++; - } - memcpy (&prefix->add.sin6, dest, sizeof(struct in6_addr)); - } - else - if (family == AF_INET) { - if (prefix == NULL) { -#ifndef NT - prefix = calloc(1, sizeof (prefix_t)); - if (prefix == NULL) - out_of_memory("patricia/new_prefix2: unable to allocate memory"); -#else - //for some reason, compiler is getting - //prefix4_t size incorrect on NT - prefix = calloc(1, sizeof (prefix_t)); - if (prefix == NULL) - out_of_memory("patricia/new_prefix2: unable to allocate memory"); -#endif /* NT */ - - dynamic_allocated++; - } - memcpy (&prefix->add.sin, dest, sizeof(struct in_addr)); - } - else { - return (NULL); - } - - prefix->bitlen = (bitlen >= 0)? bitlen: default_bitlen; - prefix->family = family; - prefix->ref_count = 0; - if (dynamic_allocated) { - prefix->ref_count++; - } -/* fprintf(stderr, "[C %s, %d]\n", prefix_toa (prefix), prefix->ref_count); */ - return (prefix); -} - -prefix_t * -New_Prefix (int family, void *dest, int bitlen) -{ - return (New_Prefix2 (family, dest, bitlen, NULL)); -} - -/* ascii2prefix - */ -prefix_t * -ascii2prefix (int family, char *string) -{ - u_long bitlen, maxbitlen = 0; - char *cp; - struct in_addr sin; - struct in6_addr sin6; - int result; - char save[MAXLINE]; - - if (string == NULL) - return (NULL); - - /* easy way to handle both families */ - if (family == 0) { - family = AF_INET; - if (strchr (string, ':')) family = AF_INET6; - } - - if (family == AF_INET) { - maxbitlen = sizeof(struct in_addr) * 8; - } - else if (family == AF_INET6) { - maxbitlen = sizeof(struct in6_addr) * 8; - } - - if ((cp = strchr (string, '/')) != NULL) { - bitlen = atol (cp + 1); - /* *cp = '\0'; */ - /* copy the string to save. Avoid destroying the string */ - assert (cp - string < MAXLINE); - memcpy (save, string, cp - string); - save[cp - string] = '\0'; - string = save; - if (bitlen > maxbitlen) - bitlen = maxbitlen; - } - else { - bitlen = maxbitlen; - } - - if (family == AF_INET) { - if ((result = my_inet_pton (AF_INET, string, &sin)) <= 0) - return (NULL); - return (New_Prefix (AF_INET, &sin, bitlen)); - } - - else if (family == AF_INET6) { -// Get rid of this with next IPv6 upgrade -#if defined(NT) && !defined(HAVE_INET_NTOP) - inet6_addr(string, &sin6); - return (New_Prefix (AF_INET6, &sin6, bitlen)); -#else - if ((result = inet_pton (AF_INET6, string, &sin6)) <= 0) - return (NULL); -#endif /* NT */ - return (New_Prefix (AF_INET6, &sin6, bitlen)); - } - else - return (NULL); -} - -prefix_t * -Ref_Prefix (prefix_t * prefix) -{ - if (prefix == NULL) - return (NULL); - if (prefix->ref_count == 0) { - /* make a copy in case of a static prefix */ - return (New_Prefix2 (prefix->family, &prefix->add, prefix->bitlen, NULL)); - } - prefix->ref_count++; -/* fprintf(stderr, "[A %s, %d]\n", prefix_toa (prefix), prefix->ref_count); */ - return (prefix); -} - -void -Deref_Prefix (prefix_t * prefix) -{ - if (prefix == NULL) - return; - /* for secure programming, raise an assert. no static prefix can call this */ - assert (prefix->ref_count > 0); - - prefix->ref_count--; - assert (prefix->ref_count >= 0); - if (prefix->ref_count <= 0) { - Delete (prefix); - return; - } -} - -/* } */ - -/* #define PATRICIA_DEBUG 1 */ - -static int num_active_patricia = 0; - -/* these routines support continuous mask only */ - -patricia_tree_t * -New_Patricia (int maxbits) -{ - patricia_tree_t *patricia = calloc(1, sizeof *patricia); - if (patricia == NULL) - out_of_memory("patricia/new_patricia: unable to allocate memory"); - - patricia->maxbits = maxbits; - patricia->head = NULL; - patricia->num_active_node = 0; - assert (maxbits <= PATRICIA_MAXBITS); /* XXX */ - num_active_patricia++; - return (patricia); -} - - -/* - * if func is supplied, it will be called as func(node->data) - * before deleting the node - */ - -void -Clear_Patricia (patricia_tree_t *patricia, data_fn_t func) -{ - assert (patricia); - if (patricia->head) { - - patricia_node_t *Xstack[PATRICIA_MAXBITS+1]; - patricia_node_t **Xsp = Xstack; - patricia_node_t *Xrn = patricia->head; - - while (Xrn) { - patricia_node_t *l = Xrn->l; - patricia_node_t *r = Xrn->r; - - if (Xrn->prefix) { - Deref_Prefix (Xrn->prefix); - if (Xrn->data && func) - func (Xrn->data); - } - else { - assert (Xrn->data == NULL); - } - Delete (Xrn); - patricia->num_active_node--; - - if (l) { - if (r) { - *Xsp++ = r; - } - Xrn = l; - } else if (r) { - Xrn = r; - } else if (Xsp != Xstack) { - Xrn = *(--Xsp); - } else { - Xrn = NULL; - } - } - } - assert (patricia->num_active_node == 0); - /* Delete (patricia); */ -} - - -void -Destroy_Patricia (patricia_tree_t *patricia, data_fn_t func) -{ - Clear_Patricia (patricia, func); - Delete (patricia); - num_active_patricia--; -} - - -/* - * if func is supplied, it will be called as func(node->prefix, node->data) - */ - -void -patricia_process (patricia_tree_t *patricia, prefix_data_fn_t func) -{ - patricia_node_t *node; - assert (func); - - PATRICIA_WALK (patricia->head, node) { - func (node->prefix, node->data); - } PATRICIA_WALK_END; -} - -size_t -patricia_walk_inorder(patricia_node_t *node, prefix_data_fn_t func) -{ - size_t n = 0; - assert(func); - - if (node->l) { - n += patricia_walk_inorder(node->l, func); - } - - if (node->prefix) { - func(node->prefix, node->data); - n++; - } - - if (node->r) { - n += patricia_walk_inorder(node->r, func); - } - - return n; -} - - -patricia_node_t * -patricia_search_exact (patricia_tree_t *patricia, prefix_t *prefix) -{ - patricia_node_t *node; - u_char *addr; - u_int bitlen; - - assert (patricia); - assert (prefix); - assert (prefix->bitlen <= patricia->maxbits); - - if (patricia->head == NULL) - return (NULL); - - node = patricia->head; - addr = prefix_touchar (prefix); - bitlen = prefix->bitlen; - - while (node->bit < bitlen) { - - if (BIT_TEST (addr[node->bit >> 3], 0x80 >> (node->bit & 0x07))) { -#ifdef PATRICIA_DEBUG - if (node->prefix) - fprintf (stderr, "patricia_search_exact: take right %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); - else - fprintf (stderr, "patricia_search_exact: take right at %u\n", - node->bit); -#endif /* PATRICIA_DEBUG */ - node = node->r; - } - else { -#ifdef PATRICIA_DEBUG - if (node->prefix) - fprintf (stderr, "patricia_search_exact: take left %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); - else - fprintf (stderr, "patricia_search_exact: take left at %u\n", - node->bit); -#endif /* PATRICIA_DEBUG */ - node = node->l; - } - - if (node == NULL) - return (NULL); - } - -#ifdef PATRICIA_DEBUG - if (node->prefix) - fprintf (stderr, "patricia_search_exact: stop at %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); - else - fprintf (stderr, "patricia_search_exact: stop at %u\n", node->bit); -#endif /* PATRICIA_DEBUG */ - if (node->bit > bitlen || node->prefix == NULL) - return (NULL); - assert (node->bit == bitlen); - assert (node->bit == node->prefix->bitlen); - if (comp_with_mask (prefix_tochar (node->prefix), prefix_tochar (prefix), - bitlen)) { -#ifdef PATRICIA_DEBUG - fprintf (stderr, "patricia_search_exact: found %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); -#endif /* PATRICIA_DEBUG */ - return (node); - } - return (NULL); -} - -bool -patricia_search_all (patricia_tree_t *patricia, prefix_t *prefix, patricia_node_t ***list, int *n) -{ - patricia_node_t *node; - patricia_node_t *stack[PATRICIA_MAXBITS + 1]; - u_char *addr; - u_int bitlen; - int cnt = 0; - - assert (patricia); - assert (prefix); - assert (prefix->bitlen <= patricia->maxbits); - assert (n); - assert (list); - assert (*list == NULL); - - *n = 0; - - if (patricia->head == NULL) - return (NULL); - - node = patricia->head; - addr = prefix_touchar (prefix); - bitlen = prefix->bitlen; - - while (node->bit < bitlen) { - - if (node->prefix) { -#ifdef PATRICIA_DEBUG - fprintf (stderr, "patricia_search_all: push %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); -#endif /* PATRICIA_DEBUG */ - stack[cnt++] = node; - } - - if (BIT_TEST (addr[node->bit >> 3], 0x80 >> (node->bit & 0x07))) { -#ifdef PATRICIA_DEBUG - if (node->prefix) - fprintf (stderr, "patricia_search_all: take right %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); - else - fprintf (stderr, "patricia_search_all: take right at %d\n", - node->bit); -#endif /* PATRICIA_DEBUG */ - node = node->r; - } else { -#ifdef PATRICIA_DEBUG - if (node->prefix) - fprintf (stderr, "patricia_search_all: take left %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); - else - fprintf (stderr, "patricia_search_all: take left at %d\n", - node->bit); -#endif /* PATRICIA_DEBUG */ - node = node->l; - } - - if (node == NULL) - break; - } - - if (node && node->prefix) - stack[cnt++] = node; - -#ifdef PATRICIA_DEBUG - if (node == NULL) - fprintf (stderr, "patricia_search_all: stop at null\n"); - else if (node->prefix) - fprintf (stderr, "patricia_search_all: stop at %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); - else - fprintf (stderr, "patricia_search_all: stop at %d\n", node->bit); -#endif /* PATRICIA_DEBUG */ - - if (cnt <= 0) - return false; - - // ok, now we have an upper bound of how much we can return. Let's just alloc that... - patricia_node_t **outlist = calloc(cnt, sizeof(patricia_node_t*)); - if (outlist == NULL) - out_of_memory("patricia/patricia_search_all: unable to allocate memory"); - - while (--cnt >= 0) { - node = stack[cnt]; -#ifdef PATRICIA_DEBUG - fprintf (stderr, "patricia_search_all: pop %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); -#endif /* PATRICIA_DEBUG */ - if (comp_with_mask (prefix_tochar (node->prefix), prefix_tochar (prefix), node->prefix->bitlen)) { -#ifdef PATRICIA_DEBUG - fprintf (stderr, "patricia_search_all: found %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); -#endif /* PATRICIA_DEBUG */ - outlist[*n] = node; - (*n)++; - } - } - *list = outlist; - return (*n == 0); -} - - -/* if inclusive != 0, "best" may be the given prefix itself */ -patricia_node_t * -patricia_search_best2 (patricia_tree_t *patricia, prefix_t *prefix, int inclusive) -{ - patricia_node_t *node; - patricia_node_t *stack[PATRICIA_MAXBITS + 1]; - u_char *addr; - u_int bitlen; - int cnt = 0; - - assert (patricia); - assert (prefix); - assert (prefix->bitlen <= patricia->maxbits); - - if (patricia->head == NULL) - return (NULL); - - node = patricia->head; - addr = prefix_touchar (prefix); - bitlen = prefix->bitlen; - - while (node->bit < bitlen) { - - if (node->prefix) { -#ifdef PATRICIA_DEBUG - fprintf (stderr, "patricia_search_best: push %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); -#endif /* PATRICIA_DEBUG */ - stack[cnt++] = node; - } - - if (BIT_TEST (addr[node->bit >> 3], 0x80 >> (node->bit & 0x07))) { -#ifdef PATRICIA_DEBUG - if (node->prefix) - fprintf (stderr, "patricia_search_best: take right %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); - else - fprintf (stderr, "patricia_search_best: take right at %u\n", - node->bit); -#endif /* PATRICIA_DEBUG */ - node = node->r; - } - else { -#ifdef PATRICIA_DEBUG - if (node->prefix) - fprintf (stderr, "patricia_search_best: take left %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); - else - fprintf (stderr, "patricia_search_best: take left at %u\n", - node->bit); -#endif /* PATRICIA_DEBUG */ - node = node->l; - } - - if (node == NULL) - break; - } - - if (inclusive && node && node->prefix) - stack[cnt++] = node; - -#ifdef PATRICIA_DEBUG - if (node == NULL) - fprintf (stderr, "patricia_search_best: stop at null\n"); - else if (node->prefix) - fprintf (stderr, "patricia_search_best: stop at %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); - else - fprintf (stderr, "patricia_search_best: stop at %u\n", node->bit); -#endif /* PATRICIA_DEBUG */ - - if (cnt <= 0) - return (NULL); - - while (--cnt >= 0) { - node = stack[cnt]; -#ifdef PATRICIA_DEBUG - fprintf (stderr, "patricia_search_best: pop %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); -#endif /* PATRICIA_DEBUG */ - if (comp_with_mask (prefix_tochar (node->prefix), - prefix_tochar (prefix), - node->prefix->bitlen) && node->prefix->bitlen <= bitlen) { -#ifdef PATRICIA_DEBUG - fprintf (stderr, "patricia_search_best: found %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); -#endif /* PATRICIA_DEBUG */ - return (node); - } - } - return (NULL); -} - - -patricia_node_t * -patricia_search_best (patricia_tree_t *patricia, prefix_t *prefix) -{ - return (patricia_search_best2 (patricia, prefix, 1)); -} - - -patricia_node_t * -patricia_lookup (patricia_tree_t *patricia, prefix_t *prefix) -{ - patricia_node_t *node, *new_node, *parent, *glue; - u_char *addr, *test_addr; - u_int bitlen, check_bit, differ_bit; - int i, j, r; - - assert (patricia); - assert (prefix); - assert (prefix->bitlen <= patricia->maxbits); - - if (patricia->head == NULL) { - node = calloc(1, sizeof *node); - if (node == NULL) - out_of_memory("patricia/patricia_lookup: unable to allocate memory"); - - node->bit = prefix->bitlen; - node->prefix = Ref_Prefix (prefix); - node->parent = NULL; - node->l = node->r = NULL; - node->data = NULL; - patricia->head = node; -#ifdef PATRICIA_DEBUG - fprintf (stderr, "patricia_lookup: new_node #0 %s/%d (head)\n", - prefix_toa (prefix), prefix->bitlen); -#endif /* PATRICIA_DEBUG */ - patricia->num_active_node++; - return (node); - } - - addr = prefix_touchar (prefix); - bitlen = prefix->bitlen; - node = patricia->head; - - while (node->bit < bitlen || node->prefix == NULL) { - - if (node->bit < patricia->maxbits && - BIT_TEST (addr[node->bit >> 3], 0x80 >> (node->bit & 0x07))) { - if (node->r == NULL) - break; -#ifdef PATRICIA_DEBUG - if (node->prefix) - fprintf (stderr, "patricia_lookup: take right %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); - else - fprintf (stderr, "patricia_lookup: take right at %u\n", node->bit); -#endif /* PATRICIA_DEBUG */ - node = node->r; - } - else { - if (node->l == NULL) - break; -#ifdef PATRICIA_DEBUG - if (node->prefix) - fprintf (stderr, "patricia_lookup: take left %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); - else - fprintf (stderr, "patricia_lookup: take left at %u\n", node->bit); -#endif /* PATRICIA_DEBUG */ - node = node->l; - } - - assert (node); - } - - assert (node->prefix); -#ifdef PATRICIA_DEBUG - fprintf (stderr, "patricia_lookup: stop at %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); -#endif /* PATRICIA_DEBUG */ - - test_addr = prefix_touchar (node->prefix); - /* find the first bit different */ - check_bit = (node->bit < bitlen)? node->bit: bitlen; - differ_bit = 0; - for (i = 0; i*8 < check_bit; i++) { - if ((r = (addr[i] ^ test_addr[i])) == 0) { - differ_bit = (i + 1) * 8; - continue; - } - /* I know the better way, but for now */ - for (j = 0; j < 8; j++) { - if (BIT_TEST (r, (0x80 >> j))) - break; - } - /* must be found */ - assert (j < 8); - differ_bit = i * 8 + j; - break; - } - if (differ_bit > check_bit) - differ_bit = check_bit; -#ifdef PATRICIA_DEBUG - fprintf (stderr, "patricia_lookup: differ_bit %d\n", differ_bit); -#endif /* PATRICIA_DEBUG */ - - parent = node->parent; - while (parent && parent->bit >= differ_bit) { - node = parent; - parent = node->parent; -#ifdef PATRICIA_DEBUG - if (node->prefix) - fprintf (stderr, "patricia_lookup: up to %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); - else - fprintf (stderr, "patricia_lookup: up to %u\n", node->bit); -#endif /* PATRICIA_DEBUG */ - } - - if (differ_bit == bitlen && node->bit == bitlen) { - if (node->prefix) { -#ifdef PATRICIA_DEBUG - fprintf (stderr, "patricia_lookup: found %s/%d\n", - prefix_toa (node->prefix), node->prefix->bitlen); -#endif /* PATRICIA_DEBUG */ - return (node); - } - node->prefix = Ref_Prefix (prefix); -#ifdef PATRICIA_DEBUG - fprintf (stderr, "patricia_lookup: new node #1 %s/%d (glue mod)\n", - prefix_toa (prefix), prefix->bitlen); -#endif /* PATRICIA_DEBUG */ - assert (node->data == NULL); - return (node); - } - - new_node = calloc(1, sizeof *new_node); - if (new_node == NULL) - out_of_memory("patricia/patricia_lookup: unable to allocate memory"); - - new_node->bit = prefix->bitlen; - new_node->prefix = Ref_Prefix (prefix); - new_node->parent = NULL; - new_node->l = new_node->r = NULL; - new_node->data = NULL; - patricia->num_active_node++; - - if (node->bit == differ_bit) { - new_node->parent = node; - if (node->bit < patricia->maxbits && - BIT_TEST (addr[node->bit >> 3], 0x80 >> (node->bit & 0x07))) { - assert (node->r == NULL); - node->r = new_node; - } - else { - assert (node->l == NULL); - node->l = new_node; - } -#ifdef PATRICIA_DEBUG - fprintf (stderr, "patricia_lookup: new_node #2 %s/%d (child)\n", - prefix_toa (prefix), prefix->bitlen); -#endif /* PATRICIA_DEBUG */ - return (new_node); - } - - if (bitlen == differ_bit) { - if (bitlen < patricia->maxbits && - BIT_TEST (test_addr[bitlen >> 3], 0x80 >> (bitlen & 0x07))) { - new_node->r = node; - } - else { - new_node->l = node; - } - new_node->parent = node->parent; - if (node->parent == NULL) { - assert (patricia->head == node); - patricia->head = new_node; - } - else if (node->parent->r == node) { - node->parent->r = new_node; - } - else { - node->parent->l = new_node; - } - node->parent = new_node; -#ifdef PATRICIA_DEBUG - fprintf (stderr, "patricia_lookup: new_node #3 %s/%d (parent)\n", - prefix_toa (prefix), prefix->bitlen); -#endif /* PATRICIA_DEBUG */ - } - else { - glue = calloc(1, sizeof *glue); - if (glue == NULL) - out_of_memory("patricia/patricia_lookup: unable to allocate memory"); - - glue->bit = differ_bit; - glue->prefix = NULL; - glue->parent = node->parent; - glue->data = NULL; - patricia->num_active_node++; - if (differ_bit < patricia->maxbits && - BIT_TEST (addr[differ_bit >> 3], 0x80 >> (differ_bit & 0x07))) { - glue->r = new_node; - glue->l = node; - } - else { - glue->r = node; - glue->l = new_node; - } - new_node->parent = glue; - - if (node->parent == NULL) { - assert (patricia->head == node); - patricia->head = glue; - } - else if (node->parent->r == node) { - node->parent->r = glue; - } - else { - node->parent->l = glue; - } - node->parent = glue; -#ifdef PATRICIA_DEBUG - fprintf (stderr, "patricia_lookup: new_node #4 %s/%d (glue+node)\n", - prefix_toa (prefix), prefix->bitlen); -#endif /* PATRICIA_DEBUG */ - } - return (new_node); -} - - -void -patricia_remove (patricia_tree_t *patricia, patricia_node_t *node) -{ - patricia_node_t *parent, *child; - - assert (patricia); - assert (node); - - if (node->r && node->l) { -#ifdef PATRICIA_DEBUG - fprintf (stderr, "patricia_remove: #0 %s/%d (r & l)\n", - prefix_toa (node->prefix), node->prefix->bitlen); -#endif /* PATRICIA_DEBUG */ - - /* this might be a placeholder node -- have to check and make sure - * there is a prefix aossciated with it ! */ - if (node->prefix != NULL) - Deref_Prefix (node->prefix); - node->prefix = NULL; - /* Also I needed to clear data pointer -- masaki */ - node->data = NULL; - return; - } - - if (node->r == NULL && node->l == NULL) { -#ifdef PATRICIA_DEBUG - fprintf (stderr, "patricia_remove: #1 %s/%d (!r & !l)\n", - prefix_toa (node->prefix), node->prefix->bitlen); -#endif /* PATRICIA_DEBUG */ - parent = node->parent; - Deref_Prefix (node->prefix); - Delete (node); - patricia->num_active_node--; - - if (parent == NULL) { - assert (patricia->head == node); - patricia->head = NULL; - return; - } - - if (parent->r == node) { - parent->r = NULL; - child = parent->l; - } - else { - assert (parent->l == node); - parent->l = NULL; - child = parent->r; - } - - if (parent->prefix) - return; - - /* we need to remove parent too */ - - if (parent->parent == NULL) { - assert (patricia->head == parent); - patricia->head = child; - } - else if (parent->parent->r == parent) { - parent->parent->r = child; - } - else { - assert (parent->parent->l == parent); - parent->parent->l = child; - } - child->parent = parent->parent; - Delete (parent); - patricia->num_active_node--; - return; - } - -#ifdef PATRICIA_DEBUG - fprintf (stderr, "patricia_remove: #2 %s/%d (r ^ l)\n", - prefix_toa (node->prefix), node->prefix->bitlen); -#endif /* PATRICIA_DEBUG */ - if (node->r) { - child = node->r; - } - else { - assert (node->l); - child = node->l; - } - parent = node->parent; - child->parent = parent; - - Deref_Prefix (node->prefix); - Delete (node); - patricia->num_active_node--; - - if (parent == NULL) { - assert (patricia->head == node); - patricia->head = child; - return; - } - - if (parent->r == node) { - parent->r = child; - } - else { - assert (parent->l == node); - parent->l = child; - } -} - -/* { from demo.c */ - -patricia_node_t * -make_and_lookup (patricia_tree_t *tree, char *string) -{ - prefix_t *prefix; - patricia_node_t *node; - - prefix = ascii2prefix (AF_INET, string); - printf ("make_and_lookup: %s/%d\n", prefix_toa (prefix), prefix->bitlen); - node = patricia_lookup (tree, prefix); - Deref_Prefix (prefix); - return (node); -} - -patricia_node_t * -try_search_exact (patricia_tree_t *tree, char *string) -{ - prefix_t *prefix; - patricia_node_t *node; - - prefix = ascii2prefix (AF_INET, string); - printf ("try_search_exact: %s/%d\n", prefix_toa (prefix), prefix->bitlen); - if ((node = patricia_search_exact (tree, prefix)) == NULL) { - printf ("try_search_exact: not found\n"); - } - else { - printf ("try_search_exact: %s/%d found\n", - prefix_toa (node->prefix), node->prefix->bitlen); - } - Deref_Prefix (prefix); - return (node); -} - -void -lookup_then_remove (patricia_tree_t *tree, char *string) -{ - patricia_node_t *node; - - if ( (node = try_search_exact(tree, string)) ) - patricia_remove (tree, node); -} - -patricia_node_t * -try_search_best (patricia_tree_t *tree, char *string) -{ - prefix_t *prefix; - patricia_node_t *node; - - prefix = ascii2prefix (AF_INET, string); - printf ("try_search_best: %s/%d\n", prefix_toa (prefix), prefix->bitlen); - if ((node = patricia_search_best (tree, prefix)) == NULL) - printf ("try_search_best: not found\n"); - else - printf ("try_search_best: %s/%d found\n", - prefix_toa (node->prefix), node->prefix->bitlen); - Deref_Prefix (prefix); - return (node); -} - -/* } */ diff --git a/src/patricia.h b/src/patricia.h deleted file mode 100644 index d4374f2b0b..0000000000 --- a/src/patricia.h +++ /dev/null @@ -1,195 +0,0 @@ -/* - * This code originates from Dave Plonka's Net::Security perl module. An adaptation - * of it in C is kept at https://github.com/CAIDA/cc-common/tree/master/libpatricia. - * That repository is considered the upstream version for Zeek's fork. We make some - * custom changes to this upstream: - * - Replace void_fn_t with data_fn_t and prefix_data_fn_t - * - Add patricia_search_all method - * - * The current version is based on commit 4a2c61374f507a420d28bd9084c976142d279605 - * from that repo. - */ - -/* - * Dave Plonka - * - * This product includes software developed by the University of Michigan, - * Merit Network, Inc., and their contributors. - * - * This file had been called "radix.h" in the MRT sources. - * - * I renamed it to "patricia.h" since it's not an implementation of a general - * radix trie. Also, pulled in various requirements from "mrt.h" and added - * some other things it could be used as a standalone API. - */ - -/* From copyright.txt: - * - * Copyright (c) 1997, 1998, 1999 - * - * - * The Regents of the University of Michigan ("The Regents") and Merit Network, - * Inc. All rights reserved. - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * 1. Redistributions of source code must retain the above - * copyright notice, this list of conditions and the - * following disclaimer. - * 2. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * 3. All advertising materials mentioning features or use of - * this software must display the following acknowledgement: - * This product includes software developed by the University of Michigan, Merit - * Network, Inc., and their contributors. - * 4. Neither the name of the University, Merit Network, nor the - * names of their contributors may be used to endorse or - * promote products derived from this software without - * specific prior written permission. - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#pragma once - -/* { from defs.h */ -#define prefix_touchar(prefix) ((u_char *)&(prefix)->add.sin) -#define MAXLINE 1024 -#define BIT_TEST(f, b) ((f) & (b)) -/* } */ - -#define addroute make_and_lookup - -#include /* for u_* definitions (on FreeBSD 5) */ - -#include /* for EAFNOSUPPORT */ -#ifndef EAFNOSUPPORT -# defined EAFNOSUPPORT WSAEAFNOSUPPORT -# include -#else -# include /* for struct in_addr */ -#endif - -#include /* for AF_INET */ - -/* { from mrt.h */ - -typedef struct _prefix4_t { - u_short family; /* AF_INET | AF_INET6 */ - u_short bitlen; /* same as mask? */ - int ref_count; /* reference count */ - struct in_addr sin; -} prefix4_t; - -typedef struct _prefix_t { - u_short family; /* AF_INET | AF_INET6 */ - u_short bitlen; /* same as mask? */ - int ref_count; /* reference count */ - union { - struct in_addr sin; - struct in6_addr sin6; - } add; -} prefix_t; - -typedef void (*data_fn_t)(void*); -typedef void (*prefix_data_fn_t)(prefix_t*, void*); - -/* } */ - -typedef struct _patricia_node_t { - u_int bit; /* flag if this node used */ - prefix_t *prefix; /* who we are in patricia tree */ - struct _patricia_node_t *l, *r; /* left and right children */ - struct _patricia_node_t *parent;/* may be used */ - void *data; /* pointer to data */ - void *user1; /* pointer to usr data (ex. route flap info) */ -} patricia_node_t; - -typedef struct _patricia_tree_t { - patricia_node_t *head; - u_int maxbits; /* for IP, 32 bit addresses */ - int num_active_node; /* for debug purpose */ -} patricia_tree_t; - - -patricia_node_t *patricia_search_exact (patricia_tree_t *patricia, prefix_t *prefix); -bool patricia_search_all (patricia_tree_t *patricia, prefix_t *prefix, patricia_node_t ***list, int *n); -patricia_node_t *patricia_search_best (patricia_tree_t *patricia, prefix_t *prefix); -patricia_node_t * patricia_search_best2 (patricia_tree_t *patricia, prefix_t *prefix, - int inclusive); -patricia_node_t *patricia_lookup (patricia_tree_t *patricia, prefix_t *prefix); -void patricia_remove (patricia_tree_t *patricia, patricia_node_t *node); -patricia_tree_t *New_Patricia (int maxbits); -void Clear_Patricia (patricia_tree_t *patricia, data_fn_t func); -void Destroy_Patricia (patricia_tree_t *patricia, data_fn_t func); - -void patricia_process (patricia_tree_t *patricia, prefix_data_fn_t func); - -void Deref_Prefix (prefix_t * prefix); -char *prefix_toa (prefix_t * prefix); - -/* { from demo.c */ - -prefix_t * -ascii2prefix (int family, char *string); - -patricia_node_t * -make_and_lookup (patricia_tree_t *tree, char *string); - -/* } */ - -#define PATRICIA_MAXBITS (sizeof(struct in6_addr) * 8) -#define PATRICIA_NBIT(x) (0x80 >> ((x) & 0x7f)) -#define PATRICIA_NBYTE(x) ((x) >> 3) - -#define PATRICIA_DATA_GET(node, type) (type *)((node)->data) -#define PATRICIA_DATA_SET(node, value) ((node)->data = (void *)(value)) - -#define PATRICIA_WALK(Xhead, Xnode) \ - do { \ - patricia_node_t *Xstack[PATRICIA_MAXBITS+1]; \ - patricia_node_t **Xsp = Xstack; \ - patricia_node_t *Xrn = (Xhead); \ - while ((Xnode = Xrn)) { \ - if (Xnode->prefix) - -#define PATRICIA_WALK_ALL(Xhead, Xnode) \ -do { \ - patricia_node_t *Xstack[PATRICIA_MAXBITS+1]; \ - patricia_node_t **Xsp = Xstack; \ - patricia_node_t *Xrn = (Xhead); \ - while ((Xnode = Xrn)) { \ - if (1) - -#define PATRICIA_WALK_BREAK { \ - if (Xsp != Xstack) { \ - Xrn = *(--Xsp); \ - } else { \ - Xrn = (patricia_node_t *) 0; \ - } \ - continue; } - -#define PATRICIA_WALK_END \ - if (Xrn->l) { \ - if (Xrn->r) { \ - *Xsp++ = Xrn->r; \ - } \ - Xrn = Xrn->l; \ - } else if (Xrn->r) { \ - Xrn = Xrn->r; \ - } else if (Xsp != Xstack) { \ - Xrn = *(--Xsp); \ - } else { \ - Xrn = (patricia_node_t *) 0; \ - } \ - } \ - } while (0) diff --git a/src/setsignal.c b/src/setsignal.c deleted file mode 100644 index d91b91f9e0..0000000000 --- a/src/setsignal.c +++ /dev/null @@ -1,52 +0,0 @@ -/* - * See the file "COPYING" in the main distribution directory for copyright. - */ - -#include "zeek/zeek-config.h" /* must appear before first ifdef */ - -#include - -#ifdef HAVE_MEMORY_H -#include -#endif -#include -#ifdef HAVE_SIGACTION -#include -#endif - -#include "zeek/setsignal.h" - -/* - * An os independent signal() with BSD semantics, e.g. the signal - * catcher is restored following service of the signal. - * - * When sigset() is available, signal() has SYSV semantics and sigset() - * has BSD semantics and call interface. Unfortunately, Linux does not - * have sigset() so we use the more complicated sigaction() interface - * there. - * - * Did I mention that signals suck? - */ -RETSIGTYPE -(*setsignal (int sig, RETSIGTYPE (*func)(int)))(int) -{ -#ifdef HAVE_SIGACTION - struct sigaction old, new; - - memset(&new, 0, sizeof(new)); - new.sa_handler = func; -#ifdef SA_RESTART - new.sa_flags |= SA_RESTART; -#endif - if (sigaction(sig, &new, &old) < 0) - return (SIG_ERR); - return (old.sa_handler); - -#else -#ifdef HAVE_SIGSET - return (sigset(sig, func)); -#else - return (signal(sig, func)); -#endif -#endif -} diff --git a/src/setsignal.h b/src/setsignal.h deleted file mode 100644 index d290265373..0000000000 --- a/src/setsignal.h +++ /dev/null @@ -1,7 +0,0 @@ -/* - * See the file "COPYING" in the main distribution directory for copyright. - * - */ -#pragma once - -RETSIGTYPE (*setsignal(int, RETSIGTYPE (*)(int)))(int); diff --git a/src/strsep.c b/src/strsep.c deleted file mode 100644 index ef0970bd40..0000000000 --- a/src/strsep.c +++ /dev/null @@ -1,93 +0,0 @@ -/*- - * Copyright (c) 1990, 1993 - * The Regents of the University of California. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the University of - * California, Berkeley and its contributors. - * 4. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include "zeek/zeek-config.h" - -#ifndef HAVE_STRSEP - -#include - -#include -#include - -char *strsep(char **, const char *); - -#if defined(LIBC_SCCS) && !defined(lint) -static char sccsid[] = "@(#)strsep.c 8.1 (Berkeley) 6/4/93"; -#endif /* LIBC_SCCS and not lint */ -#ifndef lint -static const char rcsid[] = - "$FreeBSD: src/lib/libc/string/strsep.c,v 1.2.12.1 2001/07/09 23:30:07 obrien Exp $"; -#endif - -/* - * Get next token from string *stringp, where tokens are possibly-empty - * strings separated by characters from delim. - * - * Writes NULs into the string at *stringp to end tokens. - * delim need not remain constant from call to call. - * On return, *stringp points past the last NUL written (if there might - * be further tokens), or is NULL (if there are definitely no more tokens). - * - * If *stringp is NULL, strsep returns NULL. - */ -char * -strsep(stringp, delim) - register char **stringp; - register const char *delim; -{ - register char *s; - register const char *spanp; - register int c, sc; - char *tok; - - if ((s = *stringp) == NULL) - return (NULL); - for (tok = s;;) { - c = *s++; - spanp = delim; - do { - if ((sc = *spanp++) == c) { - if (c == 0) - s = NULL; - else - s[-1] = 0; - *stringp = s; - return (tok); - } - } while (sc != 0); - } - /* NOTREACHED */ -} - -#endif diff --git a/src/supervisor/Supervisor.cc b/src/supervisor/Supervisor.cc index 4b7d67e9a4..53d9f4a77f 100644 --- a/src/supervisor/Supervisor.cc +++ b/src/supervisor/Supervisor.cc @@ -21,7 +21,7 @@ extern "C" { -#include "zeek/setsignal.h" +#include "zeek/3rdparty/setsignal.h" } #include "zeek/DebugLogger.h" diff --git a/src/threading/Formatter.cc b/src/threading/Formatter.cc index 680a4ece4c..00975567f2 100644 --- a/src/threading/Formatter.cc +++ b/src/threading/Formatter.cc @@ -6,7 +6,7 @@ #include -#include "zeek/bro_inet_ntop.h" +#include "zeek/3rdparty/bro_inet_ntop.h" #include "zeek/threading/MsgThread.h" using zeek::threading::Field; diff --git a/src/util.cc b/src/util.cc index d36161b7e0..f63dfdbc83 100644 --- a/src/util.cc +++ b/src/util.cc @@ -43,8 +43,8 @@ #include #include +#include "zeek/3rdparty/ConvertUTF.h" #include "zeek/3rdparty/doctest.h" -#include "zeek/ConvertUTF.h" #include "zeek/Desc.h" #include "zeek/Dict.h" #include "zeek/Hash.h" diff --git a/src/util.h b/src/util.h index 5126dca59a..c35da260eb 100644 --- a/src/util.h +++ b/src/util.h @@ -72,7 +72,7 @@ extern HeapLeakChecker* heap_checker; extern "C" { -#include "zeek/modp_numtoa.h" +#include "zeek/3rdparty/modp_numtoa.h" } using bro_int_t = int64_t; diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index dad543c282..0a69d78667 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -67,7 +67,7 @@ extern "C" { -#include "zeek/setsignal.h" +#include "zeek/3rdparty/setsignal.h" }; zeek::detail::ScriptCoverageManager zeek::detail::script_coverage_mgr; From b23eb760430bbe44234f2e538ece5da538ccac9f Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Fri, 5 Nov 2021 08:23:02 +0100 Subject: [PATCH 048/210] Disable formatting for files in `testing/btest/plugins` Files in that folder were previously not formatted. With this patch we now disable formatting in that folder explicitly by adding a dedicated `clang-format` config which deactivates any formatting changes. --- testing/btest/btest.cfg | 2 +- testing/btest/plugins/.clang-format | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 testing/btest/plugins/.clang-format diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index e4de72f6c2..7e5eedd02a 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -3,7 +3,7 @@ TestDirs = doc bifs language core scripts coverage signatures plugins broker TmpDir = %(testbase)s/.tmp BaselineDir = %(testbase)s/Baseline IgnoreDirs = .svn CVS .tmp -IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store +IgnoreFiles = *.tmp *.swp .clang-format #* *.trace .DS_Store MinVersion = 0.63 [environment] diff --git a/testing/btest/plugins/.clang-format b/testing/btest/plugins/.clang-format new file mode 100644 index 0000000000..9d159247d5 --- /dev/null +++ b/testing/btest/plugins/.clang-format @@ -0,0 +1,2 @@ +DisableFormat: true +SortIncludes: false From dafc9691c5856000c0f37491c5358bace9522348 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Fri, 5 Nov 2021 10:41:22 +0100 Subject: [PATCH 049/210] Remove stale files `src/DebugCmdInfoConstants.*` The files generated from `src/DebugCmdInfoConstants.in` are placed in `build/src/` by the build setup, and generated file in `src/` removed here were unused and possibly out-of-date. --- .clang-format-ignore | 3 - src/DebugCmdInfoConstants.cc | 286 ----------------------------------- src/DebugCmdInfoConstants.h | 205 ------------------------- 3 files changed, 494 deletions(-) delete mode 100644 src/DebugCmdInfoConstants.cc delete mode 100644 src/DebugCmdInfoConstants.h diff --git a/.clang-format-ignore b/.clang-format-ignore index 1300d06344..bb01885b01 100644 --- a/.clang-format-ignore +++ b/.clang-format-ignore @@ -1,5 +1,2 @@ # Ignore everything 3rdparty src/3rdparty/* - -# These files are generated code -src/DebugCmdInfoConstants.* diff --git a/src/DebugCmdInfoConstants.cc b/src/DebugCmdInfoConstants.cc deleted file mode 100644 index f1043613d8..0000000000 --- a/src/DebugCmdInfoConstants.cc +++ /dev/null @@ -1,286 +0,0 @@ - -// -// This file was automatically generated from DebugCmdInfoConstants.in -// DO NOT EDIT. -// - -#include "zeek/util.h" -namespace zeek::detail { - -void init_global_dbg_constants () { - - { - DebugCmdInfo* info; - const char * const names[] = { }; - - info = new DebugCmdInfo(dcInvalid, names, 0, false, "This function should not be called", - false); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "help" - }; - - info = new DebugCmdInfo(dcHelp, names, 1, false, "Get help with debugger commands", - false); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "quit" - }; - - info = new DebugCmdInfo(dcQuit, names, 1, false, "Exit Zeek", - false); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "next" - }; - - info = new DebugCmdInfo(dcNext, names, 1, true, "Step to the following statement, skipping function calls", - true); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "step", - "s" - }; - - info = new DebugCmdInfo(dcStep, names, 2, true, "Step to following statements, stepping in to function calls", - true); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "continue", - "c" - }; - - info = new DebugCmdInfo(dcContinue, names, 2, true, "Resume execution of the policy script", - true); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "finish" - }; - - info = new DebugCmdInfo(dcFinish, names, 1, true, "Run until the currently-executing function completes", - true); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "break", - "b" - }; - - info = new DebugCmdInfo(dcBreak, names, 2, false, "Set a breakpoint", - false); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "cond" - }; - - info = new DebugCmdInfo(dcBreakCondition, names, 1, false, "", - false); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "delete", - "d" - }; - - info = new DebugCmdInfo(dcDeleteBreak, names, 2, false, "Delete the specified breakpoints; delete all if no arguments", - false); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "clear" - }; - - info = new DebugCmdInfo(dcClearBreak, names, 1, false, "", - false); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "disable", - "dis" - }; - - info = new DebugCmdInfo(dcDisableBreak, names, 2, false, "", - false); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "enable" - }; - - info = new DebugCmdInfo(dcEnableBreak, names, 1, false, "", - false); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "ignore" - }; - - info = new DebugCmdInfo(dcIgnoreBreak, names, 1, false, "", - false); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "print", - "p", - "set" - }; - - info = new DebugCmdInfo(dcPrint, names, 3, false, "Evaluate an expression and print the result (also aliased as 'set')", - true); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "backtrace", - "bt", - "where" - }; - - info = new DebugCmdInfo(dcBacktrace, names, 3, false, "Print a stack trace (with +- N argument, inner/outer N frames only)", - false); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "frame" - }; - - info = new DebugCmdInfo(dcFrame, names, 1, false, "Select frame number N", - false); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "up" - }; - - info = new DebugCmdInfo(dcUp, names, 1, false, "Select the stack frame one level up", - false); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "down" - }; - - info = new DebugCmdInfo(dcDown, names, 1, false, "Select the stack frame one level down", - false); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "info" - }; - - info = new DebugCmdInfo(dcInfo, names, 1, false, "Get information about the debugging environment", - false); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "list", - "l" - }; - - info = new DebugCmdInfo(dcList, names, 2, false, "Print source lines surrounding specified context", - true); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "display" - }; - - info = new DebugCmdInfo(dcDisplay, names, 1, false, "", - false); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "undisplay" - }; - - info = new DebugCmdInfo(dcUndisplay, names, 1, false, "", - false); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "trace" - }; - - info = new DebugCmdInfo(dcTrace, names, 1, false, "Turn on or off execution tracing (with no arguments, prints current state.)", - false); - g_DebugCmdInfos.push_back(info); - } - -} - -} // namespace zeek::detail diff --git a/src/DebugCmdInfoConstants.h b/src/DebugCmdInfoConstants.h deleted file mode 100644 index 44330482d3..0000000000 --- a/src/DebugCmdInfoConstants.h +++ /dev/null @@ -1,205 +0,0 @@ -void InitGlobalDbgConstants () { - { - - { - DebugCmdInfo* info; - const char * const names[] = { - "help" - }; - - info = new DebugCmdInfo (dcHelp, names, 1, false, "Get help with debugger commands"); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "quit" - }; - - info = new DebugCmdInfo (dcQuit, names, 1, false, ""); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "next" - }; - - info = new DebugCmdInfo (dcNext, names, 1, true, ""); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "step" - }; - - info = new DebugCmdInfo (dcStep, names, 1, true, ""); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "continue" - }; - - info = new DebugCmdInfo (dcContinue, names, 1, true, ""); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "finish" - }; - - info = new DebugCmdInfo (dcFinish, names, 1, true, ""); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "break" - }; - - info = new DebugCmdInfo (dcBreak, names, 1, false, ""); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "cond" - }; - - info = new DebugCmdInfo (dcBreakCondition, names, 1, false, ""); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "delete" - }; - - info = new DebugCmdInfo (dcDeleteBreak, names, 1, false, ""); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "clear" - }; - - info = new DebugCmdInfo (dcClearBreak, names, 1, false, ""); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "disable" - }; - - info = new DebugCmdInfo (dcDisableBreak, names, 1, false, ""); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "enable" - }; - - info = new DebugCmdInfo (dcEnableBreak, names, 1, false, ""); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "ignore" - }; - - info = new DebugCmdInfo (dcIgnoreBreak, names, 1, false, ""); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "print" - }; - - info = new DebugCmdInfo (dcPrint, names, 1, false, ""); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "backtrace", - "bt" - }; - - info = new DebugCmdInfo (dcBacktrace, names, 2, false, ""); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "frame" - }; - - info = new DebugCmdInfo (dcFrame, names, 1, false, ""); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "info" - }; - - info = new DebugCmdInfo (dcInfo, names, 1, false, ""); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "list" - }; - - info = new DebugCmdInfo (dcList, names, 1, false, ""); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "display" - }; - - info = new DebugCmdInfo (dcDisplay, names, 1, false, ""); - g_DebugCmdInfos.push_back(info); - } - - { - DebugCmdInfo* info; - const char * const names[] = { - "undisplay" - }; - - info = new DebugCmdInfo (dcUndisplay, names, 1, false, ""); - g_DebugCmdInfos.push_back(info); - } - -} From 4e16037acd3e55008101e6cfd4711a74e89f547a Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Fri, 5 Nov 2021 09:02:40 +0100 Subject: [PATCH 050/210] Format code with `clang-format` This patch formats files not conforming to the C++ formatting with `clang-format`. --- src/script_opt/CPP/RuntimeVec.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/script_opt/CPP/RuntimeVec.cc b/src/script_opt/CPP/RuntimeVec.cc index 3b8eff9c1f..c8279e7988 100644 --- a/src/script_opt/CPP/RuntimeVec.cc +++ b/src/script_opt/CPP/RuntimeVec.cc @@ -89,9 +89,9 @@ static VectorTypePtr base_vector_type__CPP(const VectorTypePtr& vt) #define VEC_OP1_WITH_DOUBLE(name, op) \ VEC_OP1( \ name, op, case TYPE_INTERNAL_DOUBLE \ - : \ - { \ - VEC_OP1_KERNEL(AsDouble, DoubleVal, op) break; \ + : { \ + VEC_OP1_KERNEL(AsDouble, DoubleVal, op) \ + break; \ }) // The unary operations supported for vectors. @@ -152,9 +152,9 @@ VEC_OP1(comp, ~, ) #define VEC_OP2_WITH_DOUBLE(name, op) \ VEC_OP2( \ name, op, case TYPE_INTERNAL_DOUBLE \ - : \ - { \ - VEC_OP2_KERNEL(AsDouble, DoubleVal, op) break; \ + : { \ + VEC_OP2_KERNEL(AsDouble, DoubleVal, op) \ + break; \ }) // The binary operations supported for vectors. From 77e2e8278f18094f0d940f7bb70fcdcbd50b2bd3 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Fri, 5 Nov 2021 11:01:18 +0100 Subject: [PATCH 051/210] Add pre-commit config. This patch adds `clang-format` as only linter for now. This replaces the previously used script from `auxil/run-clang-format` which we remove. This requires the Python program `pre-commit` (https://pypi.org/project/pre-commit/). With that one can then run `clang-format` on the whole codebase with $ pre-commit run -a clang-format or on just the staged files # Explicitly selecting linter. $ pre-commit run clang-format # Run all linters (currently just `clang-format`). $ pre-commit `pre-commit` supports managing Git commit hooks so that linters are run on commit. Linters can be installed with $ pre-commit install The documentation at https://pre-commit.com/ covers these topics in addition to more information. --- .clang-format | 7 +---- .clang-format-ignore | 2 -- .gitmodules | 3 -- .pre-commit-config.yaml | 8 +++++ auxil/run-clang-format | 1 - ci/run-clang-format.sh | 65 ----------------------------------------- 6 files changed, 9 insertions(+), 77 deletions(-) delete mode 100644 .clang-format-ignore create mode 100644 .pre-commit-config.yaml delete mode 160000 auxil/run-clang-format delete mode 100755 ci/run-clang-format.sh diff --git a/.clang-format b/.clang-format index a105b7df33..4c628b3465 100644 --- a/.clang-format +++ b/.clang-format @@ -1,10 +1,5 @@ # Clang-format configuration for Zeek. This configuration requires # at least clang-format 12.0.1 to format correctly. -# -# The easiest way to run this from the command-line is using the -# python script in auxil/run-clang-format: -# -# python3 auxil/run-clang-format/run-clang-format.py --clang-format-executable /path/to/clang-format -r src -i Language: Cpp Standard: c++17 @@ -102,4 +97,4 @@ IncludeCategories: - Regex: '^"zeek/' Priority: 4 - Regex: '.*' - Priority: 5 \ No newline at end of file + Priority: 5 diff --git a/.clang-format-ignore b/.clang-format-ignore deleted file mode 100644 index bb01885b01..0000000000 --- a/.clang-format-ignore +++ /dev/null @@ -1,2 +0,0 @@ -# Ignore everything 3rdparty -src/3rdparty/* diff --git a/.gitmodules b/.gitmodules index e3b149e9f9..4318212fe0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -49,6 +49,3 @@ [submodule "auxil/zeek-client"] path = auxil/zeek-client url = https://github.com/zeek/zeek-client -[submodule "auxil/run-clang-format"] - path = auxil/run-clang-format - url = https://github.com/Sarcasm/run-clang-format diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..f3f52012a1 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,8 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +# +repos: +- repo: https://github.com/pre-commit/mirrors-clang-format + rev: 'v13.0.0' + hooks: + - id: clang-format diff --git a/auxil/run-clang-format b/auxil/run-clang-format deleted file mode 160000 index 39081c9c42..0000000000 --- a/auxil/run-clang-format +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 39081c9c42768ab5e8321127a7494ad1647c6a2f diff --git a/ci/run-clang-format.sh b/ci/run-clang-format.sh deleted file mode 100755 index 3bd6d5d898..0000000000 --- a/ci/run-clang-format.sh +++ /dev/null @@ -1,65 +0,0 @@ -#! /bin/sh -# -# Copyright (c) 2020 by the Zeek Project. See LICENSE for details. - -base=$(git rev-parse --show-toplevel) -fix=0 -pre_commit_hook=0 - -# Directories to run on by default. When changing, adapt .pre-commit-config.yam -# as well. -files="src" - -error() { - test "${pre_commit_hook}" = 0 && echo "$@" >&2 && exit 1 - exit 0 -} - -if [ $# != 0 ]; then - case "$1" in - --fixit) - shift - fix=1 - ;; - - --pre-commit-hook) - shift - fix=1 - pre_commit_hook=1 - ;; - - -*) - echo "usage: $(basename $0) [--fixit | --pre-commit-hook] []" - exit 1 - esac -fi - -test $# != 0 && files="$@" - -if [ -z "${CLANG_FORMAT}" ]; then - CLANG_FORMAT=$(which clang-format 2>/dev/null) -fi - -if [ -z "${CLANG_FORMAT}" -o ! -x "${CLANG_FORMAT}" ]; then - error "Cannot find clang-format. If not in PATH, set CLANG_FORMAT." -fi - -if ! (cd / && ${CLANG_FORMAT} -dump-config | grep -q SpacesInConditionalStatement); then - error "${CLANG_FORMAT} does not support SpacesInConditionalStatement. Install custom version and put it into PATH, or point CLANG_FORMAT to it." -fi - -if [ ! -e .clang-format ]; then - error "Must execute in top-level directory." -fi - -cmd="${base}/auxil/run-clang-format/run-clang-format.py -r --clang-format-executable ${CLANG_FORMAT} --exclude '*/3rdparty/*' ${files}" -tmp=/tmp/$(basename $0).$$.tmp -trap "rm -f ${tmp}" EXIT -eval "${cmd}" >"${tmp}" - -if [ "${fix}" = 1 ]; then - test -s "${tmp}" && cat "${tmp}" | git apply -p0 - true -else - cat "${tmp}" -fi From 84e8000e62a8240f99d3ab22ef3821d74246ecda Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Fri, 5 Nov 2021 11:12:40 +0100 Subject: [PATCH 052/210] Add Github action exercising pre-commit This patch adds a Github action which exercises pre-commit linters for commits to the `master` branch or for pull requests. We adds this task as a Github action since we expect it to finish quickly; running outside of Cirrus makes it possible provide feedback quickly. --- .github/workflows/pre-commit.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/workflows/pre-commit.yml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000000..925b8accd7 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,14 @@ +name: pre-commit + +on: + pull_request: + push: + branches: [master] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - uses: pre-commit/action@v2.0.3 From a011b4cb702b425497331aab9ad66bbbb88d8621 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 9 Nov 2021 15:11:27 +0000 Subject: [PATCH 053/210] Packets with TSO: address review feedback. This addresses review feedback of GH-1831 and additionally fixes one case in which PayloadLen was used in a way that would have given problematic results when TSO is enabled. --- src/IP.cc | 8 +++++++- src/IP.h | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/IP.cc b/src/IP.cc index 04ac1b6676..330f2b5a85 100644 --- a/src/IP.cc +++ b/src/IP.cc @@ -384,7 +384,13 @@ RecordValPtr IP_Hdr::ToPktHdrVal(RecordValPtr pkt_hdr, int sindex) const auto tcp_hdr = make_intrusive(tcp_hdr_type); int tcp_hdr_len = tp->th_off * 4; - int data_len = PayloadLen() - tcp_hdr_len; + + // account for cases in which the payload length in the TCP header is not set, + // or is set to an impossible value. In these cases, return 0. + int data_len = 0; + auto payload_len = PayloadLen(); + if ( payload_len >= tcp_hdr_len ) + data_len = payload_len - tcp_hdr_len; tcp_hdr->Assign(0, val_mgr->Port(ntohs(tp->th_sport), TRANSPORT_TCP)); tcp_hdr->Assign(1, val_mgr->Port(ntohs(tp->th_dport), TRANSPORT_TCP)); diff --git a/src/IP.h b/src/IP.h index 2b670cf72e..0488290b00 100644 --- a/src/IP.h +++ b/src/IP.h @@ -411,6 +411,9 @@ public: /** * Returns the length of the IP packet's payload (length of packet minus * header length or, for IPv6, also minus length of all extension headers). + * + * Also returns 0 if the IPv4 length field is set to zero - which is, e.g., + * the case when TCP segment offloading is enabled. */ uint16_t PayloadLen() const { From 65bc817fcf36b173a7ba9473ad556c7e53da24e3 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Wed, 10 Nov 2021 09:10:49 +0100 Subject: [PATCH 054/210] Install include headers from `src/3rdparty/`. This is a fixup commit for 72cbc7cd13b7c1bda98658104431c3b530ff68d6 where we move some header files from `src/` to `src/3rdparty/` but missed adding install rules for these header. Since some of these headers are exposed in installed headers they need to be installed as well. --- src/CMakeLists.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9f7eaffbab..ebc5385094 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -620,6 +620,13 @@ install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ ) install(FILES + ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/ConvertUTF.h + ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/bro_inet_ntop.h + ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/bsd-getopt-long.h + ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/modp_numtoa.h + ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/nb_dns.h + ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/patricia.h + ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/setsignal.h ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/sqlite3.h DESTINATION include/zeek/3rdparty ) From 64e34b52aaad860c7437aab765e85310dafb3f5a Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 10 Nov 2021 09:39:16 +0100 Subject: [PATCH 055/210] Tweaking a couple of debug message. As suggested during review. --- src/PolicyFile.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/PolicyFile.cc b/src/PolicyFile.cc index eaee5a9916..c1bd4eac42 100644 --- a/src/PolicyFile.cc +++ b/src/PolicyFile.cc @@ -49,7 +49,7 @@ int how_many_lines_in(const char* policy_filename) FILE* throwaway = fopen(policy_filename, "r"); if ( ! throwaway ) { - debug_msg("No such policy file: %s.\n", policy_filename); + debug_msg("Could not open policy file: %s.\n", policy_filename); return -1; } @@ -97,7 +97,7 @@ bool LoadPolicyFileText(const char* policy_filename, if ( ! f ) { - debug_msg("No such policy file: %s.\n", policy_filename); + debug_msg("Could not open policy file: %s.\n", policy_filename); return false; } @@ -152,7 +152,7 @@ bool PrintLines(const char* policy_filename, unsigned int start_line, unsigned i FILE* throwaway = fopen(policy_filename, "r"); if ( ! throwaway ) { - debug_msg("No such policy file: %s.\n", policy_filename); + debug_msg("Could not open policy file: %s.\n", policy_filename); return false; } From 82b3606cdde60c14999ab5902871a883d4af086f Mon Sep 17 00:00:00 2001 From: Abdel Date: Sun, 31 Oct 2021 00:22:38 +0200 Subject: [PATCH 056/210] case-insensitive-search-features-for-do_find_str --- src/strings.bif | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/strings.bif b/src/strings.bif index 0d3812f267..f4d20dc2cb 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -1145,7 +1145,7 @@ function count_substr%(str: string, sub: string%) : count %%{ -static int64_t do_find_str(zeek::StringVal* str, zeek::StringVal* sub, int64_t start, int64_t end, bool rfind) +static int64_t do_find_str(zeek::StringVal* str, zeek::StringVal* sub, int64_t start, int64_t end, bool rfind, bool case_sensitive) { // Don't bother if the start is after the end of the string. if ( start > str->Len() ) @@ -1168,11 +1168,19 @@ static int64_t do_find_str(zeek::StringVal* str, zeek::StringVal* sub, int64_t s return -1; string s = str->ToStdString().substr(start, end_pos); + string sb = sub->ToStdString(); size_t pos = string::npos; + + if ( !case_sensitive ) + { + transform(s.begin(), s.end(), s.begin(), ::tolower); + transform(sb.begin(), sb.end(), sb.begin(), ::tolower); + } + if ( rfind ) - pos = s.rfind(sub->ToStdString()); + pos = s.rfind(sb); else - pos = s.find(sub->ToStdString()); + pos = s.find(sb); if ( pos == string::npos ) return -1; @@ -1193,13 +1201,14 @@ static int64_t do_find_str(zeek::StringVal* str, zeek::StringVal* sub, int64_t s ## end: An optional position for the end of the substring. A value less than ## zero (such as the default -1) means a search until the end of the ## string. +## case_sensitive: An optional position for search case insensitively ## ## Returns: The position of the substring. Returns -1 if the string wasn't ## found. Prints an error if the starting position is after the ending ## position. -function find_str%(str: string, sub: string, start: count &default=0, end: int &default=-1%) : int +function find_str%(str: string, sub: string, start: count &default=0, end: int &default=-1, case_sensitive: bool &default=T%) : int %{ - return zeek::val_mgr->Int(do_find_str(str, sub, start, end, false)); + return zeek::val_mgr->Int(do_find_str(str, sub, start, end, false, case_sensitive)); %} ## The same as :zeek:see:`find_str`, but returns the highest index matching @@ -1214,9 +1223,9 @@ function find_str%(str: string, sub: string, start: count &default=0, end: int & ## Returns: The position of the substring. Returns -1 if the string wasn't ## found. Prints an error if the starting position is after the ending ## position. -function rfind_str%(str: string, sub: string, start: count &default=0, end: int &default=-1%) : int +function rfind_str%(str: string, sub: string, start: count &default=0, end: int &default=-1, case_sensitive: bool &default=T%) : int %{ - return zeek::val_mgr->Int(do_find_str(str, sub, start, end, true)); + return zeek::val_mgr->Int(do_find_str(str, sub, start, end, true, case_sensitive)); %} ## Returns whether a string starts with a substring. From effa8c403f61e1700ceada4da82e2cb9202da34c Mon Sep 17 00:00:00 2001 From: Abdel Date: Thu, 4 Nov 2021 02:02:48 +0100 Subject: [PATCH 057/210] testing-do-find-str_case-insensitive --- testing/btest/bifs/do_find_str.zeek | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 testing/btest/bifs/do_find_str.zeek diff --git a/testing/btest/bifs/do_find_str.zeek b/testing/btest/bifs/do_find_str.zeek new file mode 100644 index 0000000000..b598b5c73a --- /dev/null +++ b/testing/btest/bifs/do_find_str.zeek @@ -0,0 +1,22 @@ +# +# @TEST-EXEC: zeek -b %INPUT >out +# @TEST-EXEC: btest-diff out + +event zeek_init() + { + local a = "this is the concatenation of HTTP fields of the fOrM of the website that I am protecting"; + local b = "form"; + local c = "FORM"; + local d = "FoRm"; + local e = "om0"; + local f = "f0rm"; + local g = "fOrm"; + + + print find_str(a, b, 0, -1, F); + print find_str(a, c, 0, -1, F); + print find_str(a, d, 0, -1, F); + print find_str(a, e, 0, -1, F); + print find_str(a, f, 0, -1, F); + print find_str(a, g, 0, -1, F); + } From a6f18d514d14cdb87c50812eddb25e10633acf0b Mon Sep 17 00:00:00 2001 From: zeek-bot Date: Thu, 11 Nov 2021 00:43:13 +0000 Subject: [PATCH 058/210] Update doc submodule [nomail] [skip ci] --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index 6bcc7c3e12..207e7c225b 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 6bcc7c3e125e0bef4352d44a57ded9e9e9fc3775 +Subproject commit 207e7c225bb1ee7e7404341e416544ac18c9a016 From 362566369f90a920711d7e88d8b421b981b72211 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 9 Nov 2021 13:07:14 -0700 Subject: [PATCH 059/210] Update libkqueue submodule --- auxil/libkqueue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auxil/libkqueue b/auxil/libkqueue index 6c1717dea2..aeaeed2119 160000 --- a/auxil/libkqueue +++ b/auxil/libkqueue @@ -1 +1 @@ -Subproject commit 6c1717dea2dc34a91d32e07d2cae34b1afa0a84e +Subproject commit aeaeed21198d6f41d0cf70bda63fe0f424922ac5 From 8fece3d8ea3e373d19721b6c6a8f9ab03402eb6d Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Thu, 11 Nov 2021 14:35:43 -0800 Subject: [PATCH 060/210] Update cmake submodule [nomail] [skip ci] --- cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake b/cmake index f5f931637e..654400c9ff 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit f5f931637eeabd92e672fa63fae4016a9cd547d7 +Subproject commit 654400c9ffed760140f594603737c5e8750306ff From fe932944c41ab108ff74a8af0fd9b574bac4db83 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 4 Nov 2021 14:59:16 -0700 Subject: [PATCH 061/210] GH-1620: Add event and plugin hook to track packets not processed --- NEWS | 9 +++++ src/RunState.cc | 12 +++++- src/event.bif | 8 ++++ src/iosource/Packet.cc | 29 ++++++++++++++ src/iosource/Packet.h | 14 +++++++ src/packet_analysis/Manager.cc | 11 ++++++ src/packet_analysis/Manager.h | 9 +++++ src/packet_analysis/protocol/arp/ARP.cc | 5 +++ .../protocol/ip/IPBasedAnalyzer.cc | 4 ++ src/plugin/Manager.cc | 23 +++++++++++ src/plugin/Manager.h | 9 +++++ src/plugin/Plugin.cc | 7 ++++ src/plugin/Plugin.h | 39 +++++++++++++++++-- src/zeek.bif | 23 ++--------- .../btest/plugins/hooks-plugin/src/Plugin.cc | 16 ++++++++ .../btest/plugins/hooks-plugin/src/Plugin.h | 1 + 16 files changed, 194 insertions(+), 25 deletions(-) diff --git a/NEWS b/NEWS index d82680803e..ca92bc610e 100644 --- a/NEWS +++ b/NEWS @@ -40,6 +40,15 @@ New Functionality - The ``find_str`` and ``rfind_str`` bifs now support case-insensitive searches. +- Added a new plugin hook for capturing packets that made it through analysis + without being processed called ``Plugin::HookUnprocessedPacket``. Currently + ARP packets or packets with a valid IP-based transport header are marked as + processed. This also adds an event called ``packet_not_processed`` that + reports the same packets. + +- A new command-line option ``-c`` or ``--capture-unprocessed`` will dump any + packets not marked as being processed, similar to the new hook and event above. + Changed Functionality --------------------- diff --git a/src/RunState.cc b/src/RunState.cc index 2161e7ba30..3a8e367714 100644 --- a/src/RunState.cc +++ b/src/RunState.cc @@ -390,8 +390,16 @@ void get_final_stats() ? ((double)s.dropped / ((double)s.received + (double)s.dropped)) * 100.0 : 0.0; - reporter->Info("%" PRIu64 " packets received on interface %s, %" PRIu64 " (%.2f%%) dropped", - s.received, ps->Path().c_str(), s.dropped, dropped_pct); + + uint64_t not_processed = packet_mgr->GetUnprocessedCount(); + double unprocessed_pct = not_processed > 0 + ? ((double)not_processed / (double)s.received) * 100.0 + : 0.0; + + reporter->Info("%" PRIu64 " packets received on interface %s, %" PRIu64 + " (%.2f%%) dropped, %" PRIu64 " (%.2f%%) not processed", + s.received, ps->Path().c_str(), s.dropped, dropped_pct, not_processed, + unprocessed_pct); } } diff --git a/src/event.bif b/src/event.bif index 18ccc7dda9..135cb10680 100644 --- a/src/event.bif +++ b/src/event.bif @@ -907,3 +907,11 @@ event Pcap::file_done%(path: string%); ## ## .. zeek:see:: UnknownProtocol::first_bytes_count event unknown_protocol%(analyzer_name: string, protocol: count, first_bytes: string%); + +## An event for handling packets that reached the end of processing without +## being marked as processed. Note that this event may lead to unpredictable +## performance spikes, particularly if a network suddenly receives a burst +## of packets that are unprocessed. +## +## pkt: Data for the unprocessed packet +event packet_not_processed%(pkt: pcap_packet%); diff --git a/src/iosource/Packet.cc b/src/iosource/Packet.cc index 744b289ad2..78b0684650 100644 --- a/src/iosource/Packet.cc +++ b/src/iosource/Packet.cc @@ -68,6 +68,8 @@ void Packet::Init(int arg_link_type, pkt_timeval* arg_ts, uint32_t arg_caplen, u tunnel_type = BifEnum::Tunnel::IP; gre_version = -1; gre_link_type = DLT_RAW; + + processed = false; } Packet::~Packet() @@ -146,6 +148,33 @@ RecordValPtr Packet::ToRawPktHdrVal() const return pkt_hdr; } +RecordValPtr Packet::ToVal(const Packet* p) + { + static auto pcap_packet = zeek::id::find_type("pcap_packet"); + auto val = zeek::make_intrusive(pcap_packet); + + if ( p ) + { + val->Assign(0, static_cast(p->ts.tv_sec)); + val->Assign(1, static_cast(p->ts.tv_usec)); + val->Assign(2, p->cap_len); + val->Assign(3, p->len); + val->Assign(4, zeek::make_intrusive(p->cap_len, (const char*)p->data)); + val->Assign(5, zeek::BifType::Enum::link_encap->GetEnumVal(p->link_type)); + } + else + { + val->Assign(0, 0); + val->Assign(1, 0); + val->Assign(2, 0); + val->Assign(3, 0); + val->Assign(4, zeek::val_mgr->EmptyString()); + val->Assign(5, zeek::BifType::Enum::link_encap->GetEnumVal(BifEnum::LINK_UNKNOWN)); + } + + return val; + } + ValPtr Packet::FmtEUI48(const u_char* mac) const { char buf[20]; diff --git a/src/iosource/Packet.h b/src/iosource/Packet.h index 3215810de0..8c61abea3d 100644 --- a/src/iosource/Packet.h +++ b/src/iosource/Packet.h @@ -122,6 +122,12 @@ public: */ RecordValPtr ToRawPktHdrVal() const; + /** + * Returns a RecordVal that represents the Packet. This is used + * by the get_current_packet bif. + */ + static RecordValPtr ToVal(const Packet* p); + /** * Maximal length of a layer 2 address. */ @@ -241,6 +247,14 @@ public: */ int gre_link_type = DLT_RAW; + /** + * This flag indicates whether a packet has been processed. This can + * mean different things depending on the traffic, but generally it + * means that a packet has been logged in some way. We default to + * false, and this can be set to true for any number of reasons. + */ + bool processed = false; + private: // Renders an MAC address into its ASCII representation. ValPtr FmtEUI48(const u_char* mac) const; diff --git a/src/packet_analysis/Manager.cc b/src/packet_analysis/Manager.cc index 5cd5cf0136..e0c6f2dca6 100644 --- a/src/packet_analysis/Manager.cc +++ b/src/packet_analysis/Manager.cc @@ -7,6 +7,7 @@ #include "zeek/iosource/PktDumper.h" #include "zeek/packet_analysis/Analyzer.h" #include "zeek/packet_analysis/Dispatcher.h" +#include "zeek/plugin/Manager.h" #include "zeek/zeek-bif.h" using namespace zeek::packet_analysis; @@ -106,6 +107,16 @@ void Manager::ProcessPacket(Packet* packet) // Start packet analysis root_analyzer->ForwardPacket(packet->cap_len, packet->data, packet, packet->link_type); + if ( ! packet->processed ) + { + if ( packet_not_processed ) + event_mgr.Enqueue(packet_not_processed, Packet::ToVal(packet)); + + plugin_mgr->HookUnprocessedPacket(packet); + + total_not_processed++; + } + if ( raw_packet ) event_mgr.Enqueue(raw_packet, packet->ToRawPktHdrVal()); diff --git a/src/packet_analysis/Manager.h b/src/packet_analysis/Manager.h index db813cb492..806221df77 100644 --- a/src/packet_analysis/Manager.h +++ b/src/packet_analysis/Manager.h @@ -2,6 +2,7 @@ #pragma once +#include "zeek/Func.h" #include "zeek/PacketFilter.h" #include "zeek/iosource/Packet.h" #include "zeek/packet_analysis/Component.h" @@ -126,6 +127,12 @@ public: return pkt_filter; } + /** + * Returns the total number of packets received that weren't considered + * processed by some analyzer. + */ + uint64_t GetUnprocessedCount() const { return total_not_processed; } + private: /** * Instantiates a new analyzer instance. @@ -163,6 +170,8 @@ private: uint64_t unknown_sampling_rate = 0; double unknown_sampling_duration = 0; uint64_t unknown_first_bytes_count = 0; + + uint64_t total_not_processed = 0; }; } // namespace packet_analysis diff --git a/src/packet_analysis/protocol/arp/ARP.cc b/src/packet_analysis/protocol/arp/ARP.cc index ce70264058..89e98af66e 100644 --- a/src/packet_analysis/protocol/arp/ARP.cc +++ b/src/packet_analysis/protocol/arp/ARP.cc @@ -101,6 +101,11 @@ bool ARPAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* packet) return false; } + // ARP packets are considered processed if we get to this point. There may be issues + // with the processing of them, but they're actually an ARP packet and anything else + // will be reported via events. + packet->processed = true; + // Check the address description fields. switch ( ntohs(ah->ar_hrd) ) { diff --git a/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc b/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc index 785d914498..16e411c306 100644 --- a/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc +++ b/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc @@ -63,6 +63,10 @@ bool IPBasedAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* pkt if ( ! conn ) return false; + // If we successfuly made a connection for this packet that means it'll eventually + // get logged, which means we can mark this packet as having been processed. + pkt->processed = true; + bool is_orig = (tuple.src_addr == conn->OrigAddr()) && (tuple.src_port == conn->OrigPort()); conn->CheckFlowLabel(is_orig, ip_hdr->FlowLabel()); diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index 8215c5c56c..7270191dbf 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -1012,6 +1012,29 @@ bool Manager::HookReporter(const std::string& prefix, const EventHandlerPtr even return result; } +void Manager::HookUnprocessedPacket(const Packet* packet) const + { + HookArgumentList args; + + if ( HavePluginForHook(META_HOOK_PRE) ) + { + args.emplace_back(HookArgument{packet}); + MetaHookPre(HOOK_UNPROCESSED_PACKET, args); + } + + hook_list* l = hooks[HOOK_UNPROCESSED_PACKET]; + + if ( l ) + for ( hook_list::iterator i = l->begin(); i != l->end(); ++i ) + { + Plugin* p = (*i).second; + p->HookUnprocessedPacket(packet); + } + + if ( HavePluginForHook(META_HOOK_POST) ) + MetaHookPost(HOOK_UNPROCESSED_PACKET, args, HookArgument()); + } + void Manager::MetaHookPre(HookType hook, const HookArgumentList& args) const { if ( hook_list* l = hooks[HOOK_CALL_FUNCTION] ) diff --git a/src/plugin/Manager.h b/src/plugin/Manager.h index 4873a271ba..f2324c2b44 100644 --- a/src/plugin/Manager.h +++ b/src/plugin/Manager.h @@ -419,6 +419,15 @@ public: const zeek::detail::Location* location2, bool time, const std::string& message); + /** + * Hook for packets that are considered unprocessed by an Analyzer. This + * typically means that a packet has not had a log entry written for it by + * the time analysis finishes. + * + * @param packet The data for an unprocessed packet + */ + void HookUnprocessedPacket(const Packet* packet) const; + /** * Internal method that registers a freshly instantiated plugin with * the manager. diff --git a/src/plugin/Plugin.cc b/src/plugin/Plugin.cc index 315a844a3b..c7ef7e4f8f 100644 --- a/src/plugin/Plugin.cc +++ b/src/plugin/Plugin.cc @@ -31,6 +31,7 @@ const char* hook_name(HookType h) "SetupAnalyzerTree", "LogInit", "LogWrite", + "UnprocessedPacket", // MetaHooks "MetaHookPre", "MetaHookPost", @@ -244,7 +245,11 @@ void HookArgument::Describe(ODesc* d) const d->Add(""); d->Add(")"); + break; } + + case PACKET: + d->Add(""); break; } } @@ -432,6 +437,8 @@ bool Plugin::HookReporter(const std::string& prefix, const EventHandlerPtr event return true; } +void Plugin::HookUnprocessedPacket(const Packet* packet) { } + void Plugin::MetaHookPre(HookType hook, const HookArgumentList& args) { } void Plugin::MetaHookPost(HookType hook, const HookArgumentList& args, HookArgument result) { } diff --git a/src/plugin/Plugin.h b/src/plugin/Plugin.h index f625fc1a49..8f2e75e64b 100644 --- a/src/plugin/Plugin.h +++ b/src/plugin/Plugin.h @@ -30,6 +30,7 @@ class ODesc; class Event; class Func; class Obj; +class Packet; template class IntrusivePtr; using ValPtr = IntrusivePtr; @@ -68,6 +69,7 @@ enum HookType HOOK_LOG_INIT, //< Activates Plugin::HookLogInit HOOK_LOG_WRITE, //< Activates Plugin::HookLogWrite HOOK_REPORTER, //< Activates Plugin::HookReporter + HOOK_UNPROCESSED_PACKET, //("pcap_packet"); - const Packet* p; - auto pkt = zeek::make_intrusive(pcap_packet); + const Packet* p = nullptr; zeek::iosource::PktSrc* pkt_src = zeek::run_state::detail::current_packet_source(); if ( ! pkt_src || ! pkt_src->GetCurrentPacket(&p) ) - { - pkt->Assign(0, 0); - pkt->Assign(1, 0); - pkt->Assign(2, 0); - pkt->Assign(3, 0); - pkt->Assign(4, zeek::val_mgr->EmptyString()); - pkt->Assign(5, zeek::BifType::Enum::link_encap->GetEnumVal(BifEnum::LINK_UNKNOWN)); - return pkt; - } + p = nullptr; - pkt->Assign(0, static_cast(p->ts.tv_sec)); - pkt->Assign(1, static_cast(p->ts.tv_usec)); - pkt->Assign(2, p->cap_len); - pkt->Assign(3, p->len); - pkt->Assign(4, zeek::make_intrusive(p->cap_len, (const char*)p->data)); - pkt->Assign(5, zeek::BifType::Enum::link_encap->GetEnumVal(p->link_type)); - - return pkt; + return Packet::ToVal(p); %} ## Function to get the raw headers of the currently processed packet. diff --git a/testing/btest/plugins/hooks-plugin/src/Plugin.cc b/testing/btest/plugins/hooks-plugin/src/Plugin.cc index 1fd4054f38..05123fb341 100644 --- a/testing/btest/plugins/hooks-plugin/src/Plugin.cc +++ b/testing/btest/plugins/hooks-plugin/src/Plugin.cc @@ -26,6 +26,7 @@ zeek::plugin::Configuration Plugin::Configure() EnableHook(zeek::plugin::HOOK_SETUP_ANALYZER_TREE); EnableHook(zeek::plugin::HOOK_LOG_INIT); EnableHook(zeek::plugin::HOOK_LOG_WRITE); + EnableHook(zeek::plugin::HOOK_UNPROCESSED_PACKET); zeek::plugin::Configuration config; config.name = "Demo::Hooks"; @@ -272,3 +273,18 @@ bool Plugin::HookLogWrite(const std::string& writer, const std::string& filter, fprintf(stderr, "%.6f %-15s %s %s\n", zeek::run_state::network_time, "| HookLogWrite", info.path, d.Description()); return true; } + +void Plugin::HookUnprocessedPacket(const zeek::Packet* packet) + { + zeek::ODesc d; + d.Add("["); + d.Add("ts="); + d.Add(packet->time); + d.Add(" len="); + d.Add(packet->len); + d.Add(" header="); + d.AddN(reinterpret_cast(packet->data), 10); + d.Add("]"); + + fprintf(stderr, "%.6f %-23s %s\n", zeek::run_state::network_time, "| HookUnprocessedPacket", d.Description()); + } diff --git a/testing/btest/plugins/hooks-plugin/src/Plugin.h b/testing/btest/plugins/hooks-plugin/src/Plugin.h index b721608a6f..22b7ada9a7 100644 --- a/testing/btest/plugins/hooks-plugin/src/Plugin.h +++ b/testing/btest/plugins/hooks-plugin/src/Plugin.h @@ -24,6 +24,7 @@ protected: int num_fields, const zeek::threading::Field* const* fields, zeek::threading::Value** vals) override; void HookSetupAnalyzerTree(zeek::Connection *conn) override; + void HookUnprocessedPacket(const zeek::Packet* packet) override; void MetaHookPre(zeek::plugin::HookType hook, const zeek::plugin::HookArgumentList& args) override; void MetaHookPost(zeek::plugin::HookType hook, const zeek::plugin::HookArgumentList& args, zeek::plugin::HookArgument result) override; From 92b84a00f998be4a2c42dd505e5666ee77309f17 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 8 Nov 2021 11:04:40 -0700 Subject: [PATCH 062/210] Add command-line option to write unprocessed packets to a file This commit also changes the PcapDumper to automatically flush after every called to Dump(). This is because pcap_dump has an internal buffer of some sort that only writes to the file after a set amount of bytes. When using the new option on a low-traffic network, it might be a while before you see any packets written since it has to overcome that buffer limit first. --- src/Options.cc | 18 ++++++++++++------ src/Options.h | 1 + src/iosource/pcap/Dumper.cc | 1 + src/packet_analysis/Manager.cc | 10 +++++++++- src/packet_analysis/Manager.h | 12 +++++++++++- src/zeek-setup.cc | 2 +- 6 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/Options.cc b/src/Options.cc index 374ae84efe..559061a74c 100644 --- a/src/Options.cc +++ b/src/Options.cc @@ -90,6 +90,8 @@ void usage(const char* prog, int code) " -a|--parse-only | exit immediately after parsing scripts\n"); fprintf(stderr, " -b|--bare-mode | don't load scripts from the base/ directory\n"); + fprintf(stderr, + " -c|--capture-unprocessed | write unprocessed packets to a tcpdump file\n"); fprintf(stderr, " -d|--debug-script | activate Zeek script debugging\n"); fprintf(stderr, " -e|--exec | augment loaded scripts by given code\n"); fprintf(stderr, " -f|--filter | tcpdump filter\n"); @@ -103,8 +105,9 @@ void usage(const char* prog, int code) "allowed, pass '-' as the filename to read from stdin)\n"); fprintf(stderr, " -s|--rulefile | read rules from given file\n"); fprintf(stderr, " -t|--tracefile | activate execution tracing\n"); - fprintf(stderr, " -u|--usage-issues | find variable usage issues and exit; use " - "-uu for deeper/more expensive analysis\n"); + fprintf(stderr, + " -u|--usage-issues | find variable usage issues and exit; use " + "-uu for deeper/more expensive analysis\n"); fprintf(stderr, " -v|--version | print version and exit\n"); fprintf(stderr, " -w|--writefile | write to given tcpdump file\n"); #ifdef DEBUG @@ -165,9 +168,8 @@ void usage(const char* prog, int code) getenv("ZEEK_DNS_RESOLVER") ? getenv("ZEEK_DNS_RESOLVER") : "not set, will use first IPv4 address from /etc/resolv.conf"); - fprintf( - stderr, - " $ZEEK_DEBUG_LOG_STDERR | Use stderr for debug logs generated via the -B flag"); + fprintf(stderr, " $ZEEK_DEBUG_LOG_STDERR | Use stderr for debug logs generated via " + "the -B flag"); fprintf(stderr, "\n"); @@ -362,6 +364,7 @@ Options parse_cmdline(int argc, char** argv) constexpr struct option long_opts[] = { {"parse-only", no_argument, nullptr, 'a'}, {"bare-mode", no_argument, nullptr, 'b'}, + {"capture-unprocessed", required_argument, nullptr, 'c'}, {"debug-script", no_argument, nullptr, 'd'}, {"exec", required_argument, nullptr, 'e'}, {"filter", required_argument, nullptr, 'f'}, @@ -405,7 +408,7 @@ Options parse_cmdline(int argc, char** argv) }; char opts[256]; - util::safe_strncpy(opts, "B:e:f:G:H:I:i:j::n:O:o:p:r:s:T:t:U:w:X:CDFMNPQSWabdhmuv", + util::safe_strncpy(opts, "B:c:e:f:G:H:I:i:j::n:O:o:p:r:s:T:t:U:w:X:CDFMNPQSWabdhmuv", sizeof(opts)); int op; @@ -428,6 +431,9 @@ Options parse_cmdline(int argc, char** argv) case 'b': rval.bare_mode = true; break; + case 'c': + rval.unprocessed_output_file = optarg; + break; case 'd': rval.debug_scripts = true; break; diff --git a/src/Options.h b/src/Options.h index b19b9ea0b4..b9bd1c5e9b 100644 --- a/src/Options.h +++ b/src/Options.h @@ -72,6 +72,7 @@ struct Options std::optional random_seed_output_file; std::optional process_status_file; std::optional zeekygen_config_file; + std::optional unprocessed_output_file; std::set plugins_to_load; std::vector scripts_to_load; diff --git a/src/iosource/pcap/Dumper.cc b/src/iosource/pcap/Dumper.cc index 3b46c5c15f..3cf7a0dcf7 100644 --- a/src/iosource/pcap/Dumper.cc +++ b/src/iosource/pcap/Dumper.cc @@ -110,6 +110,7 @@ bool PcapDumper::Dump(const Packet* pkt) const struct pcap_pkthdr phdr = {.ts = pkt->ts, .caplen = pkt->cap_len, .len = pkt->len}; pcap_dump((u_char*)dumper, &phdr, pkt->data); + pcap_dump_flush(dumper); return true; } diff --git a/src/packet_analysis/Manager.cc b/src/packet_analysis/Manager.cc index e0c6f2dca6..dc53ce4bbc 100644 --- a/src/packet_analysis/Manager.cc +++ b/src/packet_analysis/Manager.cc @@ -4,6 +4,7 @@ #include "zeek/RunState.h" #include "zeek/Stats.h" +#include "zeek/iosource/Manager.h" #include "zeek/iosource/PktDumper.h" #include "zeek/packet_analysis/Analyzer.h" #include "zeek/packet_analysis/Dispatcher.h" @@ -24,7 +25,7 @@ Manager::~Manager() delete pkt_filter; } -void Manager::InitPostScript() +void Manager::InitPostScript(const std::string& unprocessed_output_file) { // Instantiate objects for all available analyzers for ( const auto& analyzerComponent : GetComponents() ) @@ -49,6 +50,10 @@ void Manager::InitPostScript() unknown_sampling_threshold = id::find_val("UnknownProtocol::sampling_threshold")->AsCount(); unknown_sampling_duration = id::find_val("UnknownProtocol::sampling_duration")->AsInterval(); unknown_first_bytes_count = id::find_val("UnknownProtocol::first_bytes_count")->AsCount(); + + if ( ! unprocessed_output_file.empty() ) + // This gets automatically cleaned up by iosource_mgr. No need to delete it locally. + unprocessed_dumper = iosource_mgr->OpenPktDumper(unprocessed_output_file, true); } void Manager::Done() { } @@ -114,6 +119,9 @@ void Manager::ProcessPacket(Packet* packet) plugin_mgr->HookUnprocessedPacket(packet); + if ( unprocessed_dumper ) + unprocessed_dumper->Dump(packet); + total_not_processed++; } diff --git a/src/packet_analysis/Manager.h b/src/packet_analysis/Manager.h index 806221df77..ecb659db23 100644 --- a/src/packet_analysis/Manager.h +++ b/src/packet_analysis/Manager.h @@ -18,6 +18,11 @@ namespace detail class PacketProfiler; } +namespace iosource + { +class PktDumper; + } + namespace packet_analysis { @@ -40,8 +45,12 @@ public: /** * Second-stage initialization of the manager. This is called late * during Zeek's initialization after any scripts are processed. + * + * @param unprocessed_output_file A path to a file where unprocessed + * packets will be written. This can be an empty string to disable + * writing packets. */ - void InitPostScript(); + void InitPostScript(const std::string& unprocessed_output_file); /** * Finished the manager's operations. @@ -172,6 +181,7 @@ private: uint64_t unknown_first_bytes_count = 0; uint64_t total_not_processed = 0; + iosource::PktDumper* unprocessed_dumper = nullptr; }; } // namespace packet_analysis diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index 25d2c439f6..5b2c6de914 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -708,7 +708,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) exit(success ? 0 : 1); } - packet_mgr->InitPostScript(); + packet_mgr->InitPostScript(options.unprocessed_output_file.value_or("")); analyzer_mgr->InitPostScript(); file_mgr->InitPostScript(); dns_mgr->InitPostScript(); From d0f8c50417c2740ba588cbd0ac038f2905965cdd Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 8 Nov 2021 11:53:54 -0700 Subject: [PATCH 063/210] Fix whitespace in help output --- src/Options.cc | 98 +++++++++++++++++++++++++------------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/src/Options.cc b/src/Options.cc index 559061a74c..3b7224a212 100644 --- a/src/Options.cc +++ b/src/Options.cc @@ -85,90 +85,90 @@ void usage(const char* prog, int code) fprintf(stderr, "usage: %s [options] [file ...]\n", prog); fprintf(stderr, "usage: %s --test [doctest-options] -- [options] [file ...]\n", prog); - fprintf(stderr, " | Zeek script file, or read stdin\n"); + fprintf(stderr, " | Zeek script file, or read stdin\n"); fprintf(stderr, - " -a|--parse-only | exit immediately after parsing scripts\n"); + " -a|--parse-only | exit immediately after parsing scripts\n"); fprintf(stderr, - " -b|--bare-mode | don't load scripts from the base/ directory\n"); + " -b|--bare-mode | don't load scripts from the base/ directory\n"); fprintf(stderr, - " -c|--capture-unprocessed | write unprocessed packets to a tcpdump file\n"); - fprintf(stderr, " -d|--debug-script | activate Zeek script debugging\n"); - fprintf(stderr, " -e|--exec | augment loaded scripts by given code\n"); - fprintf(stderr, " -f|--filter | tcpdump filter\n"); - fprintf(stderr, " -h|--help | command line help\n"); + " -c|--capture-unprocessed | write unprocessed packets to a tcpdump file\n"); + fprintf(stderr, " -d|--debug-script | activate Zeek script debugging\n"); + fprintf(stderr, " -e|--exec | augment loaded scripts by given code\n"); + fprintf(stderr, " -f|--filter | tcpdump filter\n"); + fprintf(stderr, " -h|--help | command line help\n"); fprintf(stderr, - " -i|--iface | read from given interface (only one allowed)\n"); + " -i|--iface | read from given interface (only one allowed)\n"); fprintf( stderr, - " -p|--prefix | add given prefix to Zeek script file resolution\n"); - fprintf(stderr, " -r|--readfile | read from given tcpdump file (only one " + " -p|--prefix | add given prefix to Zeek script file resolution\n"); + fprintf(stderr, " -r|--readfile | read from given tcpdump file (only one " "allowed, pass '-' as the filename to read from stdin)\n"); - fprintf(stderr, " -s|--rulefile | read rules from given file\n"); - fprintf(stderr, " -t|--tracefile | activate execution tracing\n"); + fprintf(stderr, " -s|--rulefile | read rules from given file\n"); + fprintf(stderr, " -t|--tracefile | activate execution tracing\n"); fprintf(stderr, - " -u|--usage-issues | find variable usage issues and exit; use " + " -u|--usage-issues | find variable usage issues and exit; use " "-uu for deeper/more expensive analysis\n"); - fprintf(stderr, " -v|--version | print version and exit\n"); - fprintf(stderr, " -w|--writefile | write to given tcpdump file\n"); + fprintf(stderr, " -v|--version | print version and exit\n"); + fprintf(stderr, " -w|--writefile | write to given tcpdump file\n"); #ifdef DEBUG - fprintf(stderr, " -B|--debug | Enable debugging output for selected " + fprintf(stderr, " -B|--debug | Enable debugging output for selected " "streams ('-B help' for help)\n"); #endif - fprintf(stderr, " -C|--no-checksums | ignore checksums\n"); - fprintf(stderr, " -D|--deterministic | initialize random seeds to zero\n"); - fprintf(stderr, " -F|--force-dns | force DNS\n"); - fprintf(stderr, " -G|--load-seeds | load seeds from given file\n"); - fprintf(stderr, " -H|--save-seeds | save seeds to given file\n"); - fprintf(stderr, " -I|--print-id | print out given ID\n"); - fprintf(stderr, " -N|--print-plugins | print available plugins and exit (-NN " + fprintf(stderr, " -C|--no-checksums | ignore checksums\n"); + fprintf(stderr, " -D|--deterministic | initialize random seeds to zero\n"); + fprintf(stderr, " -F|--force-dns | force DNS\n"); + fprintf(stderr, " -G|--load-seeds | load seeds from given file\n"); + fprintf(stderr, " -H|--save-seeds | save seeds to given file\n"); + fprintf(stderr, " -I|--print-id | print out given ID\n"); + fprintf(stderr, " -N|--print-plugins | print available plugins and exit (-NN " "for verbose)\n"); - fprintf(stderr, " -O|--optimize[=