From ef1546775799fef1778074d90b9c9af51bad8fb3 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Sun, 26 Jan 2020 18:14:03 +0100 Subject: [PATCH 01/53] PacketDumper: remove unused types --- src/PacketDumper.h | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/PacketDumper.h b/src/PacketDumper.h index 78469b3d7c..38a31b920d 100644 --- a/src/PacketDumper.h +++ b/src/PacketDumper.h @@ -2,9 +2,6 @@ #pragma once -#include -#include - #include class PacketDumper { @@ -20,18 +17,3 @@ protected: void SortTimeStamp(struct timeval* timestamp); }; - -struct IP_ID { - uint32_t ip, id; -}; - -struct ltipid { - bool operator()(IP_ID id1, IP_ID id2) const - { - return id1.ip != id2.ip ? (id1.ip < id2.ip) : - (id1.id < id2.id); - } -}; - -typedef std::set IP_IDSet; -uint16_t NextIP_ID(const uint32_t src_addr, const uint16_t id); From 61c3be8e16d0b819c3c7dd545fa5540ee92be225 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Mon, 27 Jan 2020 08:50:54 +0100 Subject: [PATCH 02/53] SerializationFormat: move realloc() call out of the loop Reallocate and copy the data only once. --- src/SerializationFormat.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SerializationFormat.cc b/src/SerializationFormat.cc index 1a572bd923..debb999b26 100644 --- a/src/SerializationFormat.cc +++ b/src/SerializationFormat.cc @@ -81,8 +81,8 @@ bool SerializationFormat::WriteData(const void* b, size_t count) while ( output_pos + count > output_size ) { output_size *= GROWTH_FACTOR; - output = (char*)safe_realloc(output, output_size); } + output = (char*)safe_realloc(output, output_size); memcpy(output + output_pos, b, count); output_pos += count; From 662c3aab5855e21569a862b5016f1b60b3027f52 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Mon, 27 Jan 2020 08:56:47 +0100 Subject: [PATCH 03/53] Desc: move realloc() call out of the loop --- src/Desc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Desc.cc b/src/Desc.cc index 3238248750..6d997ff004 100644 --- a/src/Desc.cc +++ b/src/Desc.cc @@ -378,8 +378,8 @@ void ODesc::Grow(unsigned int n) while ( offset + n + SLOP >= size ) { size *= 2; - base = safe_realloc(base, size); } + base = safe_realloc(base, size); } void ODesc::Clear() From 0b3317b1c23320ee19738015d08d0ab0291c1526 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Fri, 31 Jan 2020 10:17:42 +0100 Subject: [PATCH 04/53] util: optimize expand_escape() by avoiding sscanf() sscanf() is notoriously slow, and the default scripts have lots of hex escapes. This patch reduces Zeek's startup time by 9%. Before: 245.04 msec task-clock:u # 1.002 CPUs utilized 0 context-switches:u # 0.000 K/sec 0 cpu-migrations:u # 0.000 K/sec 16,411 page-faults:u # 0.067 M/sec 629,238,575 cycles:u # 2.568 GHz 1,237,236,556 instructions:u # 1.97 insn per cycle 262,223,957 branches:u # 1070.142 M/sec 3,351,083 branch-misses:u # 1.28% of all branches After: 220.99 msec task-clock:u # 1.002 CPUs utilized 0 context-switches:u # 0.000 K/sec 0 cpu-migrations:u # 0.000 K/sec 16,419 page-faults:u # 0.074 M/sec 544,603,653 cycles:u # 2.464 GHz 1,065,862,324 instructions:u # 1.96 insn per cycle 229,207,957 branches:u # 1037.181 M/sec 3,045,270 branch-misses:u # 1.33% of all branches --- src/util.cc | 73 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/src/util.cc b/src/util.cc index a9b1707297..7a88499fa1 100644 --- a/src/util.cc +++ b/src/util.cc @@ -292,6 +292,26 @@ int streq(const char* s1, const char* s2) return ! strcmp(s1, s2); } +static constexpr int ParseOctDigit(char ch) noexcept + { + if (ch >= '0' && ch <= '7') + return ch - '0'; + else + return -1; + } + +static constexpr int ParseHexDigit(char ch) noexcept + { + if (ch >= '0' && ch <= '9') + return ch - '0'; + else if (ch >= 'a' && ch <= 'f') + return 0xa + ch - 'a'; + else if (ch >= 'A' && ch <= 'F') + return 0xa + ch - 'A'; + else + return -1; + } + int expand_escape(const char*& s) { switch ( *(s++) ) { @@ -309,23 +329,29 @@ int expand_escape(const char*& s) --s; // put back the first octal digit const char* start = s; - // Don't increment inside loop control - // because if isdigit() is a macro it might - // expand into multiple increments ... + // require at least one octal digit and parse at most three - // Here we define a maximum length for escape sequence - // to allow easy handling of string like: "^H0" as - // "\0100". - - for ( int len = 0; len < 3 && isascii(*s) && isdigit(*s); - ++s, ++len) - ; - - int result; - if ( sscanf(start, "%3o", &result) != 1 ) + int result = ParseOctDigit(*s++); + if (result < 0) { - reporter->Warning("bad octal escape: %s ", start); - result = 0; + reporter->Warning("bad octal escape: %s", start); + return 0; + } + + // second digit? + int digit = ParseOctDigit(*s); + if (digit >= 0) + { + result = (result << 3) | digit; + ++s; + + // third digit? + digit = ParseOctDigit(*s); + if (digit >= 0) + { + result = (result << 3) | digit; + ++s; + } } return result; @@ -336,15 +362,20 @@ int expand_escape(const char*& s) const char* start = s; // Look at most 2 characters, so that "\x0ddir" -> "^Mdir". - for ( int len = 0; len < 2 && isascii(*s) && isxdigit(*s); - ++s, ++len) - ; - int result; - if ( sscanf(start, "%2x", &result) != 1 ) + int result = ParseHexDigit(*s++); + if (result < 0) { reporter->Warning("bad hexadecimal escape: %s", start); - result = 0; + return 0; + } + + // second digit? + int digit = ParseHexDigit(*s); + if (digit >= 0) + { + result = (result << 4) | digit; + ++s; } return result; From 800c0c713267b903400837af9d4aeccfc26ad42a Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Thu, 30 Jan 2020 01:39:58 +0100 Subject: [PATCH 05/53] parse.y: Properly set location info for functions When defining a function, remember the location where the function header was and restore it before calling `end_func()`. Inside `end_func()`, a `BroFunc` object is created using the current global location information. This came up while experimenting with zeek script profiling and wondering why the locations set for `BroFunc` were "somewhere" in the middle of functions instead of spanning them. --- src/parse.y | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/parse.y b/src/parse.y index c409942cca..6560707ab3 100644 --- a/src/parse.y +++ b/src/parse.y @@ -128,6 +128,7 @@ bool defining_global_ID = false; std::vector saved_in_init; ID* func_id = 0; +static Location func_hdr_location; EnumType *cur_enum_type = 0; static ID* cur_decl_type_id = 0; @@ -497,6 +498,7 @@ expr: | '$' TOK_ID func_params '=' { + func_hdr_location = @1; func_id = current_scope()->GenerateTemporary("anonymous-function"); func_id->SetInferReturnType(true); begin_func(func_id, @@ -1128,11 +1130,9 @@ decl: zeekygen_mgr->Identifier($2); } - | func_hdr func_body - { } + | func_hdr { func_hdr_location = @1; } func_body - | func_hdr conditional_list func_body - { } + | func_hdr { func_hdr_location = @1; } conditional_list func_body | conditional ; @@ -1208,6 +1208,7 @@ func_body: '}' { + set_location(func_hdr_location, @5); end_func($3); } ; @@ -1229,6 +1230,8 @@ anonymous_function: '}' { + set_location(@1, @7); + // Code duplication here is sad but needed. end_func actually instantiates the function // and associates it with an ID. We perform that association later and need to return // a lambda expression. From e068ad8a535797730879137a8af69864a44793d1 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Fri, 31 Jan 2020 11:31:10 +0100 Subject: [PATCH 06/53] util: don't modify the input string in tokenize_string() This saves one full copy of the input string and avoids moving memory around at O(n^2) in the erase() call in each loop iteration. --- src/util.cc | 11 ++++++----- src/util.h | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/util.cc b/src/util.cc index a9b1707297..b820aabb3a 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1501,26 +1501,27 @@ TEST_CASE("util tokenize_string") CHECK(v2.size() == 1); } -vector* tokenize_string(string input, const string& delim, +vector* tokenize_string(const string &input, const string& delim, vector* rval, int limit) { if ( ! rval ) rval = new vector(); + size_t pos = 0; size_t n; auto found = 0; - while ( (n = input.find(delim)) != string::npos ) + while ( (n = input.find(delim, pos)) != string::npos ) { ++found; - rval->push_back(input.substr(0, n)); - input.erase(0, n + 1); + rval->emplace_back(input.substr(pos, n - pos)); + pos = n + 1; if ( limit && found == limit ) break; } - rval->push_back(input); + rval->emplace_back(input.substr(pos)); return rval; } diff --git a/src/util.h b/src/util.h index 3065652bea..40c45b274e 100644 --- a/src/util.h +++ b/src/util.h @@ -145,7 +145,7 @@ inline std::string get_escaped_string(const std::string& str, bool escape_all) return get_escaped_string(str.data(), str.length(), escape_all); } -std::vector* tokenize_string(std::string input, +std::vector* tokenize_string(const std::string &input, const std::string& delim, std::vector* rval = 0, int limit = 0); From f1566bda1403d9c2f3a35d3a07c0c9c168c3b2f3 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Fri, 31 Jan 2020 13:10:04 +0100 Subject: [PATCH 07/53] util: pass std::string_view to tokenize_string() This saves some overhead because some callers pass a plain C string here which needed to be copied to a temporary std::string. --- src/util.cc | 2 +- src/util.h | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/util.cc b/src/util.cc index b820aabb3a..544b547b29 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1501,7 +1501,7 @@ TEST_CASE("util tokenize_string") CHECK(v2.size() == 1); } -vector* tokenize_string(const string &input, const string& delim, +vector* tokenize_string(const std::string_view input, const std::string_view delim, vector* rval, int limit) { if ( ! rval ) diff --git a/src/util.h b/src/util.h index 40c45b274e..9442357e0b 100644 --- a/src/util.h +++ b/src/util.h @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -145,8 +146,8 @@ inline std::string get_escaped_string(const std::string& str, bool escape_all) return get_escaped_string(str.data(), str.length(), escape_all); } -std::vector* tokenize_string(const std::string &input, - const std::string& delim, +std::vector* tokenize_string(std::string_view input, + std::string_view delim, std::vector* rval = 0, int limit = 0); extern char* copy_string(const char* s); From 0589f295fa10805af8643312be5031d0b107cc5b Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Fri, 31 Jan 2020 13:25:34 +0100 Subject: [PATCH 08/53] util: pass std::string_view to normalize_path() Reduce overhead in some callers. --- src/util.cc | 4 ++-- src/util.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/util.cc b/src/util.cc index 544b547b29..54b1041501 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1552,13 +1552,13 @@ TEST_CASE("util normalize_path") CHECK(normalize_path("zeek/../..") == ".."); } -string normalize_path(const string& path) +string normalize_path(const std::string_view path) { size_t n; vector components, final_components; string new_path; - if ( path[0] == '/' ) + if ( !path.empty() && path[0] == '/' ) new_path = "/"; tokenize_string(path, "/", &components); diff --git a/src/util.h b/src/util.h index 9442357e0b..b2b346f727 100644 --- a/src/util.h +++ b/src/util.h @@ -344,7 +344,7 @@ std::string flatten_script_name(const std::string& name, * @param path A filesystem path. * @return A canonical/shortened version of \a path. */ -std::string normalize_path(const std::string& path); +std::string normalize_path(std::string_view path); /** * Strip the ZEEKPATH component from a path. From 5c0c336c6b379c5af710bc8d7b93d7e044eab119 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Fri, 31 Jan 2020 12:03:37 +0100 Subject: [PATCH 09/53] util: skip "." completely in normalize_path() Don't copy "." segments to the final_components list only to remove it afterwards. --- src/util.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/util.cc b/src/util.cc index 54b1041501..5c8fe05013 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1567,11 +1567,11 @@ string normalize_path(const std::string_view path) for ( it = components.begin(); it != components.end(); ++it ) { if ( *it == "" ) continue; + if ( *it == "." && it != components.begin() ) continue; + final_components.push_back(*it); - if ( *it == "." && it != components.begin() ) - final_components.pop_back(); - else if ( *it == ".." ) + if ( *it == ".." ) { auto cur_idx = final_components.size() - 1; From 53c4e300241e5411d2803f4b818433594f64302d Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Fri, 31 Jan 2020 12:05:47 +0100 Subject: [PATCH 10/53] util: reserve space in normalize_path() Pessimistic reservations to ensure that it does not need to be reallocated. --- src/util.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/util.cc b/src/util.cc index 5c8fe05013..1c73b4e89f 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1557,11 +1557,13 @@ string normalize_path(const std::string_view path) size_t n; vector components, final_components; string new_path; + new_path.reserve(path.size()); if ( !path.empty() && path[0] == '/' ) new_path = "/"; tokenize_string(path, "/", &components); + final_components.reserve(components.size()); vector::const_iterator it; for ( it = components.begin(); it != components.end(); ++it ) From 37cbd98e340db27d404990e1d8781d254d677b6c Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Fri, 31 Jan 2020 12:24:57 +0100 Subject: [PATCH 11/53] util: use "auto" in normalize_path() --- src/util.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/util.cc b/src/util.cc index 1c73b4e89f..341c8412e7 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1565,8 +1565,7 @@ string normalize_path(const std::string_view path) tokenize_string(path, "/", &components); final_components.reserve(components.size()); - vector::const_iterator it; - for ( it = components.begin(); it != components.end(); ++it ) + for ( auto it = components.begin(); it != components.end(); ++it ) { if ( *it == "" ) continue; if ( *it == "." && it != components.begin() ) continue; @@ -1600,7 +1599,7 @@ string normalize_path(const std::string_view path) } } - for ( it = final_components.begin(); it != final_components.end(); ++it ) + for ( auto it = final_components.begin(); it != final_components.end(); ++it ) { new_path.append(*it); new_path.append("/"); From 763afe6f5f8462c69647b44ab9b2d2dba2e18d96 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Fri, 31 Jan 2020 12:24:16 +0100 Subject: [PATCH 12/53] util: store std::string_view in "final_components" vector Don't copy those path segments - instead, use std::string_view to store references into the existing std::strings. This saves a good amount of allocation overhead. --- src/util.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/util.cc b/src/util.cc index 341c8412e7..db91af4f54 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1555,7 +1555,8 @@ TEST_CASE("util normalize_path") string normalize_path(const std::string_view path) { size_t n; - vector components, final_components; + vector components; + vector final_components; string new_path; new_path.reserve(path.size()); From 26da10ca058901c438b4b6c46e3f8dee5073f69f Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Fri, 31 Jan 2020 12:25:48 +0100 Subject: [PATCH 13/53] util: add a tokenize_string() overload which returns string_views Additionally, it uses a single "char" as delimiter, which is also faster. This patch speeds up Zeek startup by 10%. --- src/util.cc | 28 ++++++++++++++++++++++------ src/util.h | 2 ++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/util.cc b/src/util.cc index db91af4f54..d566c588a9 100644 --- a/src/util.cc +++ b/src/util.cc @@ -837,8 +837,7 @@ bool ensure_intermediate_dirs(const char* dirname) bool absolute = dirname[0] == '/'; string path = normalize_path(dirname); - vector path_components; - tokenize_string(path, "/", &path_components); + const auto path_components = tokenize_string(path, '/'); string current_dir; @@ -1525,6 +1524,25 @@ vector* tokenize_string(const std::string_view input, const std::string_ return rval; } +vector tokenize_string(const std::string_view input, const char delim) noexcept + { + vector rval; + + size_t pos = 0; + size_t n; + auto found = 0; + + while ( (n = input.find(delim, pos)) != string::npos ) + { + ++found; + rval.push_back(input.substr(pos, n - pos)); + pos = n + 1; + } + + rval.push_back(input.substr(pos)); + return rval; + } + TEST_CASE("util normalize_path") { CHECK(normalize_path("/1/2/3") == "/1/2/3"); @@ -1555,7 +1573,6 @@ TEST_CASE("util normalize_path") string normalize_path(const std::string_view path) { size_t n; - vector components; vector final_components; string new_path; new_path.reserve(path.size()); @@ -1563,7 +1580,7 @@ string normalize_path(const std::string_view path) if ( !path.empty() && path[0] == '/' ) new_path = "/"; - tokenize_string(path, "/", &components); + const auto components = tokenize_string(path, '/'); final_components.reserve(components.size()); for ( auto it = components.begin(); it != components.end(); ++it ) @@ -1616,8 +1633,7 @@ string without_bropath_component(const string& path) { string rval = normalize_path(path); - vector paths; - tokenize_string(bro_path(), ":", &paths); + const auto paths = tokenize_string(bro_path(), ':'); for ( size_t i = 0; i < paths.size(); ++i ) { diff --git a/src/util.h b/src/util.h index b2b346f727..bd2447c23f 100644 --- a/src/util.h +++ b/src/util.h @@ -150,6 +150,8 @@ std::vector* tokenize_string(std::string_view input, std::string_view delim, std::vector* rval = 0, int limit = 0); +std::vector tokenize_string(const std::string_view input, const char delim) noexcept; + extern char* copy_string(const char* s); extern int streq(const char* s1, const char* s2); From 2b2121be603cf0131a972fb232d084ad6bb304d5 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 28 Oct 2019 13:50:24 -0700 Subject: [PATCH 14/53] Remove CQ_TimerMgr in favor of PQ_TimerMgr --- src/PriorityQueue.cc | 12 +++--- src/PriorityQueue.h | 10 ++--- src/Sessions.cc | 2 +- src/Stats.cc | 6 +-- src/Timer.cc | 87 +------------------------------------------- src/Timer.h | 24 ------------ 6 files changed, 16 insertions(+), 125 deletions(-) diff --git a/src/PriorityQueue.cc b/src/PriorityQueue.cc index 9d5278108b..26d763d79a 100644 --- a/src/PriorityQueue.cc +++ b/src/PriorityQueue.cc @@ -27,7 +27,7 @@ PriorityQueue::~PriorityQueue() PQ_Element* PriorityQueue::Remove() { if ( heap_size == 0 ) - return 0; + return nullptr; PQ_Element* top = heap[0]; @@ -43,7 +43,7 @@ PQ_Element* PriorityQueue::Remove(PQ_Element* e) { if ( e->Offset() < 0 || e->Offset() >= heap_size || heap[e->Offset()] != e ) - return 0; // not in heap + return nullptr; // not in heap e->MinimizeTime(); BubbleUp(e->Offset()); @@ -56,7 +56,7 @@ PQ_Element* PriorityQueue::Remove(PQ_Element* e) return e2; } -int PriorityQueue::Add(PQ_Element* e) +bool PriorityQueue::Add(PQ_Element* e) { SetElement(heap_size, e); @@ -70,10 +70,10 @@ int PriorityQueue::Add(PQ_Element* e) if ( heap_size >= max_heap_size ) return Resize(max_heap_size * 2); else - return 1; + return true; } -int PriorityQueue::Resize(int new_size) +bool PriorityQueue::Resize(int new_size) { PQ_Element** tmp = new PQ_Element*[new_size]; for ( int i = 0; i < max_heap_size; ++i ) @@ -84,7 +84,7 @@ int PriorityQueue::Resize(int new_size) max_heap_size = new_size; - return heap != 0; + return heap != nullptr; } void PriorityQueue::BubbleUp(int bin) diff --git a/src/PriorityQueue.h b/src/PriorityQueue.h index c749f0dabc..0368396836 100644 --- a/src/PriorityQueue.h +++ b/src/PriorityQueue.h @@ -43,20 +43,20 @@ public: // is empty. PQ_Element* Remove(); - // Removes element e. Returns e, or nil if e wasn't in the queue. + // Removes element e. Returns e, or nullptr if e wasn't in the queue. // Note that e will be modified via MinimizeTime(). PQ_Element* Remove(PQ_Element* e); - // Add a new element to the queue. Returns 0 on failure (not enough - // memory to add the element), 1 on success. - int Add(PQ_Element* e); + // Add a new element to the queue. Returns false on failure (not enough + // memory to add the element), true on success. + bool Add(PQ_Element* e); int Size() const { return heap_size; } int PeakSize() const { return peak_heap_size; } uint64_t CumulativeNum() const { return cumulative_num; } protected: - int Resize(int new_size); + bool Resize(int new_size); void BubbleUp(int bin); void BubbleDown(int bin); diff --git a/src/Sessions.cc b/src/Sessions.cc index 2cbee6b227..4f5d6b461a 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -1356,7 +1356,7 @@ TimerMgr* NetSessions::LookupTimerMgr(const TimerMgr::Tag* tag, bool create) return 0; // Create new queue for tag. - TimerMgr* mgr = new CQ_TimerMgr(*tag); + TimerMgr* mgr = new PQ_TimerMgr(*tag); DBG_LOG(DBG_TM, "tag %s, creating new non-global timer mgr %p", tag->c_str(), mgr); timer_mgrs.insert(TimerMgrMap::value_type(*tag, mgr)); double t = timer_mgr->Time() + timer_mgr_inactivity_timeout; diff --git a/src/Stats.cc b/src/Stats.cc index f586435b6a..d352d19793 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -175,11 +175,9 @@ void ProfileLogger::Log() stats.nfa_states, stats.dfa_states, stats.computed, stats.mem / 1024)); } - file->Write(fmt("%.06f Timers: current=%d max=%d mem=%dK lag=%.2fs\n", + file->Write(fmt("%.06f Timers: current=%d max=%d lag=%.2fs\n", network_time, timer_mgr->Size(), timer_mgr->PeakSize(), - int(cq_memory_allocation() + - (timer_mgr->Size() * padded_sizeof(ConnectionTimer))) / 1024, network_time - timer_mgr->LastTimestamp())); DNS_Mgr::Stats dstats; @@ -208,7 +206,7 @@ void ProfileLogger::Log() const threading::Manager::msg_stats_list& thread_stats = thread_mgr->GetMsgThreadStats(); for ( threading::Manager::msg_stats_list::const_iterator i = thread_stats.begin(); - i != thread_stats.end(); ++i ) + i != thread_stats.end(); ++i ) { threading::MsgThread::Stats s = i->second; file->Write(fmt("%0.6f %-25s in=%" PRIu64 " out=%" PRIu64 " pending=%" PRIu64 "/%" PRIu64 diff --git a/src/Timer.cc b/src/Timer.cc index 40ac0696f4..838ae6ad08 100644 --- a/src/Timer.cc +++ b/src/Timer.cc @@ -87,8 +87,8 @@ PQ_TimerMgr::~PQ_TimerMgr() void PQ_TimerMgr::Add(Timer* timer) { - DBG_LOG(DBG_TM, "Adding timer %s to TimeMgr %p", - timer_type_to_string(timer->Type()), this); + DBG_LOG(DBG_TM, "Adding timer %s to TimeMgr %p at %.6f", + timer_type_to_string(timer->Type()), this, timer->Time()); // Add the timer even if it's already expired - that way, if // multiple already-added timers are added, they'll still @@ -145,86 +145,3 @@ void PQ_TimerMgr::Remove(Timer* timer) --current_timers[timer->Type()]; delete timer; } - -CQ_TimerMgr::CQ_TimerMgr(const Tag& tag) : TimerMgr(tag) - { - cq = cq_init(60.0, 1.0); - if ( ! cq ) - reporter->InternalError("could not initialize calendar queue"); - } - -CQ_TimerMgr::~CQ_TimerMgr() - { - cq_destroy(cq); - } - -void CQ_TimerMgr::Add(Timer* timer) - { - DBG_LOG(DBG_TM, "Adding timer %s to TimeMgr %p", - timer_type_to_string(timer->Type()), this); - - // Add the timer even if it's already expired - that way, if - // multiple already-added timers are added, they'll still - // execute in sorted order. - double t = timer->Time(); - - if ( t <= 0.0 ) - // Illegal time, which cq_enqueue won't like. For our - // purposes, just treat it as an old time that's already - // expired. - t = network_time; - - if ( cq_enqueue(cq, t, timer) < 0 ) - reporter->InternalError("problem queueing timer"); - - ++current_timers[timer->Type()]; - } - -void CQ_TimerMgr::Expire() - { - double huge_t = 1e20; // larger than any Unix timestamp - for ( Timer* timer = (Timer*) cq_dequeue(cq, huge_t); - timer; timer = (Timer*) cq_dequeue(cq, huge_t) ) - { - DBG_LOG(DBG_TM, "Dispatching timer %s in TimeMgr %p", - timer_type_to_string(timer->Type()), this); - timer->Dispatch(huge_t, 1); - --current_timers[timer->Type()]; - delete timer; - } - } - -int CQ_TimerMgr::DoAdvance(double new_t, int max_expire) - { - Timer* timer; - while ( (num_expired < max_expire || max_expire == 0) && - (timer = (Timer*) cq_dequeue(cq, new_t)) ) - { - last_timestamp = timer->Time(); - DBG_LOG(DBG_TM, "Dispatching timer %s in TimeMgr %p", - timer_type_to_string(timer->Type()), this); - timer->Dispatch(new_t, 0); - --current_timers[timer->Type()]; - delete timer; - ++num_expired; - } - - return num_expired; - } - -unsigned int CQ_TimerMgr::MemoryUsage() const - { - // FIXME. - return 0; - } - -void CQ_TimerMgr::Remove(Timer* timer) - { - // This may fail if we cancel a timer which has already been removed. - // That's ok, but then we mustn't delete the timer. - if ( cq_remove(cq, timer->Time(), timer) ) - { - --current_timers[timer->Type()]; - delete timer; - } - } diff --git a/src/Timer.h b/src/Timer.h index 7c135932b6..21b21fc5ee 100644 --- a/src/Timer.h +++ b/src/Timer.h @@ -7,10 +7,6 @@ #include #include "PriorityQueue.h" -extern "C" { -#include "cq.h" -} - // If you add a timer here, adjust TimerNames in Timer.cc. enum TimerType : uint8_t { TIMER_BACKDOOR, @@ -152,24 +148,4 @@ protected: PriorityQueue* q; }; -class CQ_TimerMgr : public TimerMgr { -public: - explicit CQ_TimerMgr(const Tag& arg_tag); - ~CQ_TimerMgr() override; - - void Add(Timer* timer) override; - void Expire() override; - - int Size() const override { return cq_size(cq); } - int PeakSize() const override { return cq_max_size(cq); } - uint64_t CumulativeNum() const override { return cq_cumulative_num(cq); } - unsigned int MemoryUsage() const; - -protected: - int DoAdvance(double t, int max_expire) override; - void Remove(Timer* timer) override; - - struct cq_handle *cq; -}; - extern TimerMgr* timer_mgr; From f16f0360ff064c31bb330f4006c1fab512afd2d3 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 26 Nov 2019 09:37:12 -0700 Subject: [PATCH 15/53] Only allow a single trace file (-r) or interface (-i) option on the command-line --- scripts/base/init-bare.zeek | 3 - src/Net.cc | 73 ++++++------------ src/Net.h | 4 +- src/Options.cc | 33 +++++--- src/Options.h | 4 +- src/iosource/Manager.cc | 2 +- src/iosource/Manager.h | 10 +-- src/iosource/pcap/pcap.bif | 35 ++------- src/main.cc | 35 ++++----- src/stats.bif | 9 +-- src/supervisor/Supervisor.cc | 2 +- src/util.cc | 4 +- src/zeek.bif | 14 ++-- .../btest/Baseline/bifs.packet_sources/out | 2 +- .../Traces/tunnels/gtp/pdp_ctx_messages.trace | Bin 0 -> 2360 bytes testing/btest/bifs/packet_sources.zeek | 2 +- .../core/tunnels/gtp/pdp_ctx_messages.test | 2 +- 17 files changed, 93 insertions(+), 141 deletions(-) create mode 100644 testing/btest/Traces/tunnels/gtp/pdp_ctx_messages.trace diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 7360a5bd23..d810c500a0 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -133,9 +133,6 @@ type PacketSource: record { netmask: count; }; -## A list of packet sources being read by Zeek. -type PacketSourceList: vector of PacketSource; - ## A connection's transport-layer protocol. Note that Zeek uses the term ## "connection" broadly, using flow semantics for ICMP and UDP. type transport_proto: enum { diff --git a/src/Net.cc b/src/Net.cc index ac38eb846d..856fc75ae0 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -145,41 +145,34 @@ void net_update_time(double new_network_time) PLUGIN_HOOK_VOID(HOOK_UPDATE_NETWORK_TIME, HookUpdateNetworkTime(new_network_time)); } -void net_init(const std::vector& interfaces, - const std::vector& pcap_input_files, +void net_init(const std::string& interface, + const std::string& pcap_input_file, const std::optional& pcap_output_file, bool do_watchdog) { - if ( ! pcap_input_files.empty() ) + if ( ! pcap_input_file.empty() ) { reading_live = pseudo_realtime > 0.0; reading_traces = 1; - for ( const auto& pif : pcap_input_files ) - { - iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(pif, false); - assert(ps); + iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(pcap_input_file, false); + assert(ps); - if ( ! ps->IsOpen() ) - reporter->FatalError("problem with trace file %s (%s)", - pif.data(), ps->ErrorMsg()); - } + if ( ! ps->IsOpen() ) + reporter->FatalError("problem with trace file %s (%s)", + pcap_input_file.c_str(), ps->ErrorMsg()); } - - else if ( ! interfaces.empty() ) + else if ( ! interface.empty() ) { reading_live = 1; reading_traces = 0; - for ( const auto& iface : interfaces ) - { - iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(iface, true); - assert(ps); + iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(interface, true); + assert(ps); - if ( ! ps->IsOpen() ) - reporter->FatalError("problem with interface %s (%s)", - iface.data(), ps->ErrorMsg()); - } + if ( ! ps->IsOpen() ) + reporter->FatalError("problem with interface %s (%s)", + interface.c_str(), ps->ErrorMsg()); } else @@ -377,14 +370,9 @@ void net_run() { auto have_active_packet_source = false; - for ( auto& ps : iosource_mgr->GetPktSrcs() ) - { - if ( ps->IsOpen() ) - { - have_active_packet_source = true; - break; - } - } + iosource::PktSrc* ps = iosource_mgr->GetPktSrc(); + if ( ps && ps->IsOpen() ) + have_active_packet_source = true; if ( ! have_active_packet_source ) // Can turn off pseudo realtime now @@ -401,20 +389,13 @@ void net_run() void net_get_final_stats() { - const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); - - for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); - i != pkt_srcs.end(); i++ ) + iosource::PktSrc* ps = iosource_mgr->GetPktSrc(); + if ( ps && ps->IsLive() ) { - iosource::PktSrc* ps = *i; - - if ( ps->IsLive() ) - { - iosource::PktSrc::Stats s; - ps->Statistics(&s); - reporter->Info("%" PRIu64 " packets received on interface %s, %" PRIu64 " dropped", - s.received, ps->Path().c_str(), s.dropped); - } + iosource::PktSrc::Stats s; + ps->Statistics(&s); + reporter->Info("%" PRIu64 " packets received on interface %s, %" PRIu64 " dropped", + s.received, ps->Path().c_str(), s.dropped); } } @@ -468,12 +449,8 @@ void net_continue_processing() if ( _processing_suspended == 1 ) { reporter->Info("processing continued"); - - const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); - - for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); - i != pkt_srcs.end(); i++ ) - (*i)->ContinueAfterSuspend(); + if ( iosource::PktSrc* ps = iosource_mgr->GetPktSrc() ) + ps->ContinueAfterSuspend(); } --_processing_suspended; diff --git a/src/Net.h b/src/Net.h index dab0014dd5..a5d330141b 100644 --- a/src/Net.h +++ b/src/Net.h @@ -14,8 +14,8 @@ #include "iosource/PktSrc.h" #include "iosource/PktDumper.h" -extern void net_init(const std::vector& interfaces, - const std::vector& pcap_input_files, +extern void net_init(const std::string& interfaces, + const std::string& pcap_input_file, const std::optional& pcap_output_file, bool do_watchdog); extern void net_run(); diff --git a/src/Options.cc b/src/Options.cc index 6143657967..2468675c24 100644 --- a/src/Options.cc +++ b/src/Options.cc @@ -16,8 +16,6 @@ void zeek::Options::filter_supervisor_options() { pcap_filter = {}; - interfaces = {}; - pcap_files = {}; signature_files = {}; pcap_output_file = {}; } @@ -49,8 +47,8 @@ void zeek::Options::filter_supervised_node_options() // use-case-specific way. e.g. interfaces is already handled for the // "cluster" use-case, but don't have supervised-pcap-reading // functionality yet. - /* interfaces = og.interfaces; */ - /* pcap_files = og.pcap_files; */ + /* interface = og.interface; */ + /* pcap_file = og.pcap_file; */ pcap_output_file = og.pcap_output_file; random_seed_input_file = og.random_seed_input_file; @@ -82,9 +80,9 @@ void zeek::usage(const char* prog, int code) fprintf(stderr, " -e|--exec | augment loaded scripts by given code\n"); fprintf(stderr, " -f|--filter | tcpdump filter\n"); fprintf(stderr, " -h|--help | command line help\n"); - fprintf(stderr, " -i|--iface | read from given interface\n"); + fprintf(stderr, " -i|--iface | read from given interface (only one allowed)\n"); fprintf(stderr, " -p|--prefix | add given prefix to Zeek script file resolution\n"); - fprintf(stderr, " -r|--readfile | read from given tcpdump file\n"); + fprintf(stderr, " -r|--readfile | read from given tcpdump file (only one allowed, pass '-' as the filename to read from stdin)\n"); fprintf(stderr, " -s|--rulefile | read rules from given file\n"); fprintf(stderr, " -t|--tracefile | activate execution tracing\n"); fprintf(stderr, " -v|--version | print version and exit\n"); @@ -270,12 +268,18 @@ zeek::Options zeek::parse_cmdline(int argc, char** argv) rval.print_usage = true; break; case 'i': - if ( ! rval.pcap_files.empty() ) + if ( ! rval.interface.empty() ) { - fprintf(stderr, "Using -i is not allowed when reading pcap files"); + fprintf(stderr, "ERROR: Only a single interface option (-i) is allowed.\n"); exit(1); } - rval.interfaces.emplace_back(optarg); + else if ( ! rval.pcap_file.empty() ) + { + fprintf(stderr, "ERROR: Using -i is not allow when reading a pcap file.\n"); + exit(1); + } + + rval.interface = optarg; break; case 'j': rval.supervisor_mode = true; @@ -290,12 +294,17 @@ zeek::Options zeek::parse_cmdline(int argc, char** argv) rval.script_prefixes.emplace_back(optarg); break; case 'r': - if ( ! rval.interfaces.empty() ) + if ( ! rval.pcap_file.empty() ) { - fprintf(stderr, "Using -r is not allowed when reading a live interface"); + fprintf(stderr, "ERROR: Only a single readfile option (-r) is allowed.\n"); exit(1); } - rval.pcap_files.emplace_back(optarg); + else if ( ! rval.interface.empty() ) + { + fprintf(stderr, "Using -r is not allowed when reading a live interface.\n"); + exit(1); + } + rval.pcap_file = optarg; break; case 's': rval.signature_files.emplace_back(optarg); diff --git a/src/Options.h b/src/Options.h index 01711481d9..ca61493cb0 100644 --- a/src/Options.h +++ b/src/Options.h @@ -58,8 +58,8 @@ struct Options { std::vector doctest_args; std::optional pcap_filter; - std::vector interfaces; - std::vector pcap_files; + std::string interface; + std::string pcap_file; std::vector signature_files; std::optional pcap_output_file; diff --git a/src/iosource/Manager.cc b/src/iosource/Manager.cc index 390449da81..c76e7786bc 100644 --- a/src/iosource/Manager.cc +++ b/src/iosource/Manager.cc @@ -207,7 +207,7 @@ void Manager::Register(IOSource* src, bool dont_count) void Manager::Register(PktSrc* src) { - pkt_srcs.push_back(src); + pkt_src = src; Register(src, false); } diff --git a/src/iosource/Manager.h b/src/iosource/Manager.h index a02401a0f1..cdb8757540 100644 --- a/src/iosource/Manager.h +++ b/src/iosource/Manager.h @@ -58,13 +58,11 @@ public: */ int Size() const { return sources.size() - dont_counts; } - typedef std::list PktSrcList; - /** - * Returns a list of all registered PktSrc instances. This is a - * subset of all registered IOSource instances. + * Returns the registered PktSrc. If not source is registered yet, + * returns a nullptr. */ - const PktSrcList& GetPktSrcs() const { return pkt_srcs; } + PktSrc* GetPktSrc() const { return pkt_src; } /** * Terminate all processing immediately by removing all sources (and @@ -136,7 +134,7 @@ private: typedef std::list PktDumperList; - PktSrcList pkt_srcs; + PktSrc* pkt_src = nullptr; PktDumperList pkt_dumpers; }; diff --git a/src/iosource/pcap/pcap.bif b/src/iosource/pcap/pcap.bif index 9e6e0238ba..e7b97cf4dd 100644 --- a/src/iosource/pcap/pcap.bif +++ b/src/iosource/pcap/pcap.bif @@ -35,17 +35,9 @@ function precompile_pcap_filter%(id: PcapFilterID, s: string%): bool bool success = true; - const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); - - for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); - i != pkt_srcs.end(); i++ ) - { - iosource::PktSrc* ps = *i; - - if ( ! ps->PrecompileFilter(id->ForceAsInt(), - s->CheckString()) ) - success = false; - } + iosource::PktSrc* ps = iosource_mgr->GetPktSrc(); + if ( ps && ! ps->PrecompileFilter(id->ForceAsInt(), s->CheckString()) ) + success = false; return val_mgr->GetBool(success); %} @@ -72,16 +64,9 @@ function Pcap::install_pcap_filter%(id: PcapFilterID%): bool %{ bool success = true; - const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); - - for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); - i != pkt_srcs.end(); i++ ) - { - iosource::PktSrc* ps = *i; - - if ( ! ps->SetFilter(id->ForceAsInt()) ) - success = false; - } + iosource::PktSrc* ps = iosource_mgr->GetPktSrc(); + if ( ps && ! ps->SetFilter(id->ForceAsInt()) ) + success = false; return val_mgr->GetBool(success); %} @@ -102,13 +87,9 @@ function Pcap::install_pcap_filter%(id: PcapFilterID%): bool ## uninstall_dst_net_filter function error%(%): string %{ - const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); - - for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); - i != pkt_srcs.end(); i++ ) + iosource::PktSrc* ps = iosource_mgr->GetPktSrc(); + if ( ps ) { - iosource::PktSrc* ps = *i; - const char* err = ps->ErrorMsg(); if ( *err ) return new StringVal(err); diff --git a/src/main.cc b/src/main.cc index 9847aa2df2..5e47fa6089 100644 --- a/src/main.cc +++ b/src/main.cc @@ -234,17 +234,17 @@ void done_with_network() #ifdef USE_PERFTOOLS_DEBUG - if ( perftools_profile ) - { - HeapProfilerDump("post net_run"); - HeapProfilerStop(); - } + if ( perftools_profile ) + { + HeapProfilerDump("post net_run"); + HeapProfilerStop(); + } - if ( heap_checker && ! heap_checker->NoLeaks() ) - { - fprintf(stderr, "Memory leaks - aborting.\n"); - abort(); - } + if ( heap_checker && ! heap_checker->NoLeaks() ) + { + fprintf(stderr, "Memory leaks - aborting.\n"); + abort(); + } #endif ZEEK_LSAN_DISABLE(); @@ -560,8 +560,8 @@ int main(int argc, char** argv) if ( options.plugins_to_load.empty() && options.scripts_to_load.empty() && options.script_options_to_set.empty() && - options.pcap_files.size() == 0 && - options.interfaces.size() == 0 && + options.pcap_file.empty() && + options.interface.empty() && ! options.identifier_to_print && ! command_line_policy && ! options.print_plugins && ! options.supervisor_mode && ! zeek::Supervisor::ThisNode() ) @@ -591,7 +591,7 @@ int main(int argc, char** argv) log_mgr = new logging::Manager(); input_mgr = new input::Manager(); file_mgr = new file_analysis::Manager(); - broker_mgr = new bro_broker::Manager(! options.pcap_files.empty()); + broker_mgr = new bro_broker::Manager(! options.pcap_file.empty()); plugin_mgr->InitPreScript(); analyzer_mgr->InitPreScript(); @@ -734,9 +734,7 @@ int main(int argc, char** argv) // ### Add support for debug command file. dbg_init_debugger(0); - auto all_interfaces = options.interfaces; - - if ( options.pcap_files.empty() && options.interfaces.empty() ) + if ( options.pcap_file.empty() && options.interface.empty() ) { Val* interfaces_val = internal_val("interfaces"); if ( interfaces_val ) @@ -745,15 +743,14 @@ int main(int argc, char** argv) interfaces_val->AsString()->Render(); if ( interfaces_str[0] != '\0' ) - tokenize_string(interfaces_str, " ", &all_interfaces); + options.interface = interfaces_str; delete [] interfaces_str; } } if ( dns_type != DNS_PRIME ) - net_init(all_interfaces, options.pcap_files, - options.pcap_output_file, options.use_watchdog); + net_init(options.interface, options.pcap_file, options.pcap_output_file, options.use_watchdog); net_done = internal_handler("net_done"); diff --git a/src/stats.bif b/src/stats.bif index 8166021d4e..cf94ebb985 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -44,13 +44,8 @@ function get_net_stats%(%): NetStats uint64_t link = 0; uint64_t bytes_recv = 0; - const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); - - for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); - i != pkt_srcs.end(); i++ ) + if ( iosource::PktSrc* ps = iosource_mgr->GetPktSrc() ) { - iosource::PktSrc* ps = *i; - struct iosource::PktSrc::Stats stat; ps->Statistics(&stat); recv += stat.received; @@ -440,7 +435,7 @@ function get_broker_stats%(%): BrokerStats %{ RecordVal* r = new RecordVal(BrokerStats); int n = 0; - + auto cs = broker_mgr->GetStatistics(); r->Assign(n++, val_mgr->GetCount(static_cast(cs.num_peers))); r->Assign(n++, val_mgr->GetCount(static_cast(cs.num_stores))); diff --git a/src/supervisor/Supervisor.cc b/src/supervisor/Supervisor.cc index 4d41383a85..b6e157c1a3 100644 --- a/src/supervisor/Supervisor.cc +++ b/src/supervisor/Supervisor.cc @@ -1284,7 +1284,7 @@ void Supervisor::SupervisedNode::Init(zeek::Options* options) const options->filter_supervised_node_options(); if ( config.interface ) - options->interfaces.emplace_back(*config.interface); + options->interface = *config.interface; for ( const auto& s : config.scripts ) options->scripts_to_load.emplace_back(s); diff --git a/src/util.cc b/src/util.cc index a9b1707297..87cb6250d8 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1902,11 +1902,11 @@ double current_time(bool real) double t = double(tv.tv_sec) + double(tv.tv_usec) / 1e6; - if ( ! pseudo_realtime || real || ! iosource_mgr || iosource_mgr->GetPktSrcs().empty() ) + if ( ! pseudo_realtime || real || ! iosource_mgr || ! iosource_mgr->GetPktSrc() ) return t; // This obviously only works for a single source ... - iosource::PktSrc* src = iosource_mgr->GetPktSrcs().front(); + iosource::PktSrc* src = iosource_mgr->GetPktSrc(); if ( net_is_processing_suspended() ) return src->CurrentPacketTimestamp(); diff --git a/src/zeek.bif b/src/zeek.bif index c357c3ce5f..e491909034 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -1905,26 +1905,24 @@ function reading_traces%(%): bool return val_mgr->GetBool(reading_traces); %} -## Returns: a list of packet sources being read by Zeek. +## Returns: the packet source being read by Zeek. ## ## .. zeek:see:: reading_live_traffic reading_traces -function packet_sources%(%): PacketSourceList +function packet_source%(%): PacketSource %{ auto ps_type = internal_type("PacketSource")->AsRecordType(); - auto psl_type = internal_type("PacketSourceList")->AsVectorType(); - auto rval = make_intrusive(psl_type); + auto ps = iosource_mgr->GetPktSrc(); + auto r = make_intrusive(ps_type); - for ( const auto& ps : iosource_mgr->GetPktSrcs() ) + if ( ps ) { - auto r = make_intrusive(ps_type); r->Assign(0, val_mgr->GetBool(ps->IsLive())); r->Assign(1, new StringVal(ps->Path())); r->Assign(2, val_mgr->GetInt(ps->LinkType())); r->Assign(3, val_mgr->GetCount(ps->Netmask())); - rval->Assign(rval->Size(), r.detach()); } - return rval.detach(); + return r.detach(); %} ## Generates a table of the size of all global variables. The table index is diff --git a/testing/btest/Baseline/bifs.packet_sources/out b/testing/btest/Baseline/bifs.packet_sources/out index ce47345d99..0fa2ce4c2a 100644 --- a/testing/btest/Baseline/bifs.packet_sources/out +++ b/testing/btest/Baseline/bifs.packet_sources/out @@ -1 +1 @@ -[[live=F, path=/Users/jsiwek/pro/zeek/zeek/testing/btest/Traces/http/get.trace, link_type=1, netmask=4294967295]] +[live=F, path=/Users/tim/Desktop/projects/zeek/testing/btest/Traces/http/get.trace, link_type=1, netmask=4294967295] diff --git a/testing/btest/Traces/tunnels/gtp/pdp_ctx_messages.trace b/testing/btest/Traces/tunnels/gtp/pdp_ctx_messages.trace new file mode 100644 index 0000000000000000000000000000000000000000..2124ee4c51974a5dcf170c57b9abef9b44c854b4 GIT binary patch literal 2360 zcmb`Ie@t6d6vxkfuPu)UE$t{jWUREH@@s{*lns{&9n3!*fz61KzgU4jX~Q6-bZ&_; zdz7DZKg{;W2uqBFsIVEAMO?BUMyrX_W&g-DGchKPEc`+I2V&GLnXBjAKEf-1s4<@8 zynF8Lz2EyeC+EG&X0tp60Bw7#ix`ec%|viRTX(PX;LFZVr`J>Ct*`O!boLyL90@tS z&owtRI4kydM~_7YaaQB0@l@hj1JpvB675rZyN8?=uSTNEaJW0#U+JvF9bRX}>wI*3 z+tKNDyvk6Jhzh%gMhisen)xHrBdtoG8~;B z@Yz5ia>|XV(i2Oe)e10f!-r}&o-mco-%-J|?n*2?j*o#0Yi>0#u zeRIf}9~U}vPx-%&SE)Vwzke_?CC4SvJoTc9dKS|eMGear`x1F?HH%rgx{B2%#uF?5 z=e$}9@1sfro%lP^$!<*;!2j;t%r1<{^~UR70T6(IZSR*SuyxR}?$}fI#d0451MSY{ zM@Vn0?kUu5z;$xHaos9SH$j@D&2{tNFYyUni0fL|hVG}D0Pz~<=4LM9oLp~Q_q3+l zmBxGgY$Lt{VcMOI%`Y?qbG0awk1S#i!$tvSqlX=eBv4cRpD9HKV@h;TiH$*?nkcj3 zi)s@GB7IU=85mHGMEbBp=nid|fQ)mYjBUP--zTvyS?5ath?#0Ki%v%D&B(4|Os+T1 z9@W^7qwZ!+x?-!?@)_9Lm(!|Vp4~H!d8?+!V)AKTwh*PyCV%)!3 zSMguf6;BWwF$U4eh;ejre5Da%a=mdzkH&a2t)|7-o_An9=^u+2H%|pI+^T=~+Qdo# z*U9z9^?jQDb=2Rifn&bjA%L(|-@<|@K>D9(Ld3tjU~=U=&dK%0^~W^**V6chZhY)W zLI>_nt1{XTdqAXzo77` zqV2^y93@UyY1z}yJX>B-SyjE$T~q7v`s)1kyLP8oiTPWSK_g~w)8xt(jLG%JnP)ZT t18FrmeWMGRhj4c~a{`7g&EW~wejk|`af|Zf-$->MpC)$uf6RyS_zMr1%~k*a literal 0 HcmV?d00001 diff --git a/testing/btest/bifs/packet_sources.zeek b/testing/btest/bifs/packet_sources.zeek index f6ae5aac5a..893f840bd2 100644 --- a/testing/btest/bifs/packet_sources.zeek +++ b/testing/btest/bifs/packet_sources.zeek @@ -3,5 +3,5 @@ event zeek_init() { - print packet_sources(); + print packet_source(); } diff --git a/testing/btest/core/tunnels/gtp/pdp_ctx_messages.test b/testing/btest/core/tunnels/gtp/pdp_ctx_messages.test index 06912c1f9d..4f145252b3 100644 --- a/testing/btest/core/tunnels/gtp/pdp_ctx_messages.test +++ b/testing/btest/core/tunnels/gtp/pdp_ctx_messages.test @@ -1,4 +1,4 @@ -# @TEST-EXEC: zeek -r $TRACES/tunnels/gtp/gtp_control_prime.pcap -r $TRACES/tunnels/gtp/gtp_create_pdp_ctx.pcap %INPUT >out +# @TEST-EXEC: zeek -r $TRACES/tunnels/gtp/pdp_ctx_messages.trace %INPUT >out # @TEST-EXEC: btest-diff out event gtpv1_message(c: connection, hdr: gtpv1_hdr) From fa9a568e8f815651bb2d5310ee985a3a377c0d80 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 5 Nov 2019 15:48:21 -0700 Subject: [PATCH 16/53] Remove #include of some iosource files from Net.h --- src/Conn.cc | 1 + src/Conn.h | 1 + src/Net.h | 11 ++++++++--- src/RuleMatcher.h | 1 + src/Sessions.cc | 2 ++ src/analyzer/protocol/vxlan/VXLAN.cc | 2 ++ src/util.cc | 1 + src/zeek.bif | 2 ++ 8 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/Conn.cc b/src/Conn.cc index 9329045f79..0126db55b0 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -16,6 +16,7 @@ #include "TunnelEncapsulation.h" #include "analyzer/Analyzer.h" #include "analyzer/Manager.h" +#include "iosource/IOSource.h" void ConnectionTimer::Init(Connection* arg_conn, timer_func arg_timer, bool arg_do_expire) diff --git a/src/Conn.h b/src/Conn.h index 9a2d3e8f6a..3bfe6a1109 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -17,6 +17,7 @@ #include "analyzer/Tag.h" #include "analyzer/Analyzer.h" +#include "iosource/Packet.h" class Connection; class ConnectionTimer; diff --git a/src/Net.h b/src/Net.h index a5d330141b..54885fe45f 100644 --- a/src/Net.h +++ b/src/Net.h @@ -10,9 +10,14 @@ #include "util.h" #include "List.h" #include "Func.h" -#include "iosource/IOSource.h" -#include "iosource/PktSrc.h" -#include "iosource/PktDumper.h" + +namespace iosource { + class IOSource; + class PktSrc; + class PktDumper; + } + +class Packet; extern void net_init(const std::string& interfaces, const std::string& pcap_input_file, diff --git a/src/RuleMatcher.h b/src/RuleMatcher.h index 5804fe63d8..9ba087d837 100644 --- a/src/RuleMatcher.h +++ b/src/RuleMatcher.h @@ -18,6 +18,7 @@ #include "Rule.h" #include "RuleAction.h" #include "RuleCondition.h" +#include "iosource/Packet.h" //#define MATCHER_PRINT_STATS diff --git a/src/Sessions.cc b/src/Sessions.cc index 4f5d6b461a..a078612762 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -29,6 +29,8 @@ #include "TunnelEncapsulation.h" #include "analyzer/Manager.h" +#include "iosource/IOSource.h" +#include "iosource/PktDumper.h" // These represent NetBIOS services on ephemeral ports. They're numbered // so that we can use a single int to hold either an actual TCP/UDP server diff --git a/src/analyzer/protocol/vxlan/VXLAN.cc b/src/analyzer/protocol/vxlan/VXLAN.cc index c58e04dc48..f936ca4655 100644 --- a/src/analyzer/protocol/vxlan/VXLAN.cc +++ b/src/analyzer/protocol/vxlan/VXLAN.cc @@ -1,5 +1,7 @@ // See the file in the main distribution directory for copyright. +#include // for the DLT_EN10MB constant definition + #include "VXLAN.h" #include "TunnelEncapsulation.h" #include "Conn.h" diff --git a/src/util.cc b/src/util.cc index 87cb6250d8..8a693d3fc2 100644 --- a/src/util.cc +++ b/src/util.cc @@ -51,6 +51,7 @@ #include "Net.h" #include "Reporter.h" #include "iosource/Manager.h" +#include "iosource/PktSrc.h" #include "ConvertUTF.h" #include "3rdparty/doctest.h" diff --git a/src/zeek.bif b/src/zeek.bif index e491909034..89f181601f 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -23,6 +23,8 @@ #include "file_analysis/Manager.h" #include "iosource/Manager.h" #include "iosource/Packet.h" +#include "iosource/PktSrc.h" +#include "iosource/PktDumper.h" #include "IntrusivePtr.h" using namespace std; From eee2abf72954f392620839d34b8a252674ec9456 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 29 Oct 2019 16:21:28 -0700 Subject: [PATCH 17/53] Type and variable usage cleanup in Net.h --- src/Net.cc | 51 +++++++++++++++++++++++++-------------------------- src/Net.h | 6 +++--- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/src/Net.cc b/src/Net.cc index 856fc75ae0..61deab227c 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -42,11 +42,11 @@ extern "C" { extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *); } -iosource::PktDumper* pkt_dumper = 0; +iosource::PktDumper* pkt_dumper = nullptr; -int reading_live = 0; -int reading_traces = 0; -int have_pending_timers = 0; +bool reading_live = false; +bool reading_traces = false; +bool have_pending_timers = false; double pseudo_realtime = 0.0; double network_time = 0.0; // time according to last packet timestamp // (or current time) @@ -57,11 +57,11 @@ double last_watchdog_proc_time = 0.0; // value of above during last watchdog bool terminating = false; // whether we're done reading and finishing up bool is_parsing = false; -const Packet *current_pkt = 0; +const Packet *current_pkt = nullptr; int current_dispatched = 0; double current_timestamp = 0.0; -iosource::PktSrc* current_pktsrc = 0; -iosource::IOSource* current_iosrc = 0; +iosource::PktSrc* current_pktsrc = nullptr; +iosource::IOSource* current_iosrc = nullptr; std::list files_scanned; std::vector sig_files; @@ -153,7 +153,7 @@ void net_init(const std::string& interface, if ( ! pcap_input_file.empty() ) { reading_live = pseudo_realtime > 0.0; - reading_traces = 1; + reading_traces = true; iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(pcap_input_file, false); assert(ps); @@ -164,8 +164,8 @@ void net_init(const std::string& interface, } else if ( ! interface.empty() ) { - reading_live = 1; - reading_traces = 0; + reading_live = true; + reading_traces = false; iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(interface, true); assert(ps); @@ -176,11 +176,11 @@ void net_init(const std::string& interface, } else - // have_pending_timers = 1, possibly. We don't set + // have_pending_timers = true, possibly. We don't set // that here, though, because at this point we don't know // whether the user's zeek_init() event will indeed set // a timer. - reading_traces = reading_live = 0; + reading_traces = reading_live = false; if ( pcap_output_file ) { @@ -192,11 +192,10 @@ void net_init(const std::string& interface, reporter->FatalError("problem opening dump file %s (%s)", writefile, pkt_dumper->ErrorMsg()); - ID* id = global_scope()->Lookup("trace_output_file"); - if ( ! id ) - reporter->Error("trace_output_file not defined in bro.init"); - else + if ( ID* id = global_scope()->Lookup("trace_output_file") ) id->SetVal(new StringVal(writefile)); + else + reporter->Error("trace_output_file not defined in bro.init"); } init_ip_addr_anonymizers(); @@ -239,7 +238,7 @@ void net_packet_dispatch(double t, const Packet* pkt, iosource::PktSrc* src_ps) expire_timers(src_ps); - SegmentProfiler* sp = 0; + SegmentProfiler* sp = nullptr; if ( load_sample ) { @@ -266,13 +265,13 @@ void net_packet_dispatch(double t, const Packet* pkt, iosource::PktSrc* src_ps) { delete sp; delete sample_logger; - sample_logger = 0; + sample_logger = nullptr; } processing_start_time = 0.0; // = "we're not processing now" current_dispatched = 0; - current_iosrc = 0; - current_pktsrc = 0; + current_iosrc = nullptr; + current_pktsrc = nullptr; } void net_run() @@ -299,20 +298,20 @@ void net_run() loop_counter = 0; } #endif - current_iosrc = src; + current_iosrc = nullptr; auto communication_enabled = broker_mgr->Active(); if ( src ) src->Process(); // which will call net_packet_dispatch() else if ( reading_live && ! pseudo_realtime) - { // live but no source is currently active - double ct = current_time(); + { + // live but no source is currently active if ( ! net_is_processing_suspended() ) { // Take advantage of the lull to get up to // date on timers and events. - net_update_time(ct); + net_update_time(current_time()); expire_timers(); usleep(1); // Just yield. } @@ -353,7 +352,7 @@ void net_run() processing_start_time = 0.0; // = "we're not processing now" current_dispatched = 0; - current_iosrc = 0; + current_iosrc = nullptr; if ( signal_val == SIGTERM || signal_val == SIGINT ) // We received a signal while processing the @@ -376,7 +375,7 @@ void net_run() if ( ! have_active_packet_source ) // Can turn off pseudo realtime now - pseudo_realtime = 0; + pseudo_realtime = 0.0; } } diff --git a/src/Net.h b/src/Net.h index 54885fe45f..b1a3de427c 100644 --- a/src/Net.h +++ b/src/Net.h @@ -43,19 +43,19 @@ inline bool net_is_processing_suspended() { return _processing_suspended > 0; } // Whether we're reading live traffic. -extern int reading_live; +extern bool reading_live; // Same but for reading from traces instead. We have two separate // variables because it's possible that neither is true, and we're // instead just running timers (per the variable after this one). -extern int reading_traces; +extern bool reading_traces; // True if we have timers scheduled for the future on which we need // to wait. "Need to wait" here means that we're running live (though // perhaps not reading_live, but just running in real-time) as opposed // to reading a trace (in which case we don't want to wait in real-time // on future timers). -extern int have_pending_timers; +extern bool have_pending_timers; // If > 0, we are reading from traces but trying to mimic real-time behavior. // (In this case, both reading_traces and reading_live are true.) The value From a00d11e44db3043a9cdb6c811c11fa2bf61a7f18 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 26 Nov 2019 12:03:45 -0700 Subject: [PATCH 18/53] IOSource API changes for new loop architecture - Removed GetFD and NextTimestamp methods - Removed concept of idle sources - Renamed Init to InitSource to avoid a name clash with the thread code - Added GetNextTimeout method This commit also removes the FD_Set file since it's no longer used --- src/iosource/FD_Set.h | 101 ---------------------------------------- src/iosource/IOSource.h | 77 +++++++++--------------------- 2 files changed, 23 insertions(+), 155 deletions(-) delete mode 100644 src/iosource/FD_Set.h diff --git a/src/iosource/FD_Set.h b/src/iosource/FD_Set.h deleted file mode 100644 index 9176611c93..0000000000 --- a/src/iosource/FD_Set.h +++ /dev/null @@ -1,101 +0,0 @@ -#pragma once - -#include -#include - -namespace iosource { - -/** - * A container holding a set of file descriptors. - */ -class FD_Set { -public: - - /** - * Constructor. The set is initially empty. - */ - FD_Set() : max(-1), fds() - { } - - /** - * Insert a file descriptor in to the set. - * @param fd the fd to insert in the set. - * @return false if fd was already in the set, else true. - */ - bool Insert(int fd) - { - if ( max < fd ) - max = fd; - - return fds.insert(fd).second; - } - - /** - * Inserts all the file descriptors from another set in to this one. - * @param other a file descriptor set to merge in to this one. - */ - void Insert(const FD_Set& other) - { - for ( std::set::const_iterator it = other.fds.begin(); - it != other.fds.end(); ++it ) - Insert(*it); - } - - /** - * Empties the set. - */ - void Clear() - { max = -1; fds.clear(); } - - /** - * Insert file descriptors in to a fd_set for use with select(). - * @return the greatest file descriptor inserted. - */ - int Set(fd_set* set) const - { - for ( std::set::const_iterator it = fds.begin(); it != fds.end(); - ++it ) - FD_SET(*it, set); - - return max; - } - - /** - * @return Whether a file descriptor belonging to this set is within the - * fd_set arugment. - */ - bool Ready(fd_set* set) const - { - for ( std::set::const_iterator it = fds.begin(); it != fds.end(); - ++it ) - { - if ( FD_ISSET(*it, set) ) - return true; - } - - return false; - } - - /** - * @return whether any file descriptors have been added to the set. - */ - bool Empty() const - { - return fds.empty(); - } - - /** - * @return the greatest file descriptor of all that have been added to the - * set, or -1 if the set is empty. - */ - int Max() const - { - return max; - } - -private: - int max; - std::set fds; -}; - -} // namespace bro diff --git a/src/iosource/IOSource.h b/src/iosource/IOSource.h index e899a11754..67807fa0b4 100644 --- a/src/iosource/IOSource.h +++ b/src/iosource/IOSource.h @@ -7,8 +7,6 @@ extern "C" { } #include -#include "FD_Set.h" -#include "Timer.h" namespace iosource { @@ -21,28 +19,28 @@ public: /** * Constructor. */ - IOSource() { idle = false; closed = false; } + IOSource() { closed = false; } /** * Destructor. */ virtual ~IOSource() {} - /** - * Returns true if source has nothing ready to process. - */ - bool IsIdle() const { return idle; } - /** * Returns true if more data is to be expected in the future. * Otherwise, source may be removed. */ bool IsOpen() const { return ! closed; } + /** + * Returns true if this is a packet source. + */ + virtual bool IsPacketSource() const { return false; } + /** * Initializes the source. Can be overwritten by derived classes. */ - virtual void Init() { } + virtual void InitSource() { } /** * Finalizes the source when it's being closed. Can be overwritten by @@ -51,76 +49,48 @@ public: virtual void Done() { } /** - * Returns select'able file descriptors for this source. Leaves the - * passed values untouched if not available. + * Return the next timeout value for this source. This should be + * overridden by source classes where they have a timeout value + * that can wake up the poll. * - * @param read Pointer to container where to insert a read descriptor. + * Must be overriden by derived classes. * - * @param write Pointer to container where to insert a write descriptor. - * - * @param except Pointer to container where to insert a except descriptor. + * @return A value for the next time that the source thinks the + * poll should time out in seconds from the current time. Return + * -1 if this source should not be considered. This should be a + * a value relative to network_time, not an absolute time. */ - virtual void GetFds(FD_Set* read, FD_Set* write, FD_Set* except) = 0; + virtual double GetNextTimeout() = 0; /** - * Returns the timestamp (in \a global network time) associated with - * next data item from this source. If the source wants the data - * item to be processed with a local network time, it sets the - * argument accordingly. - * - * This method will be called only when either IsIdle() returns - * false, or select() on one of the fds returned by GetFDs() - * indicates that there's data to process. - * - * Must be overridden by derived classes. - * - * @param network_time A pointer to store the \a local network time - * associated with the next item (as opposed to global network time). - * - * @return The global network time of the next entry, or a value - * smaller than zero if none is available currently. - */ - virtual double NextTimestamp(double* network_time) = 0; - - /** - * Processes and consumes next data item. - * - * This method will be called only when either IsIdle() returns - * false, or select() on one of the fds returned by GetFDs() - * indicates that there's data to process. + * Processes and consumes next data item. This will be called by + * net_run when this IOSource has been marked ready. * * Must be overridden by derived classes. */ virtual void Process() = 0; /** - * Returns the tag of the timer manafger associated with the last + * Returns the tag of the timer manager associated with the last * procesees data item. * * Can be overridden by derived classes. * * @return The tag, or null for the global timer manager. - * + * */ - virtual TimerMgr::Tag* GetCurrentTag() { return 0; } + virtual std::string* GetCurrentTag() { return 0; } /** - * Returns a descriptual tag representing the source for debugging. + * Returns a descriptive tag representing the source for debugging. * - * Can be overridden by derived classes. + * Must be overridden by derived classes. * * @return The debugging name. */ virtual const char* Tag() = 0; protected: - /* - * Callback for derived classes to call when they have gone dry - * temporarily. - * - * @param is_idle True if the source is idle currently. - */ - void SetIdle(bool is_idle) { idle = is_idle; } /* * Callback for derived class to call when they have shutdown. @@ -130,7 +100,6 @@ protected: void SetClosed(bool is_closed) { closed = is_closed; } private: - bool idle; bool closed; }; From 062cadb1245ee5905224f22c93194e820cbf816b Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 26 Nov 2019 12:06:45 -0700 Subject: [PATCH 19/53] Minor amount of code cleanup in Pcap IO source --- src/iosource/pcap/Source.cc | 56 ++++++++++--------------------------- src/iosource/pcap/Source.h | 1 - 2 files changed, 15 insertions(+), 42 deletions(-) diff --git a/src/iosource/pcap/Source.cc b/src/iosource/pcap/Source.cc index 0c8efd643d..47df453ff0 100644 --- a/src/iosource/pcap/Source.cc +++ b/src/iosource/pcap/Source.cc @@ -24,10 +24,10 @@ PcapSource::PcapSource(const std::string& path, bool is_live) { props.path = path; props.is_live = is_live; - pd = 0; + pd = nullptr; memset(¤t_hdr, 0, sizeof(current_hdr)); memset(&last_hdr, 0, sizeof(last_hdr)); - last_data = 0; + last_data = nullptr; } void PcapSource::Open() @@ -44,8 +44,8 @@ void PcapSource::Close() return; pcap_close(pd); - pd = 0; - last_data = 0; + pd = nullptr; + last_data = nullptr; Closed(); } @@ -53,18 +53,15 @@ void PcapSource::Close() void PcapSource::OpenLive() { char errbuf[PCAP_ERRBUF_SIZE]; - char tmp_errbuf[PCAP_ERRBUF_SIZE]; // Determine interface if not specified. if ( props.path.empty() ) { pcap_if_t* devs; - if ( pcap_findalldevs(&devs, tmp_errbuf) < 0 ) + if ( pcap_findalldevs(&devs, errbuf) < 0 ) { - snprintf(errbuf, sizeof(errbuf), - "pcap_findalldevs: %s", tmp_errbuf); - Error(errbuf); + Error(fmt("pcap_findalldevs: %s", errbuf)); return; } @@ -75,30 +72,26 @@ void PcapSource::OpenLive() if ( props.path.empty() ) { - snprintf(errbuf, sizeof(errbuf), - "pcap_findalldevs: empty device name"); - Error(errbuf); + Error("pcap_findalldevs: empty device name"); return; } } else { - snprintf(errbuf, sizeof(errbuf), - "pcap_findalldevs: no devices found"); - Error(errbuf); + Error("pcap_findalldevs: no devices found"); return; } } // Determine network and netmask. uint32_t net; - if ( pcap_lookupnet(props.path.c_str(), &net, &props.netmask, tmp_errbuf) < 0 ) + if ( pcap_lookupnet(props.path.c_str(), &net, &props.netmask, errbuf) < 0 ) { // ### The lookup can fail if no address is assigned to // the interface; and libpcap doesn't have any useful notion // of error codes, just error std::strings - how bogus - so we // just kludge around the error :-(. - // sprintf(errbuf, "pcap_lookupnet %s", tmp_errbuf); + // sprintf(errbuf, "pcap_lookupnet %s", errbuf); // return; props.netmask = 0xffffff00; } @@ -156,7 +149,7 @@ void PcapSource::OpenLive() } #ifdef HAVE_LINUX - if ( pcap_setnonblock(pd, 1, tmp_errbuf) < 0 ) + if ( pcap_setnonblock(pd, 1, errbuf) < 0 ) { PcapError("pcap_setnonblock"); return; @@ -167,14 +160,9 @@ void PcapSource::OpenLive() Info(fmt("pcap bufsize = %d\n", ((struct pcap *) pd)->bufsize)); #endif - props.selectable_fd = pcap_fileno(pd); - - SetHdrSize(); - - if ( ! pd ) - // Was closed, couldn't get header size. - return; + props.selectable_fd = pcap_get_selectable_fd(pd); + props.link_type = pcap_datalink(pd); props.is_live = true; Opened(props); @@ -192,18 +180,14 @@ void PcapSource::OpenOffline() return; } - SetHdrSize(); - - if ( ! pd ) - // Was closed, unknown link layer type. - return; - props.selectable_fd = fileno(pcap_file(pd)); if ( props.selectable_fd < 0 ) InternalError("OS does not support selectable pcap fd"); + props.link_type = pcap_datalink(pd); props.is_live = false; + Opened(props); } @@ -339,16 +323,6 @@ void PcapSource::PcapError(const char* where) Close(); } -void PcapSource::SetHdrSize() - { - if ( ! pd ) - return; - - char errbuf[PCAP_ERRBUF_SIZE]; - - props.link_type = pcap_datalink(pd); - } - iosource::PktSrc* PcapSource::Instantiate(const std::string& path, bool is_live) { return new PcapSource(path, is_live); diff --git a/src/iosource/pcap/Source.h b/src/iosource/pcap/Source.h index b9ef1e6da1..df57d6612d 100644 --- a/src/iosource/pcap/Source.h +++ b/src/iosource/pcap/Source.h @@ -28,7 +28,6 @@ private: void OpenLive(); void OpenOffline(); void PcapError(const char* where = 0); - void SetHdrSize(); Properties props; Stats stats; From 4fa3e4b9b4d78e6f20f42b9dbd85216403be07db Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 26 Nov 2019 17:14:56 -0700 Subject: [PATCH 20/53] Modify IOSource Manager to implement new loop architecture - Removes entire FindSoonest method that includes all of the calls to select() for checking for ready sources - Removes FD_Set checking against IOSources - Adds system for registering and unregistering file descriptors from IOSources. This allows individual sources to mark themselves as ready to be checked by the loop as they become available. - Adds entirely new loop architecture based on checking the IOSources for when their next timeout is, and then waiting for either that timeout or when the next source is ready. This also implements the polling based on what the OS supports, instead of just calling select() on all platforms. Currently it supports kqueue, epoll, and plain poll. - Adds system for pinging the loop to force it to wake up --- CMakeLists.txt | 1 + COPYING.3rdparty | 48 ++++++ cmake | 2 +- configure | 5 + src/3rdparty | 2 +- src/Net.cc | 50 ++----- src/iosource/CMakeLists.txt | 3 +- src/iosource/Manager.cc | 287 ++++++++++++++++++++++-------------- src/iosource/Manager.h | 153 +++++++++++++------ src/iosource/PktDumper.h | 1 + src/iosource/PktSrc.h | 1 + src/main.cc | 11 +- 12 files changed, 364 insertions(+), 200 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a919b8ffe1..53c6d4faca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -362,6 +362,7 @@ include(OpenSSLTests) include(CheckNameserCompat) include(GetArchitecture) include(RequireCXX17) +include(FindKqueue) if ( (OPENSSL_VERSION VERSION_EQUAL "1.1.0") OR (OPENSSL_VERSION VERSION_GREATER "1.1.0") ) set(ZEEK_HAVE_OPENSSL_1_1 true CACHE INTERNAL "" FORCE) diff --git a/COPYING.3rdparty b/COPYING.3rdparty index 831aa519f2..5dcb1bd88d 100644 --- a/COPYING.3rdparty +++ b/COPYING.3rdparty @@ -21,6 +21,54 @@ a legal notice, here is a blessing: ============================================================================== +%%% libkqueue - kqueue(2) compatibility library + +============================================================================== + +== all source == + +Copyright (c) 2009 Mark Heily + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +== event.h == + +Copyright (c) 1999,2000,2001 Jonathan Lemon +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +============================================================================== + %%% RapidJSON - A fast JSON parser/generator for C++ with both SAX/DOM style API ============================================================================== diff --git a/cmake b/cmake index d3e1884a87..df3a537228 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit d3e1884a876781dedac716d102e8a06e1cc54380 +Subproject commit df3a5372285223cdf24b980dcac89d21704bb9e9 diff --git a/configure b/configure index cd6fd4b025..5c7570f469 100755 --- a/configure +++ b/configure @@ -74,6 +74,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]... (Zeek uses an embedded version by default) --with-caf=PATH path to C++ Actor Framework install root (a Broker dependency that is embedded by default) + --with-libkqueue=PATH path to libkqueue install root + (Zeek uses an embedded version by default) Optional Packages in Non-Standard Locations: --with-geoip=PATH path to the libmaxminddb install root @@ -320,6 +322,9 @@ while [ $# -ne 0 ]; do --with-caf=*) append_cache_entry CAF_ROOT_DIR PATH $optarg ;; + --with-libkqueue=*) + append_cache_entry LIBKQUEUE_ROOT_DIR PATH $optarg + ;; --with-rocksdb=*) append_cache_entry ROCKSDB_ROOT_DIR PATH $optarg ;; diff --git a/src/3rdparty b/src/3rdparty index fae3223639..8ddcd00ba5 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit fae32236391d9117bf996e75d56ebd01ef076bc2 +Subproject commit 8ddcd00ba53d89dc39e46f42c15db4af8c9471d8 diff --git a/src/Net.cc b/src/Net.cc index 61deab227c..c4df44b9cd 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -278,45 +278,40 @@ void net_run() { set_processing_status("RUNNING", "net_run"); + std::vector ready; + ready.reserve(iosource_mgr->TotalSize()); + while ( iosource_mgr->Size() || (BifConst::exit_only_after_terminate && ! terminating) ) { - double ts; - iosource::IOSource* src = iosource_mgr->FindSoonest(&ts); + iosource_mgr->FindReadySources(&ready); #ifdef DEBUG static int loop_counter = 0; // If no source is ready, we log only every 100th cycle, // starting with the first. - if ( src || loop_counter++ % 100 == 0 ) + if ( ! ready.empty() || loop_counter++ % 100 == 0 ) { - DBG_LOG(DBG_MAINLOOP, "realtime=%.6f iosrc=%s ts=%.6f", - current_time(), src ? src->Tag() : "", src ? ts : -1); + DBG_LOG(DBG_MAINLOOP, "realtime=%.6f ready_count=%ld", + current_time(), ready.size()); - if ( src ) + if ( ! ready.empty() ) loop_counter = 0; } #endif current_iosrc = nullptr; auto communication_enabled = broker_mgr->Active(); - if ( src ) - src->Process(); // which will call net_packet_dispatch() - - else if ( reading_live && ! pseudo_realtime) + if ( ! ready.empty() ) { - // live but no source is currently active - if ( ! net_is_processing_suspended() ) + for ( auto src : ready ) { - // Take advantage of the lull to get up to - // date on timers and events. - net_update_time(current_time()); - expire_timers(); - usleep(1); // Just yield. + DBG_LOG(DBG_MAINLOOP, "processing source %s", src->Tag()); + current_iosrc = src; + src->Process(); } } - else if ( (have_pending_timers || communication_enabled || BifConst::exit_only_after_terminate) && ! pseudo_realtime ) @@ -327,25 +322,6 @@ void net_run() // doesn't risk blocking on other inputs. net_update_time(current_time()); expire_timers(); - - // Avoid busy-waiting - pause for 100 ms. - // We pick a sleep value of 100 msec that buys - // us a lot of idle time, but doesn't delay near-term - // timers too much. (Delaying them somewhat is okay, - // since Bro timers are not high-precision anyway.) - if ( ! communication_enabled ) - usleep(100000); - else - usleep(1000); - - // Flawfinder says about usleep: - // - // This C routine is considered obsolete (as opposed - // to the shell command by the same name). The - // interaction of this function with SIGALRM and - // other timer functions such as sleep(), alarm(), - // setitimer(), and nanosleep() is unspecified. - // Use nanosleep(2) or setitimer(2) instead. } mgr.Drain(); diff --git a/src/iosource/CMakeLists.txt b/src/iosource/CMakeLists.txt index f7497c7fe6..881903d65c 100644 --- a/src/iosource/CMakeLists.txt +++ b/src/iosource/CMakeLists.txt @@ -15,8 +15,7 @@ set(iosource_SRCS Packet.cc PktDumper.cc PktSrc.cc -) + ) bro_add_subdir_library(iosource ${iosource_SRCS}) add_dependencies(bro_iosource generate_outputs) - diff --git a/src/iosource/Manager.cc b/src/iosource/Manager.cc index c76e7786bc..c5791a2272 100644 --- a/src/iosource/Manager.cc +++ b/src/iosource/Manager.cc @@ -1,17 +1,18 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include #include #include #include #include -#include - #include "Manager.h" #include "IOSource.h" +#include "Net.h" #include "PktSrc.h" #include "PktDumper.h" #include "plugin/Manager.h" +#include "broker/Manager.h" #include "util.h" @@ -19,8 +20,39 @@ using namespace iosource; +Manager::WakeupHandler::WakeupHandler() + { + iosource_mgr->RegisterFd(flare.FD(), this); + } + +Manager::WakeupHandler::~WakeupHandler() + { + iosource_mgr->UnregisterFd(flare.FD()); + } + +void Manager::WakeupHandler::Process() + { + flare.Extinguish(); + } + +void Manager::WakeupHandler::Ping(const std::string& where) + { + DBG_LOG(DBG_MAINLOOP, "Pinging WakeupHandler from %s", where.c_str()); + flare.Fire(); + } + +Manager::Manager() + { + event_queue = kqueue(); + if ( event_queue == -1 ) + reporter->FatalError("Failed to initialize kqueue: %s", strerror(errno)); + } + Manager::~Manager() { + delete wakeup; + wakeup = nullptr; + for ( SourceList::iterator i = sources.begin(); i != sources.end(); ++i ) { (*i)->src->Done(); @@ -37,6 +69,14 @@ Manager::~Manager() } pkt_dumpers.clear(); + + if ( event_queue != -1 ) + close(event_queue); + } + +void Manager::InitPostScript() + { + wakeup = new WakeupHandler(); } void Manager::RemoveAll() @@ -45,12 +85,19 @@ void Manager::RemoveAll() dont_counts = sources.size(); } -IOSource* Manager::FindSoonest(double* ts) +void Manager::Wakeup(const std::string& where) { + if ( wakeup ) + wakeup->Ping(where); + } + +void Manager::FindReadySources(std::vector* ready) + { + ready->clear(); + // Remove sources which have gone dry. For simplicity, we only // remove at most one each time. - for ( SourceList::iterator i = sources.begin(); - i != sources.end(); ++i ) + for ( SourceList::iterator i = sources.begin(); i != sources.end(); ++i ) if ( ! (*i)->src->IsOpen() ) { (*i)->src->Done(); @@ -59,124 +106,140 @@ IOSource* Manager::FindSoonest(double* ts) break; } - // Ideally, we would always call select on the fds to see which - // are ready, and return the soonest. Unfortunately, that'd mean - // one select-call per packet, which we can't afford in high-volume - // environments. Thus, we call select only every SELECT_FREQUENCY - // call (or if all sources report that they are dry). + // If there aren't any sources and exit_only_after_terminate is false, just + // return an empty set of sources. We want the main loop to end. + if ( Size() == 0 && ( ! BifConst::exit_only_after_terminate || terminating ) ) + return; - ++call_count; + double timeout = -1; + IOSource* timeout_src = nullptr; + bool time_to_poll = false; - IOSource* soonest_src = 0; - double soonest_ts = 1e20; - double soonest_local_network_time = 1e20; - bool all_idle = true; - - // Find soonest source of those which tell us they have something to - // process. - for ( SourceList::iterator i = sources.begin(); i != sources.end(); ++i ) + ++poll_counter; + if ( poll_counter % poll_interval == 0 ) { - if ( ! (*i)->src->IsIdle() ) + poll_counter = 0; + time_to_poll = true; + } + + // Find the source with the next timeout value. + for ( auto src : sources ) + { + auto iosource = src->src; + if ( iosource->IsOpen() ) { - all_idle = false; - double local_network_time = 0; - double ts = (*i)->src->NextTimestamp(&local_network_time); - if ( ts >= 0 && ts < soonest_ts ) + double next = iosource->GetNextTimeout(); + if ( timeout == -1 || ( next >= 0.0 && next < timeout ) ) { - soonest_ts = ts; - soonest_src = (*i)->src; - soonest_local_network_time = - local_network_time ? - local_network_time : ts; + timeout = next; + timeout_src = iosource; + + // If a source has a zero timeout then it's ready. Just add it to the + // list already. Only do this if it's not time to poll though, since + // we don't want things in the vector passed into Poll() or it'll end + // up inserting duplicates. + if ( timeout == 0 && ! time_to_poll ) + ready->push_back(timeout_src); } + + // Avoid calling Poll() if we can help it since on very high-traffic + // networks, we spend too much time in Poll() and end up dropping packets. + if ( ! time_to_poll && iosource == pkt_src && pkt_src->IsLive() ) + ready->push_back(pkt_src); } } - // If we found one and aren't going to select this time, - // return it. - int maxx = 0; + // If we didn't find any IOSources with zero timeouts or it's time to + // force a poll, do that and return. Otherwise return the set of ready + // sources that we have. + if ( ready->empty() || time_to_poll ) + Poll(ready, timeout, timeout_src); + } - if ( soonest_src && (call_count % SELECT_FREQUENCY) != 0 ) - goto finished; +void Manager::Poll(std::vector* ready, double timeout, IOSource* timeout_src) + { + struct timespec kqueue_timeout; + ConvertTimeout(timeout, kqueue_timeout); - // Select on the join of all file descriptors. - fd_set fd_read, fd_write, fd_except; - - FD_ZERO(&fd_read); - FD_ZERO(&fd_write); - FD_ZERO(&fd_except); - - for ( SourceList::iterator i = sources.begin(); - i != sources.end(); ++i ) + int ret = kevent(event_queue, NULL, 0, events.data(), events.size(), &kqueue_timeout); + if ( ret == -1 ) { - Source* src = (*i); - - if ( ! src->src->IsIdle() ) - // No need to select on sources which we know to - // be ready. - continue; - - src->Clear(); - src->src->GetFds(&src->fd_read, &src->fd_write, &src->fd_except); - src->SetFds(&fd_read, &fd_write, &fd_except, &maxx); + // Ignore interrupts since we may catch one during shutdown and we don't want the + // error to get printed. + if ( errno != EINTR ) + reporter->InternalWarning("Error calling kevent: %s", strerror(errno)); } - - // We can't block indefinitely even when all sources are dry: - // we're doing some IOSource-independent stuff in the main loop, - // so we need to return from time to time. (Instead of no time-out - // at all, we use a very small one. This lets FreeBSD trigger a - // BPF buffer switch on the next read when the hold buffer is empty - // while the store buffer isn't filled yet. - - struct timeval timeout; - - if ( all_idle ) + else if ( ret == 0 ) { - // Interesting: when all sources are dry, simply sleeping a - // bit *without* watching for any fd becoming ready may - // decrease CPU load. I guess that's because it allows - // the kernel's packet buffers to fill. - Robin - timeout.tv_sec = 0; - timeout.tv_usec = 20; // SELECT_TIMEOUT; - select(0, 0, 0, 0, &timeout); + if ( timeout_src ) + ready->push_back(timeout_src); } - - if ( ! maxx ) - // No selectable fd at all. - goto finished; - - timeout.tv_sec = 0; - timeout.tv_usec = 0; - - if ( select(maxx + 1, &fd_read, &fd_write, &fd_except, &timeout) > 0 ) - { // Find soonest. - for ( SourceList::iterator i = sources.begin(); - i != sources.end(); ++i ) + else + { + // kevent returns the number of events that are ready, so we only need to loop + // over that many of them. + for ( int i = 0; i < ret; i++ ) { - Source* src = (*i); - - if ( ! src->src->IsIdle() ) - continue; - - if ( src->Ready(&fd_read, &fd_write, &fd_except) ) + if ( events[i].filter == EVFILT_READ ) { - double local_network_time = 0; - double ts = src->src->NextTimestamp(&local_network_time); - if ( ts >= 0.0 && ts < soonest_ts ) - { - soonest_ts = ts; - soonest_src = src->src; - soonest_local_network_time = - local_network_time ? - local_network_time : ts; - } + std::map::const_iterator it = fd_map.find(events[i].ident); + if ( it != fd_map.end() ) + ready->push_back(it->second); } } } + } -finished: - *ts = soonest_local_network_time; - return soonest_src; +void Manager::ConvertTimeout(double timeout, struct timespec& spec) + { + // If timeout ended up -1, set it to some nominal value just to keep the loop + // from blocking forever. This is the case of exit_only_after_terminate when + // there isn't anything else going on. + if ( timeout < 0 ) + { + spec.tv_sec = 0; + spec.tv_nsec = 1e8; + } + else + { + spec.tv_sec = static_cast(timeout); + spec.tv_nsec = static_cast((timeout - spec.tv_sec) * 1e9); + } + } + +void Manager::RegisterFd(int fd, IOSource* src) + { + struct kevent event; + EV_SET(&event, fd, EVFILT_READ, EV_ADD, 0, 0, NULL); + int ret = kevent(event_queue, &event, 1, NULL, 0, NULL); + if ( ret != -1 ) + { + events.push_back({}); + DBG_LOG(DBG_MAINLOOP, "Registered fd %d from %s", fd, src->Tag()); + fd_map[fd] = src; + + Wakeup("RegisterFd"); + } + else + { + DBG_LOG(DBG_MAINLOOP, "Failed to register fd %d from %s: %s", fd, src->Tag(), strerror(errno)); + } + } + +void Manager::UnregisterFd(int fd) + { + if ( fd_map.find(fd) != fd_map.end() ) + { + struct kevent event; + EV_SET(&event, fd, EVFILT_READ, EV_DELETE, 0, 0, NULL); + int ret = kevent(event_queue, &event, 1, NULL, 0, NULL); + if ( ret != -1 ) + DBG_LOG(DBG_MAINLOOP, "Unregistered fd %d", fd); + + fd_map.erase(fd); + + Wakeup("UnregisterFd"); + } } void Manager::Register(IOSource* src, bool dont_count) @@ -195,7 +258,7 @@ void Manager::Register(IOSource* src, bool dont_count) } } - src->Init(); + src->InitSource(); Source* s = new Source; s->src = src; s->dont_count = dont_count; @@ -208,6 +271,16 @@ void Manager::Register(IOSource* src, bool dont_count) void Manager::Register(PktSrc* src) { pkt_src = src; + + // The poll interval gets defaulted to 100 which is good for cases like reading + // from pcap files and when there isn't a packet source, but is a little too + // infrequent for live sources (especially fast live sources). Set it down a + // little bit for those sources. + if ( src->IsLive() ) + poll_interval = 10; + else if ( pseudo_realtime ) + poll_interval = 1; + Register(src, false); } @@ -317,11 +390,3 @@ PktDumper* Manager::OpenPktDumper(const string& path, bool append) return pd; } - -void Manager::Source::SetFds(fd_set* read, fd_set* write, fd_set* except, - int* maxx) const - { - *maxx = std::max(*maxx, fd_read.Set(read)); - *maxx = std::max(*maxx, fd_write.Set(write)); - *maxx = std::max(*maxx, fd_except.Set(except)); - } diff --git a/src/iosource/Manager.h b/src/iosource/Manager.h index cdb8757540..ddf89db11a 100644 --- a/src/iosource/Manager.h +++ b/src/iosource/Manager.h @@ -2,30 +2,41 @@ #pragma once +#include "zeek-config.h" + #include -#include -#include "iosource/FD_Set.h" +#include +#include + +#include "IOSource.h" +#include "Flare.h" namespace iosource { -class IOSource; class PktSrc; class PktDumper; /** - * Singleton class managing all IOSources. + * Manager class for IO sources. This handles all of the polling of sources + * in the main loop. */ class Manager { public: /** * Constructor. */ - Manager() { call_count = 0; dont_counts = 0; } + Manager(); /** * Destructor. */ - ~Manager(); + virtual ~Manager(); + + /** + * Initializes some extra fields that can't be done during the + * due to dependencies on other objects being initialized first. + */ + void InitPostScript(); /** * Registers an IOSource with the manager. If the source is already @@ -41,23 +52,17 @@ public: */ void Register(IOSource* src, bool dont_count = false); - /** - * Returns the packet source with the soonest available input. This - * may block for a little while if all are dry. - * - * @param ts A pointer where to store the timestamp of the input that - * the soonest source has available next. - * - * @return The source, or null if no source has input. - */ - IOSource* FindSoonest(double* ts); - /** * Returns the number of registered and still active sources, - * excluding those that are registered as \a dont_cont. + * excluding those that are registered as \a dont_count. */ int Size() const { return sources.size() - dont_counts; } + /** + * Returns total number of sources including dont_counts; + */ + int TotalSize() const { return sources.size(); } + /** * Returns the registered PktSrc. If not source is registered yet, * returns a nullptr. @@ -93,49 +98,111 @@ public: */ PktDumper* OpenPktDumper(const std::string& path, bool append); + /** + * Finds the sources that have data ready to be processed. + * + * @param ready A vector used to return the set of sources that are ready. + */ + void FindReadySources(std::vector* ready); + + /** + * Registers a file descriptor and associated IOSource with the manager + * to be checked during FindReadySources. + * + * @param fd A file descriptor pointing at some resource that should be + * checked for readiness. + * @param src The IOSource that owns the file descriptor. + */ + void RegisterFd(int fd, IOSource* src); + + /** + * Unregisters a file descriptor from the FindReadySources checks. + */ + void UnregisterFd(int fd); + + /** + * Forces the poll in FindReadySources to wake up immediately. This method + * is called during RegisterFd and UnregisterFd since those methods cause + * changes to the active set of file descriptors. + */ + void Wakeup(const std::string& where); + private: - /** - * When looking for a source with something to process, every - * SELECT_FREQUENCY calls we will go ahead and block on a select(). - */ - static const int SELECT_FREQUENCY = 25; /** - * Microseconds to wait in an empty select if no source is ready. + * Calls the appropriate poll method to gather a set of IOSources that are + * ready for processing. + * + * @param ready a vector used to return the ready sources. + * @param timeout the value to be used for the timeout of the poll. This + * should be a value relative to the current network time, not an + * absolute time value. This may be zero to cause an infinite timeout or + * -1 to force a very short timeout. + * @param timeout_src The source associated with the current timeout value. + * This is typically a timer manager object. */ - static const int SELECT_TIMEOUT = 50; + void Poll(std::vector* ready, double timeout, IOSource* timeout_src); + /** + * Converts a double timeout value into a timespec struct used for calls + * to kevent(). + */ + void ConvertTimeout(double timeout, struct timespec& spec); + + /** + * Specialized registration method for packet sources. + */ void Register(PktSrc* src); + void RemoveAll(); - unsigned int call_count; - int dont_counts; + class WakeupHandler : public IOSource { + public: + WakeupHandler(); + ~WakeupHandler(); + + /** + * Tells the handler to wake up the loop by firing the flare. + * + * @param where a string denoting where this ping was called from. Used + * for debugging output. + */ + void Ping(const std::string& where); + + // IOSource API methods + void Process() override; + const char* Tag() override { return "WakeupHandler"; } + double GetNextTimeout() override { return -1; } + + private: + bro::Flare flare; + }; struct Source { IOSource* src; - FD_Set fd_read; - FD_Set fd_write; - FD_Set fd_except; bool dont_count; - - bool Ready(fd_set* read, fd_set* write, fd_set* except) const - { return fd_read.Ready(read) || fd_write.Ready(write) || - fd_except.Ready(except); } - - void SetFds(fd_set* read, fd_set* write, fd_set* except, - int* maxx) const; - - void Clear() - { fd_read.Clear(); fd_write.Clear(); fd_except.Clear(); } }; - typedef std::list SourceList; + using SourceList = std::vector; SourceList sources; - typedef std::list PktDumperList; + using PktDumperList = std::vector; + PktDumperList pkt_dumpers; PktSrc* pkt_src = nullptr; - PktDumperList pkt_dumpers; + + int dont_counts = 0; + int zero_timeout_count = 0; + WakeupHandler* wakeup = nullptr; + int poll_counter = 0; + int poll_interval = 100; + + int event_queue = -1; + std::map fd_map; + + // This is only used for the output of the call to kqueue in FindReadySources(). + // The actual events are stored as part of the queue. + std::vector events; }; } diff --git a/src/iosource/PktDumper.h b/src/iosource/PktDumper.h index 9a7949fc5d..2b6251ba0a 100644 --- a/src/iosource/PktDumper.h +++ b/src/iosource/PktDumper.h @@ -87,6 +87,7 @@ public: protected: friend class Manager; + friend class ManagerBase; /** * Structure to pass back information about the packet dumper to the diff --git a/src/iosource/PktSrc.h b/src/iosource/PktSrc.h index 2f08508d85..7ba9335df8 100644 --- a/src/iosource/PktSrc.h +++ b/src/iosource/PktSrc.h @@ -206,6 +206,7 @@ public: protected: friend class Manager; + friend class ManagerBase; // Methods to use by derived classes. diff --git a/src/main.cc b/src/main.cc index 5e47fa6089..72a1b5ce79 100644 --- a/src/main.cc +++ b/src/main.cc @@ -256,6 +256,8 @@ void terminate_bro() terminating = true; + iosource_mgr->Wakeup("terminate_bro"); + // File analysis termination may produce events, so do it early on in // the termination process. file_mgr->Terminate(); @@ -332,6 +334,9 @@ RETSIGTYPE sig_handler(int signo) set_processing_status("TERMINATING", "sig_handler"); signal_val = signo; + if ( ! terminating ) + iosource_mgr->Wakeup("sig_handler"); + return RETSIGVAL; } @@ -662,6 +667,7 @@ int main(int argc, char** argv) if ( reporter->Errors() > 0 ) exit(1); + iosource_mgr->InitPostScript(); plugin_mgr->InitPostScript(); zeekygen_mgr->InitPostScript(); broker_mgr->InitPostScript(); @@ -878,11 +884,6 @@ int main(int argc, char** argv) have_pending_timers = ! reading_traces && timer_mgr->Size() > 0; - iosource_mgr->Register(thread_mgr, true); - - if ( zeek::supervisor_mgr ) - iosource_mgr->Register(zeek::supervisor_mgr); - if ( iosource_mgr->Size() > 0 || have_pending_timers || BifConst::exit_only_after_terminate ) From 92bde61b787aef6aa1e2f8f2a45bdcb0b4365851 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 26 Nov 2019 12:37:22 -0700 Subject: [PATCH 21/53] Make TimerMgr an IOSource - This allows the loop to check what the next timeout is and use that as the basis for the timeout of the poll - This commit also removes the TimerMgr::Tag type, since it causes a name clash with other code in IOSource --- src/Conn.cc | 4 +-- src/Conn.h | 2 +- src/Sessions.cc | 2 +- src/Sessions.h | 6 ++-- src/Timer.cc | 46 +++++++++++++++++++++++++++++- src/Timer.h | 74 +++++++++++++++++++++++++++++++------------------ src/main.cc | 6 ++-- 7 files changed, 101 insertions(+), 39 deletions(-) diff --git a/src/Conn.cc b/src/Conn.cc index 0126db55b0..855f14558c 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -115,8 +115,8 @@ Connection::Connection(NetSessions* s, const ConnIDKey& k, double t, const ConnI ++current_connections; ++total_connections; - TimerMgr::Tag* tag = current_iosrc->GetCurrentTag(); - conn_timer_mgr = tag ? new TimerMgr::Tag(*tag) : 0; + std::string* tag = current_iosrc->GetCurrentTag(); + conn_timer_mgr = tag ? new std::string(*tag) : nullptr; if ( arg_encap ) encapsulation = new EncapsulationStack(*arg_encap); diff --git a/src/Conn.h b/src/Conn.h index 3bfe6a1109..41b8fc8283 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -321,7 +321,7 @@ protected: bool key_valid; // Timer manager to use for this conn (or nil). - TimerMgr::Tag* conn_timer_mgr; + std::string* conn_timer_mgr; timer_list timers; IPAddr orig_addr; diff --git a/src/Sessions.cc b/src/Sessions.cc index a078612762..984a8df754 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -1338,7 +1338,7 @@ bool NetSessions::WantConnection(uint16_t src_port, uint16_t dst_port, return true; } -TimerMgr* NetSessions::LookupTimerMgr(const TimerMgr::Tag* tag, bool create) +TimerMgr* NetSessions::LookupTimerMgr(const std::string* tag, bool create) { if ( ! tag ) { diff --git a/src/Sessions.h b/src/Sessions.h index f2120ba694..c627393e5d 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -104,7 +104,7 @@ public: // Looks up timer manager associated with tag. If tag is unknown and // "create" is true, creates new timer manager and stores it. Returns // global timer manager if tag is nil. - TimerMgr* LookupTimerMgr(const TimerMgr::Tag* tag, bool create = true); + TimerMgr* LookupTimerMgr(const std::string* tag, bool create = true); void ExpireTimerMgrs(); @@ -245,8 +245,8 @@ protected: PacketProfiler* pkt_profiler; // We may use independent timer managers for different sets of related - // activity. The managers are identified by an unique tag. - typedef std::map TimerMgrMap; + // activity. The managers are identified by a unique tag. + typedef std::map TimerMgrMap; TimerMgrMap timer_mgrs; }; diff --git a/src/Timer.cc b/src/Timer.cc index 838ae6ad08..a16ee3bf33 100644 --- a/src/Timer.cc +++ b/src/Timer.cc @@ -5,7 +5,11 @@ #include "util.h" #include "Timer.h" #include "Desc.h" +#include "Net.h" +#include "NetVar.h" #include "broker/Manager.h" +#include "iosource/Manager.h" +#include "iosource/PktSrc.h" // Names of timers in same order than in TimerType. const char* TimerNames[] = { @@ -55,6 +59,17 @@ void Timer::Describe(ODesc* d) const unsigned int TimerMgr::current_timers[NUM_TIMER_TYPES]; +TimerMgr::TimerMgr(const std::string& arg_tag) + { + t = 0.0; + num_expired = 0; + last_advance = last_timestamp = 0; + tag = arg_tag; + + if ( iosource_mgr ) + iosource_mgr->Register(this, true); + } + TimerMgr::~TimerMgr() { DBG_LOG(DBG_TM, "deleting timer mgr %p", this); @@ -74,8 +89,28 @@ int TimerMgr::Advance(double arg_t, int max_expire) return DoAdvance(t, max_expire); } +void TimerMgr::Process() + { + // If we don't have a source, or the source is closed, or we're reading live (which includes + // pseudo-realtime), advance the timer here to the current time since otherwise it won't + // move forward and the timers won't fire correctly. + iosource::PktSrc* pkt_src = iosource_mgr->GetPktSrc(); + if ( ! pkt_src || ! pkt_src->IsOpen() || reading_live ) + net_update_time(current_time()); -PQ_TimerMgr::PQ_TimerMgr(const Tag& tag) : TimerMgr(tag) + // Just advance the timer manager based on the current network time. This won't actually + // change the time, but will dispatch any timers that need dispatching. + current_dispatched += Advance(network_time, max_timer_expires - current_dispatched); + } + +void TimerMgr::InitPostScript() + { + if ( iosource_mgr ) + iosource_mgr->Register(this, true); + } + + +PQ_TimerMgr::PQ_TimerMgr(const std::string& tag) : TimerMgr(tag) { q = new PriorityQueue; } @@ -145,3 +180,12 @@ void PQ_TimerMgr::Remove(Timer* timer) --current_timers[timer->Type()]; delete timer; } + +double PQ_TimerMgr::GetNextTimeout() + { + Timer* top = Top(); + if ( top ) + return std::max(0.0, top->Time() - ::network_time); + + return -1; + } diff --git a/src/Timer.h b/src/Timer.h index 21b21fc5ee..8f73041416 100644 --- a/src/Timer.h +++ b/src/Timer.h @@ -2,10 +2,9 @@ #pragma once -#include - #include #include "PriorityQueue.h" +#include "iosource/IOSource.h" // If you add a timer here, adjust TimerNames in Timer.cc. enum TimerType : uint8_t { @@ -48,10 +47,10 @@ class ODesc; class Timer : public PQ_Element { public: - Timer(double t, TimerType arg_type) : PQ_Element(t), type(arg_type) { } + Timer(double t, TimerType arg_type) : PQ_Element(t), type(arg_type) {} ~Timer() override { } - TimerType Type() const { return (TimerType) type; } + TimerType Type() const { return type; } // t gives the dispatch time. is_expire is true if the // timer is being dispatched because we're expiring all @@ -65,53 +64,73 @@ protected: TimerType type; }; -class TimerMgr { +class TimerMgr : public iosource::IOSource { public: virtual ~TimerMgr(); virtual void Add(Timer* timer) = 0; - // Advance the clock to time t, expiring at most max_expire timers. - // Returns number of timers expired. + /** + * Advance the clock to time t, expiring at most max_expire timers. + * + * @param t the new time. + * @param max_expire the maximum number of timers to expire. + * @return the number of timers expired. + */ int Advance(double t, int max_expire); - // Returns the number of timers expired (so far) during the current - // or most recent advance. + /** + * Returns the number of timers expired (so far) during the current + * or most recent advance. + */ int NumExpiredDuringCurrentAdvance() { return num_expired; } - // Expire all timers. + /** + * Expire all timers. + */ virtual void Expire() = 0; - // Cancel() is a method separate from Remove because - // (1) Remove is protected, but, more importantly, (2) in some - // timer schemes we have wound up separating timer cancelation - // from removing it from the manager's data structures, because - // the manager lacked an efficient way to find it. + /** + * Removes a timer. Cancel() is a method separate from Remove() + * because (1) Remove is protected, but, more importantly, (2) + * in some timer schemes we have wound up separating timer + * cancelation from removing it from the manager's data structures, + * because the manager lacked an efficient way to find it. + * + * @param timer the timer to cancel + */ void Cancel(Timer* timer) { Remove(timer); } double Time() const { return t ? t : 1; } // 1 > 0 - typedef std::string Tag; - const Tag& GetTag() const { return tag; } + const std::string& GetTag() const { return tag; } virtual int Size() const = 0; virtual int PeakSize() const = 0; virtual uint64_t CumulativeNum() const = 0; double LastTimestamp() const { return last_timestamp; } - // Returns time of last advance in global network time. + + /** + * Returns time of last advance in global network time + */ double LastAdvance() const { return last_advance; } static unsigned int* CurrentTimers() { return current_timers; } + // IOSource API methods + virtual double GetNextTimeout() override { return -1; } + virtual void Process() override; + virtual const char* Tag() override { return fmt("TimerMgr %s", tag.c_str()); } + + /** + * Performs some extra initialization on a timer manager. This shouldn't + * need to be called for managers other than the global one. + */ + void InitPostScript(); + protected: - explicit TimerMgr(const Tag& arg_tag) - { - t = 0.0; - num_expired = 0; - last_advance = last_timestamp = 0; - tag = arg_tag; - } + explicit TimerMgr(const std::string& arg_tag); virtual int DoAdvance(double t, int max_expire) = 0; virtual void Remove(Timer* timer) = 0; @@ -119,7 +138,7 @@ protected: double t; double last_timestamp; double last_advance; - Tag tag; + std::string tag; int num_expired; @@ -128,7 +147,7 @@ protected: class PQ_TimerMgr : public TimerMgr { public: - explicit PQ_TimerMgr(const Tag& arg_tag); + explicit PQ_TimerMgr(const std::string& arg_tag); ~PQ_TimerMgr() override; void Add(Timer* timer) override; @@ -137,6 +156,7 @@ public: int Size() const override { return q->Size(); } int PeakSize() const override { return q->PeakSize(); } uint64_t CumulativeNum() const override { return q->CumulativeNum(); } + double GetNextTimeout() override; protected: int DoAdvance(double t, int max_expire) override; diff --git a/src/main.cc b/src/main.cc index 72a1b5ce79..6e9ad81d05 100644 --- a/src/main.cc +++ b/src/main.cc @@ -294,12 +294,10 @@ void terminate_bro() plugin_mgr->FinishPlugins(); delete zeekygen_mgr; - delete timer_mgr; delete event_registry; delete analyzer_mgr; delete file_mgr; - // broker_mgr is deleted via iosource_mgr - // supervisor is deleted via iosource_mgr + // broker_mgr, timer_mgr, and supervisor are deleted via iosource_mgr delete iosource_mgr; delete log_mgr; delete reporter; @@ -550,7 +548,6 @@ int main(int argc, char** argv) #endif timer_mgr = new PQ_TimerMgr(""); - // timer_mgr = new CQ_TimerMgr(); auto zeekygen_cfg = options.zeekygen_config_file.value_or(""); zeekygen_mgr = new zeekygen::Manager(zeekygen_cfg, bro_argv[0]); @@ -671,6 +668,7 @@ int main(int argc, char** argv) plugin_mgr->InitPostScript(); zeekygen_mgr->InitPostScript(); broker_mgr->InitPostScript(); + timer_mgr->InitPostScript(); if ( options.print_plugins ) { From a159d075cfab3b9e84e61c53c8fca176ba14c0f2 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 26 Nov 2019 12:47:33 -0700 Subject: [PATCH 22/53] Add Trigger manager for managing triggers created by things like 'when' statements - Adds new trigger namespace - Adds trigger::Manager class as a new IOSource for keeping track of triggers and integrating them into the loop. Previously the loop relied on the event manager Drain() method to process all triggers on every loop, but now that the loop actively waits for events to occur, triggers would not fire when they needed to. Adding them as part of the loop ensures they're checked. --- src/Event.cc | 7 ++- src/Expr.cc | 2 +- src/Frame.cc | 2 +- src/Frame.h | 8 +-- src/Stats.cc | 4 +- src/Stmt.cc | 3 +- src/Trigger.cc | 125 ++++++++++++++++++++++++++------------------- src/Trigger.h | 56 ++++++++++++-------- src/broker/Store.h | 4 +- src/main.cc | 3 ++ src/zeek.bif | 10 ++-- 11 files changed, 131 insertions(+), 93 deletions(-) diff --git a/src/Event.cc b/src/Event.cc index d461d4e670..8e8db10ed9 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -162,10 +162,9 @@ void EventMgr::Drain() // do after draining events. draining = false; - // We evaluate Triggers here. While this is somewhat unrelated to event - // processing, we ensure that it's done at a regular basis by checking - // them here. - Trigger::EvaluatePending(); + // Make sure all of the triggers get processed every time the events + // drain. + trigger_mgr->Process(); } void EventMgr::Describe(ODesc* d) const diff --git a/src/Expr.cc b/src/Expr.cc index f5db8ff0fb..1049f12929 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -4258,7 +4258,7 @@ Val* CallExpr::Eval(Frame* f) const // Check for that. if ( f ) { - Trigger* trigger = f->GetTrigger(); + trigger::Trigger* trigger = f->GetTrigger(); if ( trigger ) { diff --git a/src/Frame.cc b/src/Frame.cc index 249d988b7b..369e28db47 100644 --- a/src/Frame.cc +++ b/src/Frame.cc @@ -515,7 +515,7 @@ void Frame::CaptureClosure(Frame* c, id_list arg_outer_ids) // if (c) closure = c->SelectiveClone(outer_ids); } -void Frame::SetTrigger(Trigger* arg_trigger) +void Frame::SetTrigger(trigger::Trigger* arg_trigger) { ClearTrigger(); diff --git a/src/Frame.h b/src/Frame.h index 133ab45978..21f411ba1f 100644 --- a/src/Frame.h +++ b/src/Frame.h @@ -11,7 +11,7 @@ #include "Val.h" -class Trigger; +namespace trigger { class Trigger; } class CallExpr; class Frame : public BroObj { @@ -207,9 +207,9 @@ public: // If the frame is run in the context of a trigger condition evaluation, // the trigger needs to be registered. - void SetTrigger(Trigger* arg_trigger); + void SetTrigger(trigger::Trigger* arg_trigger); void ClearTrigger(); - Trigger* GetTrigger() const { return trigger; } + trigger::Trigger* GetTrigger() const { return trigger; } void SetCall(const CallExpr* arg_call) { call = arg_call; } void ClearCall() { call = 0; } @@ -293,7 +293,7 @@ private: bool break_before_next_stmt; bool break_on_return; - Trigger* trigger; + trigger::Trigger* trigger; const CallExpr* call; bool delayed; diff --git a/src/Stats.cc b/src/Stats.cc index d352d19793..fed8a5259a 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -188,8 +188,8 @@ void ProfileLogger::Log() dstats.requests, dstats.successful, dstats.failed, dstats.pending, dstats.cached_hosts, dstats.cached_addresses)); - Trigger::Stats tstats; - Trigger::GetStats(&tstats); + trigger::Manager::Stats tstats; + trigger_mgr->GetStats(&tstats); file->Write(fmt("%.06f Triggers: total=%lu pending=%lu\n", network_time, tstats.total, tstats.pending)); diff --git a/src/Stmt.cc b/src/Stmt.cc index d777e61b97..945bd80faf 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -1790,7 +1790,7 @@ Val* WhenStmt::Exec(Frame* f, stmt_flow_type& flow) const ::Ref(timeout); // The new trigger object will take care of its own deletion. - new Trigger(cond, s1, s2, timeout, f, is_return, location); + new trigger::Trigger(cond, s1, s2, timeout, f, is_return, location); return 0; } @@ -1857,4 +1857,3 @@ TraversalCode WhenStmt::Traverse(TraversalCallback* cb) const tc = cb->PostStmt(this); HANDLE_TC_STMT_POST(tc); } - diff --git a/src/Trigger.cc b/src/Trigger.cc index 3df2e9e43e..271edf24aa 100644 --- a/src/Trigger.cc +++ b/src/Trigger.cc @@ -2,10 +2,15 @@ #include "Trigger.h" #include "Traverse.h" +#include "iosource/Manager.h" + +using namespace trigger; // Callback class to traverse an expression, registering all relevant IDs and // Vals for change notifications. +namespace trigger { + class TriggerTraversalCallback : public TraversalCallback { public: TriggerTraversalCallback(Trigger *arg_trigger) @@ -20,6 +25,8 @@ private: Trigger* trigger; }; +} + TraversalCode TriggerTraversalCallback::PreExpr(const Expr* expr) { // We catch all expressions here which in some way reference global @@ -67,6 +74,8 @@ TraversalCode TriggerTraversalCallback::PreExpr(const Expr* expr) return TC_CONTINUE; } +namespace trigger { + class TriggerTimer : public Timer { public: TriggerTimer(double arg_timeout, Trigger* arg_trigger) @@ -102,13 +111,12 @@ protected: double time; }; +} + Trigger::Trigger(Expr* arg_cond, Stmt* arg_body, Stmt* arg_timeout_stmts, Expr* arg_timeout, Frame* arg_frame, bool arg_is_return, const Location* arg_location) { - if ( ! pending ) - pending = new list; - cond = arg_cond; body = arg_body; timeout_stmts = arg_timeout_stmts; @@ -122,8 +130,6 @@ Trigger::Trigger(Expr* arg_cond, Stmt* arg_body, Stmt* arg_timeout_stmts, location = arg_location; timeout_value = -1; - ++total_triggers; - DBG_LOG(DBG_NOTIFIERS, "%s: instantiating", Name()); if ( is_return ) @@ -215,9 +221,6 @@ void Trigger::Init() cond->Traverse(&cb); } -Trigger::TriggerList* Trigger::pending = 0; -unsigned long Trigger::total_triggers = 0; - bool Trigger::Eval() { if ( disabled ) @@ -330,47 +333,6 @@ bool Trigger::Eval() return true; } -void Trigger::QueueTrigger(Trigger* trigger) - { - assert(! trigger->disabled); - assert(pending); - if ( std::find(pending->begin(), pending->end(), trigger) == pending->end() ) - { - Ref(trigger); - pending->push_back(trigger); - } - } - -void Trigger::EvaluatePending() - { - DBG_LOG(DBG_NOTIFIERS, "evaluating all pending triggers"); - - if ( ! pending ) - return; - - // While we iterate over the list, executing statements, we may - // in fact trigger new triggers and thereby modify the list. - // Therefore, we create a new temporary list which will receive - // triggers triggered during this time. - TriggerList* orig = pending; - TriggerList tmp; - pending = &tmp; - - for ( TriggerList::iterator i = orig->begin(); i != orig->end(); ++i ) - { - Trigger* t = *i; - (*i)->Eval(); - Unref(t); - } - - pending = orig; - orig->clear(); - - // Sigh... Is this really better than a for-loop? - std::copy(tmp.begin(), tmp.end(), - insert_iterator(*pending, pending->begin())); - } - void Trigger::Timeout() { if ( disabled ) @@ -484,7 +446,7 @@ void Trigger::Cache(const CallExpr* expr, Val* v) Ref(v); - QueueTrigger(this); + trigger_mgr->Queue(this); } @@ -502,6 +464,11 @@ void Trigger::Disable() disabled = true; } +void Trigger::Modified(notifier::Modifiable* m) + { + trigger_mgr->Queue(this); + } + const char* Trigger::Name() const { assert(location); @@ -509,8 +476,62 @@ const char* Trigger::Name() const location->first_line, location->last_line); } -void Trigger::GetStats(Stats* stats) + + +Manager::Manager() : IOSource() + { + pending = new TriggerList(); + iosource_mgr->Register(this, true); + } + +Manager::~Manager() + { + delete pending; + } + +double Manager::GetNextTimeout() + { + return pending->empty() ? -1 : network_time + 0.100; + } + +void Manager::Process() + { + DBG_LOG(DBG_NOTIFIERS, "evaluating all pending triggers"); + + // While we iterate over the list, executing statements, we may + // in fact trigger new triggers and thereby modify the list. + // Therefore, we create a new temporary list which will receive + // triggers triggered during this time. + TriggerList* orig = pending; + TriggerList tmp; + pending = &tmp; + + for ( TriggerList::iterator i = orig->begin(); i != orig->end(); ++i ) + { + Trigger* t = *i; + (*i)->Eval(); + Unref(t); + } + + pending = orig; + orig->clear(); + + std::swap(tmp, *pending); + } + +void Manager::Queue(Trigger* trigger) + { + if ( std::find(pending->begin(), pending->end(), trigger) == pending->end() ) + { + Ref(trigger); + pending->push_back(trigger); + total_triggers++; + iosource_mgr->Wakeup(Tag()); + } + } + +void Manager::GetStats(Stats* stats) { stats->total = total_triggers; - stats->pending = pending ? pending->size() : 0; + stats->pending = pending->size(); } diff --git a/src/Trigger.h b/src/Trigger.h index e9fcc087a2..c3efd62138 100644 --- a/src/Trigger.h +++ b/src/Trigger.h @@ -5,7 +5,9 @@ #include "Notifier.h" #include "Traverse.h" +#include "iosource/IOSource.h" +namespace trigger { // Triggers are the heart of "when" statements: expressions that when // they become true execute a body of statements. @@ -58,10 +60,11 @@ public: void Describe(ODesc* d) const override { d->Add(""); } + // Overidden from Notifier. We queue the trigger and evaluate it // later to avoid race conditions. - void Modified(notifier::Modifiable* m) override - { QueueTrigger(this); } + void Modified(notifier::Modifiable* m) override; + // Overridden from notifer::Receiver. If we're still waiting // on an ID/Val to be modified at termination time, we can't hope // for any further progress to be made, so just Unref ourselves. @@ -69,18 +72,6 @@ public: const char* Name() const; - static void QueueTrigger(Trigger* trigger); - - // Evaluates all queued Triggers. - static void EvaluatePending(); - - struct Stats { - unsigned long total; - unsigned long pending; - }; - - static void GetStats(Stats* stats); - private: friend class TriggerTraversalCallback; friend class TriggerTimer; @@ -107,11 +98,36 @@ private: std::vector> objs; - typedef map ValCache; + using ValCache = map; ValCache cache; - - typedef list TriggerList; - static TriggerList* pending; - - static unsigned long total_triggers; }; + +class Manager : public iosource::IOSource { +public: + + Manager(); + ~Manager(); + + double GetNextTimeout() override; + void Process() override; + const char* Tag() override { return "TriggerMgr"; } + + void Queue(Trigger* trigger); + + struct Stats { + unsigned long total; + unsigned long pending; + }; + + void GetStats(Stats* stats); + +private: + + using TriggerList = std::list; + TriggerList* pending; + unsigned long total_triggers = 0; + }; + +} + +extern trigger::Manager* trigger_mgr; diff --git a/src/broker/Store.h b/src/broker/Store.h index e1a20775f0..0ba3d2ad5d 100644 --- a/src/broker/Store.h +++ b/src/broker/Store.h @@ -66,7 +66,7 @@ inline RecordVal* query_result(RecordVal* data) */ class StoreQueryCallback { public: - StoreQueryCallback(Trigger* arg_trigger, const CallExpr* arg_call, + StoreQueryCallback(trigger::Trigger* arg_trigger, const CallExpr* arg_call, broker::store store) : trigger(arg_trigger), call(arg_call), store(move(store)) { @@ -101,7 +101,7 @@ public: private: - Trigger* trigger; + trigger::Trigger* trigger; const CallExpr* call; broker::store store; }; diff --git a/src/main.cc b/src/main.cc index 6e9ad81d05..00ad77a2e6 100644 --- a/src/main.cc +++ b/src/main.cc @@ -41,6 +41,7 @@ extern "C" { #include "Stats.h" #include "Brofiler.h" #include "Traverse.h" +#include "Trigger.h" #include "supervisor/Supervisor.h" #include "threading/Manager.h" @@ -93,6 +94,7 @@ zeekygen::Manager* zeekygen_mgr = 0; iosource::Manager* iosource_mgr = 0; bro_broker::Manager* broker_mgr = 0; zeek::Supervisor* zeek::supervisor_mgr = 0; +trigger::Manager* trigger_mgr = 0; std::vector zeek_script_prefixes; Stmt* stmts; @@ -594,6 +596,7 @@ int main(int argc, char** argv) input_mgr = new input::Manager(); file_mgr = new file_analysis::Manager(); broker_mgr = new bro_broker::Manager(! options.pcap_file.empty()); + trigger_mgr = new trigger::Manager(); plugin_mgr->InitPreScript(); analyzer_mgr->InitPreScript(); diff --git a/src/zeek.bif b/src/zeek.bif index 89f181601f..a42491daee 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -3503,7 +3503,7 @@ function dump_packet%(pkt: pcap_packet, file_name: string%) : bool class LookupHostCallback : public DNS_Mgr::LookupCallback { public: - LookupHostCallback(Trigger* arg_trigger, const CallExpr* arg_call, + LookupHostCallback(trigger::Trigger* arg_trigger, const CallExpr* arg_call, bool arg_lookup_name) { Ref(arg_trigger); @@ -3556,7 +3556,7 @@ public: } private: - Trigger* trigger; + trigger::Trigger* trigger; const CallExpr* call; bool lookup_name; }; @@ -3575,7 +3575,7 @@ function lookup_addr%(host: addr%) : string %{ // FIXME: It should be easy to adapt the function to synchronous // lookups if we're reading a trace. - Trigger* trigger = frame->GetTrigger(); + trigger::Trigger* trigger = frame->GetTrigger(); if ( ! trigger) { @@ -3604,7 +3604,7 @@ function lookup_hostname_txt%(host: string%) : string %{ // FIXME: Is should be easy to adapt the function to synchronous // lookups if we're reading a trace. - Trigger* trigger = frame->GetTrigger(); + trigger::Trigger* trigger = frame->GetTrigger(); if ( ! trigger) { @@ -3633,7 +3633,7 @@ function lookup_hostname%(host: string%) : addr_set %{ // FIXME: Is should be easy to adapt the function to synchronous // lookups if we're reading a trace. - Trigger* trigger = frame->GetTrigger(); + trigger::Trigger* trigger = frame->GetTrigger(); if ( ! trigger) { From 8b9160fb7e3046fea0e91393cff5ed3641e2be81 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 26 Nov 2019 12:54:51 -0700 Subject: [PATCH 23/53] Threading changes for the new loop architecture - threading::Manager is no longer an IOSource. - threading::MsgThread is now an IOSource. This allows threads themselves to signal when they have data to process instead of continually checking each of the threads on every loop pass. - Make the thread heartbeat timer an actual timer and let it fire as necessary instead of checking to see if it should fire --- src/Timer.cc | 1 + src/Timer.h | 3 +- src/threading/Manager.cc | 100 ++++++++++++++++++++++--------------- src/threading/Manager.h | 44 +++++++++------- src/threading/MsgThread.cc | 40 +++++++++++++-- src/threading/MsgThread.h | 19 ++++++- 6 files changed, 141 insertions(+), 66 deletions(-) diff --git a/src/Timer.cc b/src/Timer.cc index a16ee3bf33..686eca9e68 100644 --- a/src/Timer.cc +++ b/src/Timer.cc @@ -43,6 +43,7 @@ const char* TimerNames[] = { "TriggerTimer", "ParentProcessIDCheck", "TimerMgrExpireTimer", + "ThreadHeartbeat", }; const char* timer_type_to_string(TimerType type) diff --git a/src/Timer.h b/src/Timer.h index 8f73041416..6ad7cda003 100644 --- a/src/Timer.h +++ b/src/Timer.h @@ -38,8 +38,9 @@ enum TimerType : uint8_t { TIMER_TRIGGER, TIMER_PPID_CHECK, TIMER_TIMERMGR_EXPIRE, + TIMER_THREAD_HEARTBEAT, }; -const int NUM_TIMER_TYPES = int(TIMER_TIMERMGR_EXPIRE) + 1; +const int NUM_TIMER_TYPES = int(TIMER_THREAD_HEARTBEAT) + 1; extern const char* timer_type_to_string(TimerType type); diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index b07ab0d3d5..a8d721b76c 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -1,33 +1,44 @@ +#include +#include #include "Manager.h" #include "NetVar.h" +#include "iosource/Manager.h" using namespace threading; +void HeartbeatTimer::Dispatch(double t, int is_expire) + { + if ( is_expire ) + return; + + thread_mgr->SendHeartbeats(); + thread_mgr->StartHeartbeatTimer(); + } + Manager::Manager() { DBG_LOG(DBG_THREADING, "Creating thread manager ..."); did_process = true; next_beat = 0; - terminating = false; - SetIdle(true); } Manager::~Manager() { if ( all_threads.size() ) Terminate(); + + if ( heartbeat_timer ) + delete heartbeat_timer; } void Manager::Terminate() { DBG_LOG(DBG_THREADING, "Terminating thread manager ..."); - terminating = true; - // First process remaining thread output for the message threads. - do Process(); while ( did_process ); + do Flush(); while ( did_process ); // Signal all to stop. @@ -46,17 +57,15 @@ void Manager::Terminate() all_threads.clear(); msg_threads.clear(); - - SetIdle(true); - SetClosed(true); - terminating = false; } void Manager::AddThread(BasicThread* thread) { DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name()); all_threads.push_back(thread); - SetIdle(false); + + if ( ! heartbeat_timer ) + StartHeartbeatTimer(); } void Manager::AddMsgThread(MsgThread* thread) @@ -65,34 +74,6 @@ void Manager::AddMsgThread(MsgThread* thread) msg_threads.push_back(thread); } -void Manager::GetFds(iosource::FD_Set* read, iosource::FD_Set* write, - iosource::FD_Set* except) - { - } - -double Manager::NextTimestamp(double* network_time) - { -// fprintf(stderr, "N %.6f %.6f did_process=%d next_next=%.6f\n", ::network_time, timer_mgr->Time(), (int)did_process, next_beat); - - if ( ::network_time && (did_process || ::network_time > next_beat || ! next_beat) ) - // If we had something to process last time (or out heartbeat - // is due or not set yet), we want to check for more asap. - return timer_mgr->Time(); - - for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ ) - { - MsgThread* t = *i; - - if ( t->MightHaveOut() || t->Killed() ) - // Even if the thread doesn't have output, it may be killed/done, - // which should also signify that processing is needed. The - // "processing" in that case is joining the thread and deleting it. - return timer_mgr->Time(); - } - - return -1.0; - } - void Manager::KillThreads() { DBG_LOG(DBG_THREADING, "Killing threads ..."); @@ -107,7 +88,46 @@ void Manager::KillThread(BasicThread* thread) thread->Kill(); } -void Manager::Process() +void Manager::SendHeartbeats() + { + for ( MsgThread* thread : msg_threads ) + thread->Heartbeat(); + + // Since this is a regular timer, this is also an ideal place to check whether we have + // and dead threads and to delete them. + all_thread_list to_delete; + for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ ) + { + BasicThread* t = *i; + + if ( t->Killed() ) + to_delete.push_back(t); + } + + for ( all_thread_list::iterator i = to_delete.begin(); i != to_delete.end(); i++ ) + { + BasicThread* t = *i; + t->WaitForStop(); + + all_threads.remove(t); + + MsgThread* mt = dynamic_cast(t); + + if ( mt ) + msg_threads.remove(mt); + + t->Join(); + delete t; + } + } + +void Manager::StartHeartbeatTimer() + { + heartbeat_timer = new HeartbeatTimer(network_time + BifConst::Threading::heartbeat_interval); + timer_mgr->Add(heartbeat_timer); + } + +void Manager::Flush() { bool do_beat = false; @@ -192,5 +212,3 @@ const threading::Manager::msg_stats_list& threading::Manager::GetMsgThreadStats( return stats; } - - diff --git a/src/threading/Manager.h b/src/threading/Manager.h index bf2cb429b4..132d2238c4 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -3,13 +3,25 @@ #include -#include "iosource/IOSource.h" - #include "BasicThread.h" #include "MsgThread.h" +#include "Timer.h" namespace threading { +class HeartbeatTimer : public Timer { +public: + HeartbeatTimer(double t) : Timer(t, TIMER_THREAD_HEARTBEAT) {} + virtual ~HeartbeatTimer() {} + + void Dispatch(double t, int is_expire); + +protected: + + void Init(); + int do_expire; +}; + /** * The thread manager coordinates all child threads. Once a BasicThread is * instantitated, it gets addedd to the manager, which will delete it later @@ -20,7 +32,7 @@ namespace threading { * their outgoing message queue on a regular basis and feeds data sent into * the rest of Bro. It also triggers the regular heartbeats. */ -class Manager : public iosource::IOSource +class Manager { public: /** @@ -30,9 +42,9 @@ public: Manager(); /** - * Destructir. + * Destructor. */ - ~Manager() override; + ~Manager(); /** * Terminates the manager's processor. The method signals all threads @@ -80,6 +92,7 @@ public: protected: friend class BasicThread; friend class MsgThread; + friend class HeartbeatTimer; /** * Registers a new basic thread with the manager. This is @@ -99,26 +112,17 @@ protected: */ void AddMsgThread(MsgThread* thread); - /** - * Part of the IOSource interface. - */ - void GetFds(iosource::FD_Set* read, iosource::FD_Set* write, - iosource::FD_Set* except) override; + void Flush(); /** - * Part of the IOSource interface. + * Sends heartbeat messages to all active message threads. */ - double NextTimestamp(double* network_time) override; + void SendHeartbeats(); /** - * Part of the IOSource interface. + * Sets up a timer to periodically send heartbeat messages to all threads. */ - void Process() override; - - /** - * Part of the IOSource interface. - */ - const char* Tag() override { return "threading::Manager"; } + void StartHeartbeatTimer(); private: typedef std::list all_thread_list; @@ -132,6 +136,8 @@ private: bool terminating; // True if we are in Terminate(). msg_stats_list stats; + + HeartbeatTimer* heartbeat_timer = nullptr; }; } diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index 01f90921e8..41267147d0 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -1,11 +1,12 @@ +#include +#include +#include #include "DebugLogger.h" #include "MsgThread.h" #include "Manager.h" - -#include -#include +#include "iosource/Manager.h" using namespace threading; @@ -179,6 +180,17 @@ MsgThread::MsgThread() : BasicThread(), queue_in(this, 0), queue_out(0, this) child_sent_finish = false; failed = false; thread_mgr->AddMsgThread(this); + + iosource_mgr->RegisterFd(flare.FD(), this); + + SetClosed(false); + } + +MsgThread::~MsgThread() + { + // Unregister this thread from the iosource manager so it doesn't wake + // up the main poll anymore. + iosource_mgr->UnregisterFd(flare.FD()); } // Set by Bro's main signal handler. @@ -252,6 +264,8 @@ void MsgThread::OnWaitForStop() void MsgThread::OnKill() { + SetClosed(true); + // Send a message to unblock the reader if its currently waiting for // input. This is just an optimization to make it terminate more // quickly, even without the message it will eventually time out. @@ -344,6 +358,8 @@ void MsgThread::SendOut(BasicOutputMessage* msg, bool force) queue_out.Put(msg); ++cnt_sent_out; + + flare.Fire(); } BasicOutputMessage* MsgThread::RetrieveOut() @@ -418,3 +434,21 @@ void MsgThread::GetStats(Stats* stats) queue_out.GetStats(&stats->queue_out_stats); } +void MsgThread::Process() + { + flare.Extinguish(); + + while ( HasOut() ) + { + Message* msg = RetrieveOut(); + assert(msg); + + if ( ! msg->Process() ) + { + reporter->Error("%s failed, terminating thread", msg->Name()); + SignalStop(); + } + + delete msg; + } + } diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index 1f5ec0e017..b920410e73 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -5,6 +5,8 @@ #include "BasicThread.h" #include "Queue.h" +#include "iosource/IOSource.h" +#include "Flare.h" namespace threading { @@ -23,7 +25,7 @@ class HeartbeatMessage; * that happens, the thread stops accepting any new messages, finishes * processes all remaining ones still in the queue, and then exits. */ -class MsgThread : public BasicThread +class MsgThread : public BasicThread, public iosource::IOSource { public: /** @@ -34,6 +36,11 @@ public: */ MsgThread(); + /** + * Destructor. + */ + virtual ~MsgThread(); + /** * Sends a message to the child thread. The message will be proceesed * once the thread has retrieved it from its incoming queue. @@ -175,6 +182,13 @@ public: */ void GetStats(Stats* stats); + /** + * Overridden from iosource::IOSource. + */ + void Process() override; + const char* Tag() override { return Name(); } + double GetNextTimeout() override { return -1; } + protected: friend class Manager; friend class HeartbeatMessage; @@ -229,7 +243,6 @@ protected: /** * Overriden from BasicThread. - * */ void Run() override; void OnWaitForStop() override; @@ -308,6 +321,8 @@ private: bool child_finished; // Child thread is finished. bool child_sent_finish; // Child thread asked to be finished. bool failed; // Set to true when a command failed. + + bro::Flare flare; }; /** From c4d9566294918bf8c0c5636769d3e68bfed9f8ee Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 26 Nov 2019 12:58:41 -0700 Subject: [PATCH 24/53] DNS_Mgr changes to match the new IOSource API and loop architecture --- src/DNS_Mgr.cc | 63 +++++++++++++++++--------------------------------- src/DNS_Mgr.h | 14 ++++------- src/main.cc | 1 + 3 files changed, 27 insertions(+), 51 deletions(-) diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 5599074b0d..903485a652 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -392,7 +392,6 @@ DNS_Mgr::DNS_Mgr(DNS_MgrMode arg_mode) successful = 0; failed = 0; nb_dns = nullptr; - next_timestamp = -1.0; } DNS_Mgr::~DNS_Mgr() @@ -404,7 +403,7 @@ DNS_Mgr::~DNS_Mgr() delete [] dir; } -void DNS_Mgr::Init() +void DNS_Mgr::InitSource() { if ( did_init ) return; @@ -440,7 +439,9 @@ void DNS_Mgr::Init() nb_dns = nb_dns_init2(err, (struct sockaddr*)&ss); } - if ( ! nb_dns ) + if ( nb_dns ) + iosource_mgr->RegisterFd(nb_dns_fd(nb_dns), this); + else reporter->Warning("problem initializing NB-DNS: %s", err); did_init = true; @@ -460,11 +461,6 @@ void DNS_Mgr::InitPostScript() // Registering will call Init() iosource_mgr->Register(this, true); - // We never set idle to false, having the main loop only calling us from - // time to time. If we're issuing more DNS requests than we can handle - // in this way, we are having problems anyway ... - SetIdle(true); - const char* cache_dir = dir ? dir : "."; cache_name = new char[strlen(cache_dir) + 64]; sprintf(cache_name, "%s/%s", cache_dir, ".zeek-dns-cache"); @@ -503,7 +499,7 @@ TableVal* DNS_Mgr::LookupHost(const char* name) if ( mode == DNS_FAKE ) return fake_name_lookup_result(name); - Init(); + InitSource(); if ( ! nb_dns ) return empty_addr_set(); @@ -558,7 +554,7 @@ TableVal* DNS_Mgr::LookupHost(const char* name) Val* DNS_Mgr::LookupAddr(const IPAddr& addr) { - Init(); + InitSource(); if ( mode != DNS_PRIME ) { @@ -1078,7 +1074,7 @@ static void resolve_lookup_cb(DNS_Mgr::LookupCallback* callback, void DNS_Mgr::AsyncLookupAddr(const IPAddr& host, LookupCallback* callback) { - Init(); + InitSource(); if ( mode == DNS_FAKE ) { @@ -1116,7 +1112,7 @@ void DNS_Mgr::AsyncLookupAddr(const IPAddr& host, LookupCallback* callback) void DNS_Mgr::AsyncLookupName(const string& name, LookupCallback* callback) { - Init(); + InitSource(); if ( mode == DNS_FAKE ) { @@ -1154,7 +1150,7 @@ void DNS_Mgr::AsyncLookupName(const string& name, LookupCallback* callback) void DNS_Mgr::AsyncLookupNameText(const string& name, LookupCallback* callback) { - Init(); + InitSource(); if ( mode == DNS_FAKE ) { @@ -1242,30 +1238,6 @@ void DNS_Mgr::IssueAsyncRequests() } } -void DNS_Mgr::GetFds(iosource::FD_Set* read, iosource::FD_Set* write, - iosource::FD_Set* except) - { - if ( ! nb_dns ) - return; - - read->Insert(nb_dns_fd(nb_dns)); - } - -double DNS_Mgr::NextTimestamp(double* network_time) - { - if ( asyncs_timeouts.empty() ) - // No pending requests. - return -1.0; - - if ( next_timestamp < 0 ) - // Store the timestamp to help prevent starvation by some other - // IOSource always trying to use the same timestamp - // (assuming network_time does actually increase). - next_timestamp = timer_mgr->Time(); - - return next_timestamp; - } - void DNS_Mgr::CheckAsyncAddrRequest(const IPAddr& addr, bool timeout) { // Note that this code is a mirror of that for CheckAsyncHostRequest. @@ -1369,7 +1341,7 @@ void DNS_Mgr::CheckAsyncHostRequest(const char* host, bool timeout) void DNS_Mgr::Flush() { - DoProcess(); + Process(); HostMap::iterator it; for ( it = host_mappings.begin(); it != host_mappings.end(); ++it ) @@ -1389,13 +1361,15 @@ void DNS_Mgr::Flush() text_mappings.clear(); } -void DNS_Mgr::Process() +double DNS_Mgr::GetNextTimeout() { - DoProcess(); - next_timestamp = -1.0; + if ( asyncs_timeouts.empty() ) + return -1; + + return network_time + DNS_TIMEOUT; } -void DNS_Mgr::DoProcess() +void DNS_Mgr::Process() { if ( ! nb_dns ) return; @@ -1513,3 +1487,8 @@ void DNS_Mgr::GetStats(Stats* stats) stats->cached_texts = text_mappings.size(); } +void DNS_Mgr::Terminate() + { + if ( nb_dns ) + iosource_mgr->UnregisterFd(nb_dns_fd(nb_dns)); + } diff --git a/src/DNS_Mgr.h b/src/DNS_Mgr.h index 00397f351f..6b5975f876 100644 --- a/src/DNS_Mgr.h +++ b/src/DNS_Mgr.h @@ -91,6 +91,8 @@ public: void GetStats(Stats* stats); + void Terminate(); + protected: friend class LookupCallback; friend class DNS_Mgr_Request; @@ -127,16 +129,11 @@ protected: void CheckAsyncHostRequest(const char* host, bool timeout); void CheckAsyncTextRequest(const char* host, bool timeout); - // Process outstanding requests. - void DoProcess(); - // IOSource interface. - void GetFds(iosource::FD_Set* read, iosource::FD_Set* write, - iosource::FD_Set* except) override; - double NextTimestamp(double* network_time) override; void Process() override; - void Init() override; - const char* Tag() override { return "DNS_Mgr"; } + void InitSource() override; + const char* Tag() override { return "DNS_Mgr"; } + double GetNextTimeout() override; DNS_MgrMode mode; @@ -241,7 +238,6 @@ protected: unsigned long num_requests; unsigned long successful; unsigned long failed; - double next_timestamp; }; extern DNS_Mgr* dns_mgr; diff --git a/src/main.cc b/src/main.cc index 00ad77a2e6..93d3c652c2 100644 --- a/src/main.cc +++ b/src/main.cc @@ -290,6 +290,7 @@ void terminate_bro() input_mgr->Terminate(); thread_mgr->Terminate(); broker_mgr->Terminate(); + dns_mgr->Terminate(); mgr.Drain(); From c5462eaa80a8a03a238adf430dfa2b6950d9f9b2 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 26 Nov 2019 12:59:28 -0700 Subject: [PATCH 25/53] Broker manager changes to match the new IOSource API and loop architecture --- src/broker/Manager.cc | 61 ++++++++++--------------------------------- src/broker/Manager.h | 11 ++------ 2 files changed, 16 insertions(+), 56 deletions(-) diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 392057e0f9..a7ad3a1df6 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -127,15 +127,12 @@ Manager::Manager(bool arg_reading_pcaps) reading_pcaps = arg_reading_pcaps; after_zeek_init = false; peer_count = 0; - times_processed_without_idle = 0; log_batch_size = 0; log_batch_interval = 0; log_topic_func = nullptr; vector_of_data_type = nullptr; log_id_type = nullptr; writer_id_type = nullptr; - - SetIdle(false); } Manager::~Manager() @@ -209,12 +206,18 @@ void Manager::InitPostScript() auto cqs = get_option("Broker::congestion_queue_size")->AsCount(); bstate = std::make_shared(std::move(config), cqs); + + iosource_mgr->RegisterFd(bstate->subscriber.fd(), this); + iosource_mgr->RegisterFd(bstate->status_subscriber.fd(), this); } void Manager::Terminate() { FlushLogBuffers(); + iosource_mgr->UnregisterFd(bstate->subscriber.fd()); + iosource_mgr->UnregisterFd(bstate->status_subscriber.fd()); + vector stores_to_close; for ( auto& x : data_stores ) @@ -275,8 +278,6 @@ void Manager::FlushPendingQueries() } } } - - SetIdle(false); } uint16_t Manager::Listen(const string& addr, uint16_t port) @@ -815,24 +816,6 @@ bool Manager::Unsubscribe(const string& topic_prefix) return true; } -void Manager::GetFds(iosource::FD_Set* read, iosource::FD_Set* write, - iosource::FD_Set* except) - { - read->Insert(bstate->subscriber.fd()); - read->Insert(bstate->status_subscriber.fd()); - - for ( auto& x : data_stores ) - read->Insert(x.second->proxy.mailbox().descriptor()); - } - -double Manager::NextTimestamp(double* local_network_time) - { - // We're only asked for a timestamp if either (1) a FD was ready - // or (2) we're not idle (and we go idle if when Process is no-op), - // so there's no case where returning -1 to signify a skip will help. - return timer_mgr->Time(); - } - void Manager::DispatchMessage(const broker::topic& topic, broker::data msg) { switch ( broker::zeek::Message::type(msg) ) { @@ -885,6 +868,10 @@ void Manager::DispatchMessage(const broker::topic& topic, broker::data msg) void Manager::Process() { + // Ensure that time gets update before processing broker messages, or events + // based on them might get scheduled wrong. + net_update_time(current_time()); + bool had_input = false; auto status_msgs = bstate->status_subscriber.poll(); @@ -949,30 +936,6 @@ void Manager::Process() // network_time, may as well do so now because otherwise the // broker/cluster logs will end up using timestamp 0. net_update_time(current_time()); - - ++times_processed_without_idle; - - // The max number of Process calls allowed to happen in a row without - // idling is chosen a bit arbitrarily, except 12 is around half of the - // SELECT_FREQUENCY (25). - // - // But probably the general idea should be for it to have some relation - // to the SELECT_FREQUENCY: less than it so other busy IOSources can - // fit several Process loops in before the next poll event (e.g. the - // select() call ), but still large enough such that we don't have to - // wait long before the next poll ourselves after being forced to idle. - if ( times_processed_without_idle > 12 ) - { - times_processed_without_idle = 0; - SetIdle(true); - } - else - SetIdle(false); - } - else - { - times_processed_without_idle = 0; - SetIdle(true); } } @@ -1484,6 +1447,7 @@ StoreHandleVal* Manager::MakeMaster(const string& name, broker::backend type, Ref(handle); data_stores.emplace(name, handle); + iosource_mgr->RegisterFd(handle->proxy.mailbox().descriptor(), this); if ( bstate->endpoint.use_real_time() ) return handle; @@ -1520,6 +1484,7 @@ StoreHandleVal* Manager::MakeClone(const string& name, double resync_interval, Ref(handle); data_stores.emplace(name, handle); + iosource_mgr->RegisterFd(handle->proxy.mailbox().descriptor(), this); return handle; } @@ -1538,6 +1503,8 @@ bool Manager::CloseStore(const string& name) if ( s == data_stores.end() ) return false; + iosource_mgr->UnregisterFd(s->second->proxy.mailbox().descriptor()); + for ( auto i = pending_queries.begin(); i != pending_queries.end(); ) if ( i->second->Store().name() == name ) { diff --git a/src/broker/Manager.h b/src/broker/Manager.h index 3b36d189dd..43698a22d3 100644 --- a/src/broker/Manager.h +++ b/src/broker/Manager.h @@ -346,15 +346,9 @@ private: __attribute__((format (printf, 2, 3))); // IOSource interface overrides: - void GetFds(iosource::FD_Set* read, iosource::FD_Set* write, - iosource::FD_Set* except) override; - - double NextTimestamp(double* local_network_time) override; - void Process() override; - - const char* Tag() override - { return "Broker::Manager"; } + const char* Tag() override { return "Broker::Manager"; } + double GetNextTimeout() override { return -1; } struct LogBuffer { // Indexed by topic string. @@ -392,7 +386,6 @@ private: bool reading_pcaps; bool after_zeek_init; int peer_count; - int times_processed_without_idle; size_t log_batch_size; double log_batch_interval; From 4751783d567f43cdb13c24170651fc36b22e83c5 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 26 Nov 2019 12:59:55 -0700 Subject: [PATCH 26/53] PktSrc iosource changes to match the new IOSource API --- src/iosource/PktSrc.cc | 109 ++++++++++++++++++----------------------- src/iosource/PktSrc.h | 16 ++++-- 2 files changed, 60 insertions(+), 65 deletions(-) diff --git a/src/iosource/PktSrc.cc b/src/iosource/PktSrc.cc index 0eae93ac27..c61aebc2d1 100644 --- a/src/iosource/PktSrc.cc +++ b/src/iosource/PktSrc.cc @@ -109,9 +109,20 @@ void PktSrc::Opened(const Properties& arg_props) return; } + if ( props.is_live ) + { Info(fmt("listening on %s\n", props.path.c_str())); + // We only register the file descriptor if we're in live + // mode because libpcap's file descriptor for trace files + // isn't a reliable way to know whether we actually have + // data to read. + if ( props.selectable_fd != -1 ) + if ( ! iosource_mgr->RegisterFd(props.selectable_fd, this) ) + reporter->FatalError("Failed to register pktsrc fd with iosource_mgr"); + } + DBG_LOG(DBG_PKTIO, "Opened source %s", props.path.c_str()); } @@ -119,6 +130,9 @@ void PktSrc::Closed() { SetClosed(true); + if ( props.is_live && props.selectable_fd != -1 ) + iosource_mgr->UnregisterFd(props.selectable_fd, this); + DBG_LOG(DBG_PKTIO, "Closed source %s", props.path.c_str()); } @@ -166,7 +180,7 @@ double PktSrc::CheckPseudoTime() return pseudo_time <= ct ? bro_start_time + pseudo_time : 0; } -void PktSrc::Init() +void PktSrc::InitSource() { Open(); } @@ -177,55 +191,6 @@ void PktSrc::Done() Close(); } -void PktSrc::GetFds(iosource::FD_Set* read, iosource::FD_Set* write, - iosource::FD_Set* except) - { - if ( pseudo_realtime ) - { - // Select would give erroneous results. But we simulate it - // by setting idle accordingly. - SetIdle(CheckPseudoTime() == 0); - return; - } - - if ( IsOpen() && props.selectable_fd >= 0 ) - read->Insert(props.selectable_fd); - - // TODO: This seems like a hack that should be removed, but doing so - // causes the main run loop to spin more frequently and increase cpu usage. - // See also commit 9cd85be308. - if ( read->Empty() ) - read->Insert(0); - - if ( write->Empty() ) - write->Insert(0); - - if ( except->Empty() ) - except->Insert(0); - } - -double PktSrc::NextTimestamp(double* local_network_time) - { - if ( ! IsOpen() ) - return -1.0; - - if ( ! ExtractNextPacketInternal() ) - return -1.0; - - if ( pseudo_realtime ) - { - // Delay packet if necessary. - double packet_time = CheckPseudoTime(); - if ( packet_time ) - return packet_time; - - SetIdle(true); - return -1.0; - } - - return current_packet.time; - } - void PktSrc::Process() { if ( ! IsOpen() ) @@ -248,7 +213,7 @@ void PktSrc::Process() net_packet_dispatch(current_packet.time, ¤t_packet, this); } - have_packet = 0; + have_packet = false; DoneWithPacket(); } @@ -267,10 +232,7 @@ bool PktSrc::ExtractNextPacketInternal() // Don't return any packets if processing is suspended (except for the // very first packet which we need to set up times). if ( net_is_processing_suspended() && first_timestamp ) - { - SetIdle(true); - return 0; - } + return false; if ( pseudo_realtime ) current_wallclock = current_time(true); @@ -280,15 +242,14 @@ bool PktSrc::ExtractNextPacketInternal() if ( current_packet.time < 0 ) { Weird("negative_packet_timestamp", ¤t_packet); - return 0; + return false; } if ( ! first_timestamp ) first_timestamp = current_packet.time; - SetIdle(false); have_packet = true; - return 1; + return true; } if ( pseudo_realtime && ! IsOpen() ) @@ -297,8 +258,7 @@ bool PktSrc::ExtractNextPacketInternal() iosource_mgr->Terminate(); } - SetIdle(true); - return 0; + return false; } bool PktSrc::PrecompileBPFFilter(int index, const std::string& filter) @@ -321,7 +281,7 @@ bool PktSrc::PrecompileBPFFilter(int index, const std::string& filter) Error(msg); delete code; - return 0; + return false; } // Store it in vector. @@ -369,3 +329,30 @@ bool PktSrc::GetCurrentPacket(const Packet** pkt) *pkt = ¤t_packet; return true; } + +double PktSrc::GetNextTimeout() + { + // If there's no file descriptor for the source, which is the case for some interfaces like + // myricom, we can't rely on the polling mechanism to wait for data to be available. As gross + // as it is, just spin with a short timeout here so that it will continually poll the + // interface. The old IOSource code had a 20ns timeout between calls to select() so just + // use that. + if ( props.selectable_fd == -1 ) + return 0.00000002; + + // If we're live we want poll to do what it has to with the file descriptor. If we're not live + // but we're not in pseudo-realtime mode, let the loop just spin as fast as it can. If we're + // in pseudo-realtime mode, find the next time that a packet is ready and have poll block until + // then. + if ( IsLive() || net_is_processing_suspended() ) + return -1; + else if ( ! pseudo_realtime ) + return 0; + + if ( ! have_packet ) + ExtractNextPacketInternal(); + + double pseudo_time = current_packet.time - first_timestamp; + double ct = (current_time(true) - first_wallclock) * pseudo_realtime; + return std::max(0.0, pseudo_time - ct); + } diff --git a/src/iosource/PktSrc.h b/src/iosource/PktSrc.h index 7ba9335df8..8207e030e9 100644 --- a/src/iosource/PktSrc.h +++ b/src/iosource/PktSrc.h @@ -204,6 +204,17 @@ public: */ virtual void Statistics(Stats* stats) = 0; + /** + * Return the next timeout value for this source. This should be + * overridden by source classes where they have a timeout value + * that can wake up the poll. + * + * @return A value for the next time that the source thinks the + * poll should time out in seconds from the current time. Return + * -1 if this should should not be considered. + */ + virtual double GetNextTimeout() override; + protected: friend class Manager; friend class ManagerBase; @@ -343,11 +354,8 @@ private: bool ExtractNextPacketInternal(); // IOSource interface implementation. - void Init() override; + void InitSource() override; void Done() override; - void GetFds(iosource::FD_Set* read, iosource::FD_Set* write, - iosource::FD_Set* except) override; - double NextTimestamp(double* local_network_time) override; void Process() override; const char* Tag() override; From a2a2ff325fed463c1663872a6514e765bc8d2876 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 26 Nov 2019 13:00:36 -0700 Subject: [PATCH 27/53] Have terminate_processing() raise SIGTERM instead of calling the signal handler directly --- src/util.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util.cc b/src/util.cc index 8a693d3fc2..36d346a882 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1838,7 +1838,7 @@ RETSIGTYPE sig_handler(int signo); void terminate_processing() { if ( ! terminating ) - sig_handler(SIGTERM); + raise(SIGTERM); } extern const char* proc_status_file; From 2dcc93678794fb4b6f09157c327c208c79a058e7 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 26 Nov 2019 13:01:30 -0700 Subject: [PATCH 28/53] Test changes caused by minor order-of-operation changes related to the new loop architecture --- .../Baseline/language.expire_subnet/output | 4 ++-- testing/btest/Traces/snmp/leak_test.pcap | Bin 0 -> 11168 bytes .../frameworks/netcontrol/basic-cluster.zeek | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 testing/btest/Traces/snmp/leak_test.pcap diff --git a/testing/btest/Baseline/language.expire_subnet/output b/testing/btest/Baseline/language.expire_subnet/output index 9bb289af0c..5557d41190 100644 --- a/testing/btest/Baseline/language.expire_subnet/output +++ b/testing/btest/Baseline/language.expire_subnet/output @@ -15,11 +15,11 @@ Accessed table nums: two; three Accessed table nets: two; zero, three Time: 7.0 secs 518.0 msecs 828.15361 usecs +Expired Subnet: 192.168.4.0/24 --> four at 8.0 secs 835.0 msecs 30.078888 usecs +Expired Subnet: 192.168.1.0/24 --> one at 8.0 secs 835.0 msecs 30.078888 usecs Expired Num: 4 --> four at 8.0 secs 835.0 msecs 30.078888 usecs Expired Num: 1 --> one at 8.0 secs 835.0 msecs 30.078888 usecs Expired Num: 0 --> zero at 8.0 secs 835.0 msecs 30.078888 usecs -Expired Subnet: 192.168.4.0/24 --> four at 8.0 secs 835.0 msecs 30.078888 usecs -Expired Subnet: 192.168.1.0/24 --> one at 8.0 secs 835.0 msecs 30.078888 usecs Expired Subnet: 192.168.0.0/16 --> zero at 15.0 secs 150.0 msecs 681.018829 usecs Expired Subnet: 192.168.3.0/24 --> three at 15.0 secs 150.0 msecs 681.018829 usecs Expired Subnet: 192.168.2.0/24 --> two at 15.0 secs 150.0 msecs 681.018829 usecs diff --git a/testing/btest/Traces/snmp/leak_test.pcap b/testing/btest/Traces/snmp/leak_test.pcap new file mode 100644 index 0000000000000000000000000000000000000000..0abdc0a6a4ba3d0dbb6fcd67c0fc1d86b98eddb4 GIT binary patch literal 11168 zcmd5?3vg7`89w*sW(`Xc2w*fpU}Ja$LXwjwAwb9*B7yM62OuHIKvDuUAtsep5_yOP z3Pr5iR!tVDxISuSd@)K_u>}PkM;#rXD6`t4g2gIM(W(9Zv-jRTyC-fFM@J9)le;&Y z{r>O!&pqEc=U!(_jB_hvZ06+DL=8W`F4@NUDfkkSGqEJdS-5V=6Kig)`nGe zuFQ!g`T4GtvYO_$hShkN&eQo={H?(<*v$Io74@|>Yh5Xe8k*}{R@OBCY^*CApU8Bj zTqk~}@w657HQBkDwRQRQSBw!pVGl8Xe)En0*n#)!5Y(pO+ozSadFSGoB8pA8_`@gJ zps816zZ}KN87uQ7FxC+l_wxNC2InCv3&*o=#-sQ1(YnSA$J*BAs~T#%5_FB;jrAcubOzi zgy~KVV7mD50H$LQ&<$rxIF0U84g!YB6 zTH3txkq(8TNreY>L0@>$)+8O8WEs18ij-5=P+g<Y0 z9nM_W=Dixn6p9X2iNk~rf0uMfXZdg3!P6y7cTxZ|o)3{R4Ts5t5hn4ypK#C!`|!p# z?{*whD12}5pa$T3tIan9d>g-T@MRKq*F0StYx14W8+Pl&gDHAlDk6 z)KjiT|I7(l>Dig-7=W^Pat|NLGh(uGbJMNh{F2}g`8nY7N1BvB@n}a-BOaR%w0Rfe zm_pHfx(D@!=Fi)jb7-D(VHO`J@#q?+YxEv3K8O!=M2kL_8mmz!qcMl@LUE7Z*XCV> z^Aw7Hm5C#Sen%z!ILrM{1;;RBk8|$f0o=j3-hyy-EyBi?`*;4MZQjLzq)_+57!+Hxe2lo?pt+T+LN8h1%%XpJ} zue{~{m%@5WlHmNf#5oJzQlof_h54PlW%gTPy~QPXKV|dI2JgZh#frD2nY{b#Eh|0? z>n)cH4Gu^eWW!r_D&At@e=pwh@z-I!1$9ARIA&{-15L7wEsD3$xQ|!wyrrMk9?n~m zmHwXtZ`q-Ei-p-IZz+my59ckY0r>8;`R0OeKNxwXJ zOM&7o7H+S-Wo2P|IB!ASz&9|?OaR}4lXZ%>(0EUL`{XTgmF?lYWwhXYNa8#J-m+Nn z77M%2-m*R37jE4G&B2}KjsA5@KDhJf9g4SPncREjEoBM5aO)ND?RlacEk}7nN=NSd?mQKZ6EX+Q6 zOW`74IB!W2d~de-P6FS?&(A8}l56t)LA~XkwZ3rPGG6GkO6o6@;4NDfZ?SUy(Y$5a zdS5ti!L^U;j`l44PO^Im()o(;Zm-jXKtdqUE0GQ1^S@fHiW*WPmQ z*S>Jxg6lkJoS6c?1t+gmyk&yPw@==(zfw#<7yv4%qv$rI4h4mI( z+u%;?CV#C9!JS8UD&9i-Re1GoPt4;E>n$0Avq$1w2ydxWyv4%owYSjT2Q^x+?R>!J zZNxE!vQ}N}!Lv4r+t<{!w(w!WA2BpXf9ZR${}uENk(!?r zaS7#g^l*TaqVc}OOri1BQlBegyQ)GsxqkwjL2`X_tc=RJXA3WfJ!55^DhF0px+fVZbQpLgCZ_s`DX>e^(JdFN(+d0=ZV+i7lR z<{Hs~Ey4-e@|F;N#@hbV<;_F;kCn|sX%o`^P=3%RByl92Wv-XL9pI|yd^j;%=v*l2 zT*5rFL%6zs!dwr4FEy&`!4G}jEjXr7>e}wXIEcF5WY@J6b*(!$jbAO7b*8ROF{@hA zV}CY6&l=Rrm9r6a*2x&Uc2eD0%jcAiR2+7oj`O*Tq3DMg;sgYtC$P7bJVUy<_} za^akJEq+I#=(apDSLk-Tq+1!AHS7jnCgtPCM90c!d}t++59Jv$iXkDKr97yP@)*`- zd}gm}6n&~Z7{j0sjbZ+|PdSaBb8inC!)W}RimxYJW&9L9E2`)5b6T&n1tuYoGmcx6->GwsJ^}8+1T>I zk@?VoLaA$oV?WjPi!!OMvsu>j-duf3t;gCljwgF@lC(D5h1Gh5<02XpvTWe#s)_IeI~JK>?t zj5S!3vtxBfnEh&$xVPR|_6+fE_a5JQ;N#AvH;plL@Lt$FoL7OT zJ%^vF>y8m7@oYXo%rA8TH^*|x<>?I#-TdSlxLTNKN`}|f!Bc`&ek|j_|sRERonO7 zLpfKOoX1$4@1y&$`ClKUaXuKo>oVO)x^(>OCTC?lAWXvUKu{y>QB7slOL0u$S6bpe zJNB&Lo)h2>^vTm(IrdG1vE8m9Y}!)^j0=Xbd(FN8_YT5BBm9Q<%L3Yma<4YIkF>bo zy#m}P9ja5c?}GM2-rV+IlRIhe?~)^+z2EX}?Kk=11^%1=D63wM_Y?~M`LUl0{uu%O z$e`$=KXdHigzz63gil^8<5xto&GDm){LkKm)}U-&c^@=*C6zRIt-5Co9ILPRYmXYx zLmy}z2*Q^&Xu7IAoCg0A8qhUo_J3qhw87W21`>X-1`e$vQvYd_1G(EzI#4{tf=!@wIhN8Ijo(*`AiegO)G)TMUHl9p%VEfVWSIt^r z-6WMdK0N>==5xeZ#=9Wl`H}pnJ>#)%Ag0~&dlS#yJG||{q(=O<96z1hfMW`U-!&$` zT^7Grj)32^Z6i3=3K4vuZn)NGH0yd1Gc-KQ5@Eo04GbuF@?xC9=pP?K|98cjpY#uU zfD+Hs5ckLF4I1TA{=(_x-{F`-;l9<(rPj)Ybt9W~n+EVkB^RU+VGQF#jhGHLVr$2} zmA_e9FrLFR-@(gFEGOib20M@xlqGG0_JeykA_icTEH;o-gg;-(`xt^6<$dtQ)5-gB zOrdCan~t+Zao5kc@)jCy>^GM7uY1xklV^maX67B$+EU-Vx~8e#h{jji4I+pBfx)`b z6GUy4sR=!7hDP}lk02&fPIHpka-)8byX7LdEwIQRw zqRN{lK@Z9`LHAn1B24*O9r(godu&kjQpN_>4b-$tmeI8e;>1$I{I;=SD=?`MN5`VS zjb4Fc3WeiUCdb_t$CHK3&B_mb#c8j|nz!vs7k%2$IzMkC~uY}xAiz7!MJi@i+BZND=?G(4?BLa~Lbb)KmN0M$N lOu}Z}K+b=9lz%>AK8K3EGZ~jEZ9je(rxi!U^lR~B{twas#2f$s literal 0 HcmV?d00001 diff --git a/testing/btest/scripts/base/frameworks/netcontrol/basic-cluster.zeek b/testing/btest/scripts/base/frameworks/netcontrol/basic-cluster.zeek index 2ff506b39f..b3aa5344f2 100644 --- a/testing/btest/scripts/base/frameworks/netcontrol/basic-cluster.zeek +++ b/testing/btest/scripts/base/frameworks/netcontrol/basic-cluster.zeek @@ -21,7 +21,7 @@ redef Cluster::nodes = { @TEST-END-FILE redef Log::default_rotation_interval = 0secs; -#redef exit_only_after_terminate = T; +redef exit_only_after_terminate = T; @load base/frameworks/netcontrol @@ -75,5 +75,9 @@ event NetControl::rule_added(r: NetControl::Rule, p: NetControl::PluginState, ms event NetControl::rule_destroyed(r: NetControl::Rule) { if ( r$entity?$ip ) + { print "Rule destroyed", r$id, r$cid, |NetControl::find_rules_subnet(r$entity$ip)|; + if ( Cluster::local_node_type() == Cluster::WORKER ) + schedule 2sec { terminate_me() }; + } } From be42608b51195119e0f52aaf5eedad0e4220cf02 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 2 Dec 2019 17:10:57 -0700 Subject: [PATCH 29/53] Remove concept of multiple timer managers - All timers are now handled by a single global timer manager, which simplifies how they handled by the IOSource manager. - This change flows down a number of changes to other parts of the code. The timer manager tag field is removed, which means that matching connections to a timer manager is also removed. This removes the ability to tag a connection as internal or external, since that's how the connections where differentiated. This in turn removes the `current_conns_extern` field from the `ConnStats` record type in the script layer. --- scripts/base/init-bare.zeek | 1 - scripts/policy/protocols/conn/weirds.zeek | 8 +- src/Conn.cc | 38 +----- src/Conn.h | 10 -- src/Net.cc | 11 +- src/Sessions.cc | 145 +--------------------- src/Sessions.h | 34 ----- src/Stats.cc | 3 +- src/Timer.cc | 21 ++-- src/Timer.h | 9 +- src/analyzer/Analyzer.cc | 5 +- src/iosource/IOSource.h | 11 -- src/main.cc | 2 +- src/stats.bif | 1 - src/zeek.bif | 13 -- 15 files changed, 28 insertions(+), 284 deletions(-) diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index d810c500a0..65cbb97abe 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -548,7 +548,6 @@ type NetStats: record { type ConnStats: record { total_conns: count; ##< current_conns: count; ##< - current_conns_extern: count; ##< sess_current_conns: count; ##< num_packets: count; diff --git a/scripts/policy/protocols/conn/weirds.zeek b/scripts/policy/protocols/conn/weirds.zeek index 7f3c9431a4..ea3d9a19bc 100644 --- a/scripts/policy/protocols/conn/weirds.zeek +++ b/scripts/policy/protocols/conn/weirds.zeek @@ -1,4 +1,4 @@ -##! This script handles core generated connection related "weird" events to +##! This script handles core generated connection related "weird" events to ##! push weird information about connections into the weird framework. ##! For live operational deployments, this can frequently cause load issues ##! due to large numbers of these events and quite possibly shouldn't be @@ -29,8 +29,6 @@ event rexmit_inconsistency(c: connection, t1: string, t2: string, tcp_flags: str event content_gap(c: connection, is_orig: bool, seq: count, length: count) { NOTICE([$note=Content_Gap, $conn=c, - $msg=fmt("%s content gap (%s %d/%d)%s", - id_string(c$id), is_orig ? ">" : "<", seq, length, - is_external_connection(c) ? " [external]" : "")]); + $msg=fmt("%s content gap (%s %d/%d)", + id_string(c$id), is_orig ? ">" : "<", seq, length)]); } - diff --git a/src/Conn.cc b/src/Conn.cc index 855f14558c..1f96af5a7d 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -53,7 +53,6 @@ void ConnectionTimer::Dispatch(double t, int is_expire) uint64_t Connection::total_connections = 0; uint64_t Connection::current_connections = 0; -uint64_t Connection::external_connections = 0; Connection::Connection(NetSessions* s, const ConnIDKey& k, double t, const ConnID* id, uint32_t flow, const Packet* pkt, @@ -115,23 +114,10 @@ Connection::Connection(NetSessions* s, const ConnIDKey& k, double t, const ConnI ++current_connections; ++total_connections; - std::string* tag = current_iosrc->GetCurrentTag(); - conn_timer_mgr = tag ? new std::string(*tag) : nullptr; - if ( arg_encap ) encapsulation = new EncapsulationStack(*arg_encap); else encapsulation = 0; - - if ( conn_timer_mgr ) - { - ++external_connections; - // We schedule a timer which removes this connection from memory - // indefinitively into the future. Ii will expire when the timer - // mgr is drained but not before. - ADD_TIMER(&Connection::RemoveConnectionTimer, 1e20, 1, - TIMER_REMOVE_CONNECTION); - } } Connection::~Connection() @@ -148,12 +134,9 @@ Connection::~Connection() } delete root_analyzer; - delete conn_timer_mgr; delete encapsulation; --current_connections; - if ( conn_timer_mgr ) - --external_connections; } void Connection::CheckEncapsulation(const EncapsulationStack* arg_encap) @@ -512,14 +495,14 @@ void Connection::ConnectionEvent(EventHandlerPtr f, analyzer::Analyzer* a, val_l // "this" is passed as a cookie for the event mgr.QueueEvent(f, std::move(vl), SOURCE_LOCAL, - a ? a->GetID() : 0, GetTimerMgr(), this); + a ? a->GetID() : 0, timer_mgr, this); } void Connection::ConnectionEventFast(EventHandlerPtr f, analyzer::Analyzer* a, val_list vl) { // "this" is passed as a cookie for the event mgr.QueueEventFast(f, std::move(vl), SOURCE_LOCAL, - a ? a->GetID() : 0, GetTimerMgr(), this); + a ? a->GetID() : 0, timer_mgr, this); } void Connection::ConnectionEvent(EventHandlerPtr f, analyzer::Analyzer* a, val_list* vl) @@ -547,7 +530,7 @@ void Connection::AddTimer(timer_func timer, double t, bool do_expire, return; Timer* conn_timer = new ConnectionTimer(this, timer, t, do_expire, type); - GetTimerMgr()->Add(conn_timer); + timer_mgr->Add(conn_timer); timers.push_back(conn_timer); } @@ -566,25 +549,12 @@ void Connection::CancelTimers() std::copy(timers.begin(), timers.end(), std::back_inserter(tmp)); for ( const auto& timer : tmp ) - GetTimerMgr()->Cancel(timer); + timer_mgr->Cancel(timer); timers_canceled = 1; timers.clear(); } -TimerMgr* Connection::GetTimerMgr() const - { - if ( ! conn_timer_mgr ) - // Global manager. - return timer_mgr; - - // We need to check whether the local timer manager still exists; - // it may have already been timed out, in which case we fall back - // to the global manager (though this should be rare). - TimerMgr* local_mgr = sessions->LookupTimerMgr(conn_timer_mgr, false); - return local_mgr ? local_mgr : timer_mgr; - } - void Connection::FlipRoles() { IPAddr tmp_addr = resp_addr; diff --git a/src/Conn.h b/src/Conn.h index 41b8fc8283..a7ea9d3820 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -228,11 +228,6 @@ public: void Describe(ODesc* d) const override; void IDString(ODesc* d) const; - TimerMgr* GetTimerMgr() const; - - // Returns true if connection has been received externally. - bool IsExternal() const { return conn_timer_mgr != 0; } - // Statistics. // Just a lower bound. @@ -243,8 +238,6 @@ public: { return total_connections; } static uint64_t CurrentConnections() { return current_connections; } - static uint64_t CurrentExternalConnections() - { return external_connections; } // Returns true if the history was already seen, false otherwise. int CheckHistory(uint32_t mask, char code) @@ -320,8 +313,6 @@ protected: ConnIDKey key; bool key_valid; - // Timer manager to use for this conn (or nil). - std::string* conn_timer_mgr; timer_list timers; IPAddr orig_addr; @@ -353,7 +344,6 @@ protected: // Count number of connections. static uint64_t total_connections; static uint64_t current_connections; - static uint64_t external_connections; string history; uint32_t hist_seen; diff --git a/src/Net.cc b/src/Net.cc index c4df44b9cd..f50a835fce 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -213,13 +213,10 @@ void net_init(const std::string& interface, void expire_timers(iosource::PktSrc* src_ps) { SegmentProfiler(segment_logger, "expiring-timers"); - TimerMgr* tmgr = - src_ps ? sessions->LookupTimerMgr(src_ps->GetCurrentTag()) - : timer_mgr; current_dispatched += - tmgr->Advance(network_time, - max_timer_expires - current_dispatched); + timer_mgr->Advance(network_time, + max_timer_expires - current_dispatched); } void net_packet_dispatch(double t, const Packet* pkt, iosource::PktSrc* src_ps) @@ -227,10 +224,8 @@ void net_packet_dispatch(double t, const Packet* pkt, iosource::PktSrc* src_ps) if ( ! bro_start_network_time ) bro_start_network_time = t; - TimerMgr* tmgr = sessions->LookupTimerMgr(src_ps->GetCurrentTag()); - // network_time never goes back. - net_update_time(tmgr->Time() < t ? t : tmgr->Time()); + net_update_time(timer_mgr->Time() < t ? t : timer_mgr->Time()); current_pktsrc = src_ps; current_iosrc = src_ps; diff --git a/src/Sessions.cc b/src/Sessions.cc index 984a8df754..981a73350b 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -42,32 +42,6 @@ enum NetBIOS_Service { NetSessions* sessions; -void TimerMgrExpireTimer::Dispatch(double t, int is_expire) - { - if ( mgr->LastAdvance() + timer_mgr_inactivity_timeout < timer_mgr->Time() ) - { - // Expired. - DBG_LOG(DBG_TM, "TimeMgr %p has timed out", mgr); - mgr->Expire(); - - // Make sure events are executed. They depend on the TimerMgr. - ::mgr.Drain(); - - sessions->timer_mgrs.erase(mgr->GetTag()); - delete mgr; - } - else - { - // Reinstall timer. - if ( ! is_expire ) - { - double n = mgr->LastAdvance() + - timer_mgr_inactivity_timeout; - timer_mgr->Add(new TimerMgrExpireTimer(n, mgr)); - } - } - } - void IPTunnelTimer::Dispatch(double t, int is_expire) { NetSessions::IPTunnelMap::const_iterator it = @@ -210,59 +184,6 @@ void NetSessions::NextPacket(double t, const Packet* pkt) DumpPacket(pkt); } -int NetSessions::CheckConnectionTag(Connection* conn) - { - if ( current_iosrc->GetCurrentTag() ) - { - // Packet is tagged. - if ( conn->GetTimerMgr() == timer_mgr ) - { - // Connection uses global timer queue. But the - // packet has a tag that means we got it externally, - // probably from the Time Machine. - DBG_LOG(DBG_TM, "got packet with tag %s for already" - "known connection, reinstantiating", - current_iosrc->GetCurrentTag()->c_str()); - return 0; - } - else - { - // Connection uses local timer queue. - TimerMgrMap::iterator i = - timer_mgrs.find(*current_iosrc->GetCurrentTag()); - if ( i != timer_mgrs.end() && - conn->GetTimerMgr() != i->second ) - { - // Connection uses different local queue - // than the tag for the current packet - // indicates. - // - // This can happen due to: - // (1) getting same packets with - // different tags - // (2) timer mgr having already expired - DBG_LOG(DBG_TM, "packet ignored due old/inconsistent tag"); - return -1; - } - - return 1; - } - } - - // Packet is not tagged. - if ( conn->GetTimerMgr() != timer_mgr ) - { - // Connection does not use the global timer queue. That - // means that this is a live packet belonging to a - // connection for which we have already switched to - // processing external input. - DBG_LOG(DBG_TM, "packet ignored due to processing it in external data"); - return -1; - } - - return 1; - } - static unsigned int gre_header_len(uint16_t flags) { unsigned int len = 4; // Always has 2 byte flags and 2 byte protocol type. @@ -735,14 +656,9 @@ void NetSessions::DoNextPacket(double t, const Packet* pkt, const IP_Hdr* ip_hdr else { // We already know that connection. - int consistent = CheckConnectionTag(conn); - if ( consistent < 0 ) - return; - - if ( ! consistent || conn->IsReuse(t, data) ) + if ( conn->IsReuse(t, data) ) { - if ( consistent ) - conn->Event(connection_reused, 0); + conn->Event(connection_reused, 0); Remove(conn); conn = NewConn(key, t, &id, data, proto, ip_hdr->FlowLabel(), pkt, encapsulation); @@ -1158,8 +1074,6 @@ void NetSessions::Drain() ic->Done(); ic->RemovalEvent(); } - - ExpireTimerMgrs(); } void NetSessions::GetStats(SessionStats& s) const @@ -1233,25 +1147,9 @@ Connection* NetSessions::NewConn(const ConnIDKey& k, double t, const ConnID* id, return 0; } - bool external = conn->IsExternal(); - - if ( external ) - conn->AppendAddl(fmt("tag=%s", - conn->GetTimerMgr()->GetTag().c_str())); - if ( new_connection ) - { conn->Event(new_connection, 0); - if ( external && connection_external ) - { - conn->ConnectionEventFast(connection_external, 0, { - conn->BuildConnVal(), - new StringVal(conn->GetTimerMgr()->GetTag().c_str()), - }); - } - } - return conn; } @@ -1338,45 +1236,6 @@ bool NetSessions::WantConnection(uint16_t src_port, uint16_t dst_port, return true; } -TimerMgr* NetSessions::LookupTimerMgr(const std::string* tag, bool create) - { - if ( ! tag ) - { - DBG_LOG(DBG_TM, "no tag, using global timer mgr %p", timer_mgr); - return timer_mgr; - } - - TimerMgrMap::iterator i = timer_mgrs.find(*tag); - if ( i != timer_mgrs.end() ) - { - DBG_LOG(DBG_TM, "tag %s, using non-global timer mgr %p", tag->c_str(), i->second); - return i->second; - } - else - { - if ( ! create ) - return 0; - - // Create new queue for tag. - TimerMgr* mgr = new PQ_TimerMgr(*tag); - DBG_LOG(DBG_TM, "tag %s, creating new non-global timer mgr %p", tag->c_str(), mgr); - timer_mgrs.insert(TimerMgrMap::value_type(*tag, mgr)); - double t = timer_mgr->Time() + timer_mgr_inactivity_timeout; - timer_mgr->Add(new TimerMgrExpireTimer(t, mgr)); - return mgr; - } - } - -void NetSessions::ExpireTimerMgrs() - { - for ( TimerMgrMap::iterator i = timer_mgrs.begin(); - i != timer_mgrs.end(); ++i ) - { - i->second->Expire(); - delete i->second; - } - } - void NetSessions::DumpPacket(const Packet *pkt, int len) { if ( ! pkt_dumper ) diff --git a/src/Sessions.h b/src/Sessions.h index c627393e5d..e98dd53a9c 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -44,20 +44,6 @@ struct SessionStats { uint64_t num_packets; }; -// Drains and deletes a timer manager if it hasn't seen any advances -// for an interval timer_mgr_inactivity_timeout. -class TimerMgrExpireTimer : public Timer { -public: - TimerMgrExpireTimer(double t, TimerMgr* arg_mgr) - : Timer(t, TIMER_TIMERMGR_EXPIRE), mgr(arg_mgr) - { } - - void Dispatch(double t, int is_expire) override; - -protected: - TimerMgr* mgr; -}; - class NetSessions { public: NetSessions(); @@ -101,13 +87,6 @@ public: return packet_filter; } - // Looks up timer manager associated with tag. If tag is unknown and - // "create" is true, creates new timer manager and stores it. Returns - // global timer manager if tag is nil. - TimerMgr* LookupTimerMgr(const std::string* tag, bool create = true); - - void ExpireTimerMgrs(); - analyzer::stepping_stone::SteppingStoneManager* GetSTPManager() { return stp_manager; } unsigned int CurrentConnections() @@ -168,7 +147,6 @@ public: protected: friend class ConnCompressor; - friend class TimerMgrExpireTimer; friend class IPTunnelTimer; using ConnectionMap = std::map; @@ -180,13 +158,6 @@ protected: Connection* LookupConn(const ConnectionMap& conns, const ConnIDKey& key); - // Check whether the tag of the current packet is consistent with - // the given connection. Returns: - // -1 if current packet is to be completely ignored. - // 0 if tag is not consistent and new conn should be instantiated. - // 1 if tag is consistent, i.e., packet is part of connection. - int CheckConnectionTag(Connection* conn); - // Returns true if the port corresonds to an application // for which there's a Bro analyzer (even if it might not // be used by the present policy script), or it's more @@ -243,11 +214,6 @@ protected: int dump_this_packet; // if true, current packet should be recorded uint64_t num_packets_processed; PacketProfiler* pkt_profiler; - - // We may use independent timer managers for different sets of related - // activity. The managers are identified by a unique tag. - typedef std::map TimerMgrMap; - TimerMgrMap timer_mgrs; }; diff --git a/src/Stats.cc b/src/Stats.cc index fed8a5259a..cbc58c197e 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -117,12 +117,11 @@ void ProfileLogger::Log() int conn_mem_use = expensive ? sessions->ConnectionMemoryUsage() : 0; - file->Write(fmt("%.06f Conns: total=%" PRIu64 " current=%" PRIu64 "/%" PRIi32 " ext=%" PRIu64 " mem=%" PRIi32 "K avg=%.1f table=%" PRIu32 "K connvals=%" PRIu32 "K\n", + file->Write(fmt("%.06f Conns: total=%" PRIu64 " current=%" PRIu64 "/%" PRIi32 " mem=%" PRIi32 "K avg=%.1f table=%" PRIu32 "K connvals=%" PRIu32 "K\n", network_time, Connection::TotalConnections(), Connection::CurrentConnections(), sessions->CurrentConnections(), - Connection::CurrentExternalConnections(), conn_mem_use, expensive ? (conn_mem_use / double(sessions->CurrentConnections())) : 0, expensive ? sessions->MemoryAllocation() / 1024 : 0, diff --git a/src/Timer.cc b/src/Timer.cc index 686eca9e68..dc0fb5e470 100644 --- a/src/Timer.cc +++ b/src/Timer.cc @@ -60,12 +60,11 @@ void Timer::Describe(ODesc* d) const unsigned int TimerMgr::current_timers[NUM_TIMER_TYPES]; -TimerMgr::TimerMgr(const std::string& arg_tag) +TimerMgr::TimerMgr() { t = 0.0; num_expired = 0; last_advance = last_timestamp = 0; - tag = arg_tag; if ( iosource_mgr ) iosource_mgr->Register(this, true); @@ -73,13 +72,11 @@ TimerMgr::TimerMgr(const std::string& arg_tag) TimerMgr::~TimerMgr() { - DBG_LOG(DBG_TM, "deleting timer mgr %p", this); } int TimerMgr::Advance(double arg_t, int max_expire) { - DBG_LOG(DBG_TM, "advancing %stimer mgr %p to %.6f", - this == timer_mgr ? "global " : "", this, arg_t); + DBG_LOG(DBG_TM, "advancing timer mgr to %.6f", arg_t); t = arg_t; last_timestamp = 0; @@ -111,7 +108,7 @@ void TimerMgr::InitPostScript() } -PQ_TimerMgr::PQ_TimerMgr(const std::string& tag) : TimerMgr(tag) +PQ_TimerMgr::PQ_TimerMgr() : TimerMgr() { q = new PriorityQueue; } @@ -123,8 +120,8 @@ PQ_TimerMgr::~PQ_TimerMgr() void PQ_TimerMgr::Add(Timer* timer) { - DBG_LOG(DBG_TM, "Adding timer %s to TimeMgr %p at %.6f", - timer_type_to_string(timer->Type()), this, timer->Time()); + DBG_LOG(DBG_TM, "Adding timer %s (%p) at %.6f", + timer_type_to_string(timer->Type()), timer, timer->Time()); // Add the timer even if it's already expired - that way, if // multiple already-added timers are added, they'll still @@ -140,8 +137,8 @@ void PQ_TimerMgr::Expire() Timer* timer; while ( (timer = Remove()) ) { - DBG_LOG(DBG_TM, "Dispatching timer %s in TimeMgr %p", - timer_type_to_string(timer->Type()), this); + DBG_LOG(DBG_TM, "Dispatching timer %s (%p)", + timer_type_to_string(timer->Type()), timer); timer->Dispatch(t, 1); --current_timers[timer->Type()]; delete timer; @@ -162,8 +159,8 @@ int PQ_TimerMgr::DoAdvance(double new_t, int max_expire) // whether we should delete it too. (void) Remove(); - DBG_LOG(DBG_TM, "Dispatching timer %s in TimeMgr %p", - timer_type_to_string(timer->Type()), this); + DBG_LOG(DBG_TM, "Dispatching timer %s (%p)", + timer_type_to_string(timer->Type()), timer); timer->Dispatch(new_t, 0); delete timer; diff --git a/src/Timer.h b/src/Timer.h index 6ad7cda003..75df60a47f 100644 --- a/src/Timer.h +++ b/src/Timer.h @@ -104,8 +104,6 @@ public: double Time() const { return t ? t : 1; } // 1 > 0 - const std::string& GetTag() const { return tag; } - virtual int Size() const = 0; virtual int PeakSize() const = 0; virtual uint64_t CumulativeNum() const = 0; @@ -122,7 +120,7 @@ public: // IOSource API methods virtual double GetNextTimeout() override { return -1; } virtual void Process() override; - virtual const char* Tag() override { return fmt("TimerMgr %s", tag.c_str()); } + virtual const char* Tag() override { return "TimerMgr"; } /** * Performs some extra initialization on a timer manager. This shouldn't @@ -131,7 +129,7 @@ public: void InitPostScript(); protected: - explicit TimerMgr(const std::string& arg_tag); + TimerMgr(); virtual int DoAdvance(double t, int max_expire) = 0; virtual void Remove(Timer* timer) = 0; @@ -139,7 +137,6 @@ protected: double t; double last_timestamp; double last_advance; - std::string tag; int num_expired; @@ -148,7 +145,7 @@ protected: class PQ_TimerMgr : public TimerMgr { public: - explicit PQ_TimerMgr(const std::string& arg_tag); + PQ_TimerMgr(); ~PQ_TimerMgr() override; void Add(Timer* timer) override; diff --git a/src/analyzer/Analyzer.cc b/src/analyzer/Analyzer.cc index 131782afb3..4c5c84b221 100644 --- a/src/analyzer/Analyzer.cc +++ b/src/analyzer/Analyzer.cc @@ -731,7 +731,7 @@ void Analyzer::AddTimer(analyzer_timer_func timer, double t, Timer* analyzer_timer = new AnalyzerTimer(this, timer, t, do_expire, type); - Conn()->GetTimerMgr()->Add(analyzer_timer); + timer_mgr->Add(analyzer_timer); timers.push_back(analyzer_timer); } @@ -751,7 +751,7 @@ void Analyzer::CancelTimers() // TODO: could be a for_each for ( auto timer : tmp ) - Conn()->GetTimerMgr()->Cancel(timer); + timer_mgr->Cancel(timer); timers_canceled = 1; timers.clear(); @@ -923,4 +923,3 @@ void TransportLayerAnalyzer::PacketContents(const u_char* data, int len) Event(packet_contents, contents); } } - diff --git a/src/iosource/IOSource.h b/src/iosource/IOSource.h index 67807fa0b4..cc8cf6a97f 100644 --- a/src/iosource/IOSource.h +++ b/src/iosource/IOSource.h @@ -70,17 +70,6 @@ public: */ virtual void Process() = 0; - /** - * Returns the tag of the timer manager associated with the last - * procesees data item. - * - * Can be overridden by derived classes. - * - * @return The tag, or null for the global timer manager. - * - */ - virtual std::string* GetCurrentTag() { return 0; } - /** * Returns a descriptive tag representing the source for debugging. * diff --git a/src/main.cc b/src/main.cc index 93d3c652c2..0760c6b664 100644 --- a/src/main.cc +++ b/src/main.cc @@ -550,7 +550,7 @@ int main(int argc, char** argv) createCurrentDoc("1.0"); // Set a global XML document #endif - timer_mgr = new PQ_TimerMgr(""); + timer_mgr = new PQ_TimerMgr(); auto zeekygen_cfg = options.zeekygen_config_file.value_or(""); zeekygen_mgr = new zeekygen::Manager(zeekygen_cfg, bro_argv[0]); diff --git a/src/stats.bif b/src/stats.bif index cf94ebb985..522bfe8913 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -88,7 +88,6 @@ function get_conn_stats%(%): ConnStats r->Assign(n++, val_mgr->GetCount(Connection::TotalConnections())); r->Assign(n++, val_mgr->GetCount(Connection::CurrentConnections())); - r->Assign(n++, val_mgr->GetCount(Connection::CurrentExternalConnections())); r->Assign(n++, val_mgr->GetCount(sessions->CurrentConnections())); SessionStats s; diff --git a/src/zeek.bif b/src/zeek.bif index a42491daee..7129add1c6 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -1788,19 +1788,6 @@ function log10%(d: double%): double # # =========================================================================== -## Determines whether a connection has been received externally. For example, -## Broccoli or the Time Machine can send packets to Zeek via a mechanism that is -## one step lower than sending events. This function checks whether the packets -## of a connection stem from one of these external *packet sources*. -## -## c: The connection to test. -## -## Returns: True if *c* has been received externally. -function is_external_connection%(c: connection%) : bool - %{ - return val_mgr->GetBool(c && c->IsExternal()); - %} - ## Returns the ID of the analyzer which raised the current event. ## ## Returns: The ID of the analyzer which raised the current event, or 0 if From fda8e3fd794c0a5789a1c2a6dbf92afaf7bf6364 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 12 Dec 2019 15:14:28 -0700 Subject: [PATCH 30/53] Update NEWS and docs submodule --- NEWS | 21 +++++++++++++++++++++ doc | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index a8b147493e..f4e147fc06 100644 --- a/NEWS +++ b/NEWS @@ -102,9 +102,30 @@ Changed Functionality the ``portnum`` component of the returned ``URI`` value is left uninitialized. +- Replace old ``select``-based IO loop with a new architecture that doesn't + spin checking for active IO sources. The new architecture now waits for the + the sources to actively notify it when activity occurs and only processes + data once it's ready. This helps heavily reduce the CPU usage on idle + network connections. This includes a couple of breaking changes: + + - Only a single packet source is allowed to be specified from the + command-line now. If you pass combinations of multiple ``-r`` and/or + ``-i`` flags, Zeek will return an error at startup. + - The IOSource API changed fairly wildly. The ``GetFds()`` and + ``NextTimestamp`` methods no longer exist. If you had previously + implemented a custom IOSource, you will need to look at the new API + and make changes to your code to accomodate it. This does not include + packet sources, which should remain functional with little to no + changes, since the entirety of the changes should be in ``PktSrc``. + Removed Functionality --------------------- +- Removed the ``current_conns_extern`` field from the ConnStats record + type. Zeek only maintains a single timer manager now, and without the + manager tags that came with multiple tiemr managers, we don't track + whether a connection is external anymore. + Deprecated Functionality ------------------------ diff --git a/doc b/doc index 566174d004..359a79fc4c 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 566174d004c14d061fcf7c03e4829f20d46aaad8 +Subproject commit 359a79fc4ccc48df74daa7d66a93b01b669e4ddf From 6ddcc87c48e1af67ad3579bccf8a87ae20a6c4cc Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 28 Jan 2020 17:20:21 -0700 Subject: [PATCH 31/53] Show percentage of packets dropped in the final process output --- src/Net.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Net.cc b/src/Net.cc index f50a835fce..8148fb9ae9 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -364,8 +364,9 @@ void net_get_final_stats() { iosource::PktSrc::Stats s; ps->Statistics(&s); - reporter->Info("%" PRIu64 " packets received on interface %s, %" PRIu64 " dropped", - s.received, ps->Path().c_str(), s.dropped); + double dropped_pct = s.dropped > 0.0 ? ((double)s.dropped / ((double)s.received + (double)s.dropped)) * 100.0 : 0.0; + reporter->Info("%" PRIu64 " packets received on interface %s, %" PRIu64 " (%.2f%%) dropped", + s.received, ps->Path().c_str(), s.dropped, dropped_pct); } } From 0cfb115c1be934881508ffd0d2d3051fe0d1d03f Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 29 Jan 2020 16:35:29 -0700 Subject: [PATCH 32/53] Update Supervisor code for the new IOSource API --- src/main.cc | 3 +++ src/supervisor/Supervisor.cc | 17 ++++++++++------- src/supervisor/Supervisor.h | 12 +++++++----- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/main.cc b/src/main.cc index 0760c6b664..ffcce06bc3 100644 --- a/src/main.cc +++ b/src/main.cc @@ -674,6 +674,9 @@ int main(int argc, char** argv) broker_mgr->InitPostScript(); timer_mgr->InitPostScript(); + if ( zeek::supervisor_mgr ) + zeek::supervisor_mgr->InitPostScript(); + if ( options.print_plugins ) { bool success = show_plugins(options.print_plugins); diff --git a/src/supervisor/Supervisor.cc b/src/supervisor/Supervisor.cc index b6e157c1a3..4ef342ffb7 100644 --- a/src/supervisor/Supervisor.cc +++ b/src/supervisor/Supervisor.cc @@ -10,6 +10,7 @@ #include #include +#include "iosource/Manager.h" #include "Supervisor.h" #include "Reporter.h" #include "DebugLogger.h" @@ -160,7 +161,6 @@ Supervisor::Supervisor(Supervisor::Config cfg, StemState ss) { DBG_LOG(DBG_SUPERVISOR, "forked stem process %d", stem_pid); setsignal(SIGCHLD, supervisor_signal_handler); - SetIdle(true); int status; auto res = waitpid(stem_pid, &status, WNOHANG); @@ -197,6 +197,9 @@ Supervisor::~Supervisor() return; } + iosource_mgr->UnregisterFd(signal_flare.FD(), this); + iosource_mgr->UnregisterFd(stem_pipe->InFD(), this); + DBG_LOG(DBG_SUPERVISOR, "shutdown, killing stem process %d", stem_pid); auto kill_res = kill(stem_pid, SIGTERM); @@ -352,16 +355,16 @@ void Supervisor::HandleChildSignal() } } -void Supervisor::GetFds(iosource::FD_Set* read, iosource::FD_Set* write, - iosource::FD_Set* except) +void Supervisor::InitPostScript() { - read->Insert(signal_flare.FD()); - read->Insert(stem_pipe->InFD()); + iosource_mgr->Register(this); + iosource_mgr->RegisterFd(signal_flare.FD(), this); + iosource_mgr->RegisterFd(stem_pipe->InFD(), this); } -double Supervisor::NextTimestamp(double* local_network_time) +double Supervisor::GetNextTimeout() { - return timer_mgr->Time(); + return -1; } void Supervisor::Process() diff --git a/src/supervisor/Supervisor.h b/src/supervisor/Supervisor.h index 32b07e5aff..36d00f1808 100644 --- a/src/supervisor/Supervisor.h +++ b/src/supervisor/Supervisor.h @@ -285,6 +285,12 @@ public: */ ~Supervisor(); + /** + * Perform some initialization that needs to happen after scripts are loaded + * and the IOSource manager is created. + */ + void InitPostScript(); + /** * @return the process ID of the Stem. */ @@ -347,11 +353,7 @@ public: private: // IOSource interface overrides: - void GetFds(iosource::FD_Set* read, iosource::FD_Set* write, - iosource::FD_Set* except) override; - - double NextTimestamp(double* local_network_time) override; - + double GetNextTimeout() override; void Process() override; size_t ProcessMessages(); From fea0339aca57e4dd3592c37f31241485bd887c95 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 30 Jan 2020 13:34:10 -0700 Subject: [PATCH 33/53] Add better error messaging when RegisterFd/UnregisterFd fail --- src/DNS_Mgr.cc | 9 +++++++-- src/broker/Manager.cc | 12 +++++++----- src/iosource/Manager.cc | 21 +++++++++++++++------ src/iosource/Manager.h | 4 ++-- src/supervisor/Supervisor.cc | 7 +++++-- src/threading/MsgThread.cc | 5 +++-- 6 files changed, 39 insertions(+), 19 deletions(-) diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 903485a652..93eba847cc 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -440,9 +440,14 @@ void DNS_Mgr::InitSource() } if ( nb_dns ) - iosource_mgr->RegisterFd(nb_dns_fd(nb_dns), this); + { + if ( ! iosource_mgr->RegisterFd(nb_dns_fd(nb_dns), this) ) + reporter->FatalError("Failed to register nb_dns file descriptor with iosource_mgr"); + } else + { reporter->Warning("problem initializing NB-DNS: %s", err); + } did_init = true; } @@ -1490,5 +1495,5 @@ void DNS_Mgr::GetStats(Stats* stats) void DNS_Mgr::Terminate() { if ( nb_dns ) - iosource_mgr->UnregisterFd(nb_dns_fd(nb_dns)); + iosource_mgr->UnregisterFd(nb_dns_fd(nb_dns), this); } diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index a7ad3a1df6..1e54ce33a0 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -207,16 +207,18 @@ void Manager::InitPostScript() auto cqs = get_option("Broker::congestion_queue_size")->AsCount(); bstate = std::make_shared(std::move(config), cqs); - iosource_mgr->RegisterFd(bstate->subscriber.fd(), this); - iosource_mgr->RegisterFd(bstate->status_subscriber.fd(), this); + if ( ! iosource_mgr->RegisterFd(bstate->subscriber.fd(), this) ) + reporter->FatalError("Failed to register broker subscriber with iosource_mgr"); + if ( ! iosource_mgr->RegisterFd(bstate->status_subscriber.fd(), this) ) + reporter->FatalError("Failed to register broker status subscriber with iosource_mgr"); } void Manager::Terminate() { FlushLogBuffers(); - iosource_mgr->UnregisterFd(bstate->subscriber.fd()); - iosource_mgr->UnregisterFd(bstate->status_subscriber.fd()); + iosource_mgr->UnregisterFd(bstate->subscriber.fd(), this); + iosource_mgr->UnregisterFd(bstate->status_subscriber.fd(), this); vector stores_to_close; @@ -1503,7 +1505,7 @@ bool Manager::CloseStore(const string& name) if ( s == data_stores.end() ) return false; - iosource_mgr->UnregisterFd(s->second->proxy.mailbox().descriptor()); + iosource_mgr->UnregisterFd(s->second->proxy.mailbox().descriptor(), this); for ( auto i = pending_queries.begin(); i != pending_queries.end(); ) if ( i->second->Store().name() == name ) diff --git a/src/iosource/Manager.cc b/src/iosource/Manager.cc index c5791a2272..66b2beac5f 100644 --- a/src/iosource/Manager.cc +++ b/src/iosource/Manager.cc @@ -22,12 +22,13 @@ using namespace iosource; Manager::WakeupHandler::WakeupHandler() { - iosource_mgr->RegisterFd(flare.FD(), this); + if ( ! iosource_mgr->RegisterFd(flare.FD(), this) ) + reporter->FatalError("Failed to register WakeupHandler's fd with iosource_mgr"); } Manager::WakeupHandler::~WakeupHandler() { - iosource_mgr->UnregisterFd(flare.FD()); + iosource_mgr->UnregisterFd(flare.FD(), this); } void Manager::WakeupHandler::Process() @@ -207,7 +208,7 @@ void Manager::ConvertTimeout(double timeout, struct timespec& spec) } } -void Manager::RegisterFd(int fd, IOSource* src) +bool Manager::RegisterFd(int fd, IOSource* src) { struct kevent event; EV_SET(&event, fd, EVFILT_READ, EV_ADD, 0, 0, NULL); @@ -219,14 +220,16 @@ void Manager::RegisterFd(int fd, IOSource* src) fd_map[fd] = src; Wakeup("RegisterFd"); + return true; } else { - DBG_LOG(DBG_MAINLOOP, "Failed to register fd %d from %s: %s", fd, src->Tag(), strerror(errno)); + reporter->Error("Failed to register fd %d from %s: %s", fd, src->Tag(), strerror(errno)); + return false; } } -void Manager::UnregisterFd(int fd) +bool Manager::UnregisterFd(int fd, IOSource* src) { if ( fd_map.find(fd) != fd_map.end() ) { @@ -234,11 +237,17 @@ void Manager::UnregisterFd(int fd) EV_SET(&event, fd, EVFILT_READ, EV_DELETE, 0, 0, NULL); int ret = kevent(event_queue, &event, 1, NULL, 0, NULL); if ( ret != -1 ) - DBG_LOG(DBG_MAINLOOP, "Unregistered fd %d", fd); + DBG_LOG(DBG_MAINLOOP, "Unregistered fd %d from %s", fd, src->Tag()); fd_map.erase(fd); Wakeup("UnregisterFd"); + return true; + } + else + { + reporter->Error("Attempted to unregister an unknown file descriptor %d from %s", fd, src->Tag()); + return false; } } diff --git a/src/iosource/Manager.h b/src/iosource/Manager.h index ddf89db11a..1cd8eb965d 100644 --- a/src/iosource/Manager.h +++ b/src/iosource/Manager.h @@ -113,12 +113,12 @@ public: * checked for readiness. * @param src The IOSource that owns the file descriptor. */ - void RegisterFd(int fd, IOSource* src); + bool RegisterFd(int fd, IOSource* src); /** * Unregisters a file descriptor from the FindReadySources checks. */ - void UnregisterFd(int fd); + bool UnregisterFd(int fd, IOSource* src); /** * Forces the poll in FindReadySources to wake up immediately. This method diff --git a/src/supervisor/Supervisor.cc b/src/supervisor/Supervisor.cc index 4ef342ffb7..124c9b7771 100644 --- a/src/supervisor/Supervisor.cc +++ b/src/supervisor/Supervisor.cc @@ -358,8 +358,11 @@ void Supervisor::HandleChildSignal() void Supervisor::InitPostScript() { iosource_mgr->Register(this); - iosource_mgr->RegisterFd(signal_flare.FD(), this); - iosource_mgr->RegisterFd(stem_pipe->InFD(), this); + + if ( ! iosource_mgr->RegisterFd(signal_flare.FD(), this) ) + reporter->FatalError("Failed registration for signal_flare with iosource_mgr"); + if ( ! iosource_mgr->RegisterFd(stem_pipe->InFD(), this) ) + reporter->FatalError("Failed registration for stem_pipe with iosource_mgr"); } double Supervisor::GetNextTimeout() diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index 41267147d0..a1aac9ea15 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -181,7 +181,8 @@ MsgThread::MsgThread() : BasicThread(), queue_in(this, 0), queue_out(0, this) failed = false; thread_mgr->AddMsgThread(this); - iosource_mgr->RegisterFd(flare.FD(), this); + if ( ! iosource_mgr->RegisterFd(flare.FD(), this) ) + reporter->FatalError("Failed to register MsgThread fd with iosource_mgr"); SetClosed(false); } @@ -190,7 +191,7 @@ MsgThread::~MsgThread() { // Unregister this thread from the iosource manager so it doesn't wake // up the main poll anymore. - iosource_mgr->UnregisterFd(flare.FD()); + iosource_mgr->UnregisterFd(flare.FD(), this); } // Set by Bro's main signal handler. From 719011cef02a2a140578afd0163f2338896e2a89 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 31 Jan 2020 11:20:28 -0800 Subject: [PATCH 34/53] Remove extra fmt() in a reporter->Error() call Else compiler may warn about format string not being a string literal --- CHANGES | 20 ++++++++++++++++++++ VERSION | 2 +- src/parse.y | 2 +- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index e097c641ca..e8be148c8f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,24 @@ +3.1.0-dev.510 | 2020-01-31 11:20:28 -0800 + + * Remove extra fmt() in a reporter->Error() call (Jon Siwek, Corelight) + + * parse.y: Properly set location info for functions (Arne Welzel) + + When defining a function, remember the location where the function header + was and restore it before calling `end_func()`. Inside `end_func()`, a + `BroFunc` object is created using the current global location information. + + This came up while experimenting with zeek script profiling and wondering + why the locations set for `BroFunc` were "somewhere" in the middle of + functions instead of spanning them. + + * Desc: move realloc() call out of the loop (Max Kellermann) + + * SerializationFormat: move realloc() call out of the loop (Max Kellermann) + + * PacketDumper: remove unused types (Max Kellermann) + 3.1.0-dev.503 | 2020-01-31 11:00:32 -0800 * Show percentage of packets dropped in the final process output (Tim Wojtulewicz, Corelight) diff --git a/VERSION b/VERSION index 870c6e67b8..277b45b4e9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.0-dev.503 +3.1.0-dev.510 diff --git a/src/parse.y b/src/parse.y index 4eea70fe84..4487998c5d 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1168,7 +1168,7 @@ func_hdr: if ( streq("bro_init", name) || streq("bro_done", name) || streq("bro_script_loaded", name) ) { auto base = std::string(name).substr(4); - reporter->Error(fmt("event %s() is no longer available, use zeek_%s() instead", name, base.c_str())); + reporter->Error("event %s() is no longer available, use zeek_%s() instead", name, base.c_str()); } begin_func($2, current_module.c_str(), From b0a5eb27b19d336585ad35660904c7b08063fdf1 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 31 Jan 2020 14:19:21 -0800 Subject: [PATCH 35/53] Updating submodule(s). [nomail] --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index 024d1ecc95..ebc8450955 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 024d1ecc95195fc7e6f229f3c3b5a921e9c7de00 +Subproject commit ebc8450955fafa859932bd06cac85d828c772d3e From cd74d6f39262c7d1db1ee0daf8f258c4fdc58a0f Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 31 Jan 2020 15:07:24 -0800 Subject: [PATCH 36/53] Change various functions to by-value std::string_view args --- src/util.cc | 12 ++++++------ src/util.h | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/util.cc b/src/util.cc index 51bfc8639c..de3bacbe9d 100644 --- a/src/util.cc +++ b/src/util.cc @@ -63,7 +63,7 @@ #endif #endif -static bool starts_with(const std::string_view& s, const std::string& beginning) +static bool starts_with(std::string_view s, std::string_view beginning) { if ( beginning.size() > s.size() ) return false; @@ -78,7 +78,7 @@ TEST_CASE("util starts_with") CHECK(starts_with("abcde", "abcedf") == false); } -static bool ends_with(const std::string_view& s, const std::string& ending) +static bool ends_with(std::string_view s, std::string_view ending) { if ( ending.size() > s.size() ) return false; @@ -1315,7 +1315,7 @@ TEST_CASE("util is_package_loader") const array script_extensions = {".zeek", ".bro"}; -void warn_if_legacy_script(const std::string_view& filename) +void warn_if_legacy_script(std::string_view filename) { if ( ends_with(filename, ".bro") ) { @@ -1541,7 +1541,7 @@ TEST_CASE("util tokenize_string") CHECK(svs == expect); } -vector* tokenize_string(const std::string_view input, const std::string_view delim, +vector* tokenize_string(std::string_view input, std::string_view delim, vector* rval, int limit) { if ( ! rval ) @@ -1565,7 +1565,7 @@ vector* tokenize_string(const std::string_view input, const std::string_ return rval; } -vector tokenize_string(const std::string_view input, const char delim) noexcept +vector tokenize_string(std::string_view input, const char delim) noexcept { vector rval; @@ -1609,7 +1609,7 @@ TEST_CASE("util normalize_path") CHECK(normalize_path("zeek/../..") == ".."); } -string normalize_path(const std::string_view path) +string normalize_path(std::string_view path) { size_t n; vector final_components; diff --git a/src/util.h b/src/util.h index bd2447c23f..a8e23f39ef 100644 --- a/src/util.h +++ b/src/util.h @@ -150,7 +150,7 @@ std::vector* tokenize_string(std::string_view input, std::string_view delim, std::vector* rval = 0, int limit = 0); -std::vector tokenize_string(const std::string_view input, const char delim) noexcept; +std::vector tokenize_string(std::string_view input, const char delim) noexcept; extern char* copy_string(const char* s); extern int streq(const char* s1, const char* s2); @@ -278,7 +278,7 @@ extern std::string bro_prefixes(); extern const std::array script_extensions; /** Prints a warning if the filename ends in .bro. */ -void warn_if_legacy_script(const std::string_view& filename); +void warn_if_legacy_script(std::string_view filename); bool is_package_loader(const std::string& path); From 2f3611374361b6cbb134bfb17e96980e67724a20 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 31 Jan 2020 15:40:15 -0800 Subject: [PATCH 37/53] Change packet source fields of Options to std::optional --- src/Net.cc | 16 ++++++++-------- src/Net.h | 4 ++-- src/Options.cc | 11 +++++++---- src/Options.h | 4 ++-- src/main.cc | 7 +++---- 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/Net.cc b/src/Net.cc index 8148fb9ae9..77b7ad01fc 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -145,34 +145,34 @@ void net_update_time(double new_network_time) PLUGIN_HOOK_VOID(HOOK_UPDATE_NETWORK_TIME, HookUpdateNetworkTime(new_network_time)); } -void net_init(const std::string& interface, - const std::string& pcap_input_file, +void net_init(const std::optional& interface, + const std::optional& pcap_input_file, const std::optional& pcap_output_file, bool do_watchdog) { - if ( ! pcap_input_file.empty() ) + if ( pcap_input_file ) { reading_live = pseudo_realtime > 0.0; reading_traces = true; - iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(pcap_input_file, false); + iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(*pcap_input_file, false); assert(ps); if ( ! ps->IsOpen() ) reporter->FatalError("problem with trace file %s (%s)", - pcap_input_file.c_str(), ps->ErrorMsg()); + pcap_input_file->c_str(), ps->ErrorMsg()); } - else if ( ! interface.empty() ) + else if ( interface ) { reading_live = true; reading_traces = false; - iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(interface, true); + iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(*interface, true); assert(ps); if ( ! ps->IsOpen() ) reporter->FatalError("problem with interface %s (%s)", - interface.c_str(), ps->ErrorMsg()); + interface->c_str(), ps->ErrorMsg()); } else diff --git a/src/Net.h b/src/Net.h index b1a3de427c..655bab8c8e 100644 --- a/src/Net.h +++ b/src/Net.h @@ -19,8 +19,8 @@ namespace iosource { class Packet; -extern void net_init(const std::string& interfaces, - const std::string& pcap_input_file, +extern void net_init(const std::optional& interfaces, + const std::optional& pcap_input_file, const std::optional& pcap_output_file, bool do_watchdog); extern void net_run(); diff --git a/src/Options.cc b/src/Options.cc index 2468675c24..27fc648ce3 100644 --- a/src/Options.cc +++ b/src/Options.cc @@ -268,12 +268,13 @@ zeek::Options zeek::parse_cmdline(int argc, char** argv) rval.print_usage = true; break; case 'i': - if ( ! rval.interface.empty() ) + if ( rval.interface ) { fprintf(stderr, "ERROR: Only a single interface option (-i) is allowed.\n"); exit(1); } - else if ( ! rval.pcap_file.empty() ) + + if ( rval.pcap_file ) { fprintf(stderr, "ERROR: Using -i is not allow when reading a pcap file.\n"); exit(1); @@ -294,16 +295,18 @@ zeek::Options zeek::parse_cmdline(int argc, char** argv) rval.script_prefixes.emplace_back(optarg); break; case 'r': - if ( ! rval.pcap_file.empty() ) + if ( rval.pcap_file ) { fprintf(stderr, "ERROR: Only a single readfile option (-r) is allowed.\n"); exit(1); } - else if ( ! rval.interface.empty() ) + + if ( rval.interface ) { fprintf(stderr, "Using -r is not allowed when reading a live interface.\n"); exit(1); } + rval.pcap_file = optarg; break; case 's': diff --git a/src/Options.h b/src/Options.h index ca61493cb0..496a5fa203 100644 --- a/src/Options.h +++ b/src/Options.h @@ -58,8 +58,8 @@ struct Options { std::vector doctest_args; std::optional pcap_filter; - std::string interface; - std::string pcap_file; + std::optional interface; + std::optional pcap_file; std::vector signature_files; std::optional pcap_output_file; diff --git a/src/main.cc b/src/main.cc index ffcce06bc3..c4fae32fb4 100644 --- a/src/main.cc +++ b/src/main.cc @@ -565,8 +565,7 @@ int main(int argc, char** argv) if ( options.plugins_to_load.empty() && options.scripts_to_load.empty() && options.script_options_to_set.empty() && - options.pcap_file.empty() && - options.interface.empty() && + ! options.pcap_file && ! options.interface && ! options.identifier_to_print && ! command_line_policy && ! options.print_plugins && ! options.supervisor_mode && ! zeek::Supervisor::ThisNode() ) @@ -596,7 +595,7 @@ int main(int argc, char** argv) log_mgr = new logging::Manager(); input_mgr = new input::Manager(); file_mgr = new file_analysis::Manager(); - broker_mgr = new bro_broker::Manager(! options.pcap_file.empty()); + broker_mgr = new bro_broker::Manager(options.pcap_file.has_value()); trigger_mgr = new trigger::Manager(); plugin_mgr->InitPreScript(); @@ -745,7 +744,7 @@ int main(int argc, char** argv) // ### Add support for debug command file. dbg_init_debugger(0); - if ( options.pcap_file.empty() && options.interface.empty() ) + if ( ! options.pcap_file && ! options.interface ) { Val* interfaces_val = internal_val("interfaces"); if ( interfaces_val ) From 63344b32f59d47d3b1a52519979aa8584ba5bff5 Mon Sep 17 00:00:00 2001 From: Fabrice Fontaine Date: Sat, 1 Feb 2020 14:28:43 +0100 Subject: [PATCH 38/53] CMakeLists.txt: fix cross-compilation with binpac When cross-compiling, BINPAC_EXE_PATH will be set by the user to the host binpac binary which is fine however aux/binpac won't be built which will raise a build failure as target binpac (headers, library) won't be installed or built Signed-off-by: Fabrice Fontaine --- CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 53c6d4faca..f4401d05c8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -178,8 +178,7 @@ FindRequiredPackage(OpenSSL) FindRequiredPackage(BIND) FindRequiredPackage(ZLIB) -if (NOT BINPAC_EXE_PATH AND - EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/aux/binpac/CMakeLists.txt) +if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/aux/binpac/CMakeLists.txt) set(ENABLE_STATIC_ONLY_SAVED ${ENABLE_STATIC_ONLY}) From d12d0332a707d19579f5253e183f9e1e99b91a15 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Sat, 1 Feb 2020 19:23:17 -0700 Subject: [PATCH 39/53] Change order of includes in iosource Manager, fixes build on FreeBSD 11 --- src/iosource/Manager.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/iosource/Manager.cc b/src/iosource/Manager.cc index 66b2beac5f..ed79a30763 100644 --- a/src/iosource/Manager.cc +++ b/src/iosource/Manager.cc @@ -1,7 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include #include +#include #include #include #include From efaa7565390bb46ea7ff713d24976b95a0cc3f35 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Sat, 1 Feb 2020 19:37:31 -0700 Subject: [PATCH 40/53] Use ranged-for loops in a few places in iosource Manager --- src/iosource/Manager.cc | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/iosource/Manager.cc b/src/iosource/Manager.cc index ed79a30763..b93bb03c60 100644 --- a/src/iosource/Manager.cc +++ b/src/iosource/Manager.cc @@ -255,11 +255,11 @@ void Manager::Register(IOSource* src, bool dont_count) { // First see if we already have registered that source. If so, just // adjust dont_count. - for ( SourceList::iterator i = sources.begin(); i != sources.end(); ++i ) + for ( const auto& iosrc : sources ) { - if ( (*i)->src == src ) + if ( iosrc->src == src ) { - if ( (*i)->dont_count != dont_count ) + if ( iosrc->dont_count != dont_count ) // Adjust the global counter. dont_counts += (dont_count ? 1 : -1); @@ -323,12 +323,8 @@ PktSrc* Manager::OpenPktSrc(const std::string& path, bool is_live) PktSrcComponent* component = 0; std::list all_components = plugin_mgr->Components(); - - for ( std::list::const_iterator i = all_components.begin(); - i != all_components.end(); i++ ) + for ( const auto& c : all_components ) { - PktSrcComponent* c = *i; - if ( c->HandlesPrefix(prefix) && (( is_live && c->DoesLive() ) || (! is_live && c->DoesTrace())) ) @@ -369,13 +365,11 @@ PktDumper* Manager::OpenPktDumper(const string& path, bool append) PktDumperComponent* component = 0; std::list all_components = plugin_mgr->Components(); - - for ( std::list::const_iterator i = all_components.begin(); - i != all_components.end(); i++ ) + for ( const auto& c : all_components ) { - if ( (*i)->HandlesPrefix(prefix) ) + if ( c->HandlesPrefix(prefix) ) { - component = (*i); + component = c; break; } } From edb87821da873907acb97ce19bf8f64a0dbfeab5 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Sat, 1 Feb 2020 19:46:35 -0700 Subject: [PATCH 41/53] Increase timeout value for live interfaces without file descriptors --- src/iosource/PktSrc.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/iosource/PktSrc.cc b/src/iosource/PktSrc.cc index c61aebc2d1..16eaf5aec7 100644 --- a/src/iosource/PktSrc.cc +++ b/src/iosource/PktSrc.cc @@ -335,10 +335,10 @@ double PktSrc::GetNextTimeout() // If there's no file descriptor for the source, which is the case for some interfaces like // myricom, we can't rely on the polling mechanism to wait for data to be available. As gross // as it is, just spin with a short timeout here so that it will continually poll the - // interface. The old IOSource code had a 20ns timeout between calls to select() so just - // use that. + // interface. The old IOSource code had a 20 microsecond timeout between calls to select() + // so just use that. if ( props.selectable_fd == -1 ) - return 0.00000002; + return 0.00002; // If we're live we want poll to do what it has to with the file descriptor. If we're not live // but we're not in pseudo-realtime mode, let the loop just spin as fast as it can. If we're From 1e2dd4466c9854e18e8774051c7cb86419f9c330 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 31 Jan 2020 19:16:13 -0800 Subject: [PATCH 42/53] Change requested memory in Cirrus CI config The ASan build does require quite a bit more, but also the default release builds should do ok with slightly less. --- .cirrus.yml | 13 +++++++++---- CHANGES | 7 +++++++ VERSION | 2 +- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index b6dd77e461..a421d1e1cb 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,6 +1,6 @@ cpus: &CPUS 8 btest_jobs: &BTEST_JOBS 8 -memory: &MEMORY 8GB +memory: &MEMORY 6GB config: &CONFIG --build-type=release --enable-cpp-tests memcheck_config: &MEMCHECK_CONFIG --build-type=debug --enable-cpp-tests --sanitizers=address @@ -95,10 +95,11 @@ ubuntu16_task: macos_task: osx_instance: image: catalina-base + # cpu/memory setting is implicitly 2 core / 4 thread and 8GB, and + # trying to set it explicitly results in an error. prepare_script: ./ci/macos/prepare.sh << : *CI_TEMPLATE env: - # Currently only available as 2 core / 4 thread (and 8GB) instances. ZEEK_CI_CPUS: 4 ZEEK_CI_BTEST_JOBS: 4 # No permission to write to default location of /zeek @@ -109,7 +110,9 @@ freebsd_task: freebsd_instance: # FreeBSD 12 EOL: June 30, 2024 image_family: freebsd-12-1 - << : *RESOURCES_TEMPLATE + cpu: 8 + # Not allowed to request less than 8GB for an 8 CPU FreeBSD VM. + memory: 8GB prepare_script: ./ci/freebsd/prepare.sh << : *CI_TEMPLATE @@ -117,7 +120,9 @@ memcheck_task: container: # Just uses a recent/common distro to run memory error/leak checks. dockerfile: ci/ubuntu-18.04/Dockerfile - << : *RESOURCES_TEMPLATE + cpu: 8 + # AddressSanitizer uses a lot more memory than a typical config. + memory: 16GB << : *CI_TEMPLATE env: ZEEK_CI_CONFIGURE_FLAGS: *MEMCHECK_CONFIG diff --git a/CHANGES b/CHANGES index e6c981301c..7391447b9a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,11 @@ +3.1.0-dev.527 | 2020-02-03 10:11:00 -0800 + + * Change requested memory in Cirrus CI config (Jon Siwek, Corelight) + + The ASan build does require quite a bit more, but also the default + release builds should do ok with slightly less. + 3.1.0-dev.526 | 2020-02-03 10:04:00 -0800 * Increase timeout value for live interfaces without file descriptors (Tim Wojtulewicz, Corelight) diff --git a/VERSION b/VERSION index 167919b1ad..affde92154 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.0-dev.526 +3.1.0-dev.527 From 2cda94e8eef69cc3472834517fdd8646a8f656f9 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 31 Jan 2020 19:26:29 -0800 Subject: [PATCH 43/53] Disable Travis leak test Travis environment doesn't offer enough memory for running ASan very well, but Cirrus has it covered anyway. --- .travis.yml | 3 ++- CHANGES | 7 +++++++ VERSION | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6f5791cef4..0d9ac5be26 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,8 @@ env: - distro: fedora_30 - distro: ubuntu_16.04 - distro: ubuntu_18.04 - - distro: ubuntu_18.04_leaktest +# Travis environment doesn't have enough memory to support ASan well. +# - distro: ubuntu_18.04_leaktest install: sh ci/travis-job install $distro diff --git a/CHANGES b/CHANGES index 808b9959bd..1e0565a9b4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,11 @@ +3.1.0-dev.530 | 2020-02-03 13:21:16 -0800 + + * Disable Travis leak test (Jon Siwek, Corelight) + + Travis environment doesn't offer enough memory for running ASan very + well, but Cirrus has it covered anyway. + 3.1.0-dev.529 | 2020-02-03 13:04:30 -0800 * CMakeLists.txt: fix cross-compilation with binpac (Fabrice Fontaine) diff --git a/VERSION b/VERSION index 9ab9ece2ac..0e558a147e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.0-dev.529 +3.1.0-dev.530 From c20551e48ee32e157a4994930efbc544c025da08 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Mon, 3 Feb 2020 13:23:22 -0800 Subject: [PATCH 44/53] Update submodule [nomail] --- doc | 2 +- scripts/base/protocols/ssl/ct-list.zeek | 2 +- scripts/base/protocols/ssl/mozilla-ca-list.zeek | 7 +++---- src/3rdparty | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/doc b/doc index ebc8450955..a7f0e850a4 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit ebc8450955fafa859932bd06cac85d828c772d3e +Subproject commit a7f0e850a42c3624b1460967ab9436e4d4f34644 diff --git a/scripts/base/protocols/ssl/ct-list.zeek b/scripts/base/protocols/ssl/ct-list.zeek index d6efa95b34..17355f6265 100644 --- a/scripts/base/protocols/ssl/ct-list.zeek +++ b/scripts/base/protocols/ssl/ct-list.zeek @@ -1,6 +1,6 @@ # # Do not edit this file. This file is automatically generated by gen-ct-list.pl -# File generated at Fri Aug 2 12:42:14 2019 +# File generated at Mon Feb 3 12:39:48 2020 # File generated from https://www.gstatic.com/ct/log_list/all_logs_list.json # diff --git a/scripts/base/protocols/ssl/mozilla-ca-list.zeek b/scripts/base/protocols/ssl/mozilla-ca-list.zeek index 6096097d36..374a9c1875 100644 --- a/scripts/base/protocols/ssl/mozilla-ca-list.zeek +++ b/scripts/base/protocols/ssl/mozilla-ca-list.zeek @@ -1,6 +1,6 @@ # Don't edit! This file is automatically generated. -# Generated at: 2019-08-02 12:30:04 -0700 -# Generated from: NSS 3.45 +# Generated at: 2020-02-03 13:10:45 -0800 +# Generated from: NSS 3.49.2 # # The original source file comes with this licensing statement: # @@ -34,7 +34,6 @@ redef root_certs += { ["CN=DigiCert Assured ID Root CA,OU=www.digicert.com,O=DigiCert Inc,C=US"] = "\x30\x82\x03\xB7\x30\x82\x02\x9F\xA0\x03\x02\x01\x02\x02\x10\x0C\xE7\xE0\xE5\x17\xD8\x46\xFE\x8F\xE5\x60\xFC\x1B\xF0\x30\x39\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x65\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x15\x30\x13\x06\x03\x55\x04\x0A\x13\x0C\x44\x69\x67\x69\x43\x65\x72\x74\x20\x49\x6E\x63\x31\x19\x30\x17\x06\x03\x55\x04\x0B\x13\x10\x77\x77\x77\x2E\x64\x69\x67\x69\x63\x65\x72\x74\x2E\x63\x6F\x6D\x31\x24\x30\x22\x06\x03\x55\x04\x03\x13\x1B\x44\x69\x67\x69\x43\x65\x72\x74\x20\x41\x73\x73\x75\x72\x65\x64\x20\x49\x44\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x1E\x17\x0D\x30\x36\x31\x31\x31\x30\x30\x30\x30\x30\x30\x30\x5A\x17\x0D\x33\x31\x31\x31\x31\x30\x30\x30\x30\x30\x30\x30\x5A\x30\x65\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x15\x30\x13\x06\x03\x55\x04\x0A\x13\x0C\x44\x69\x67\x69\x43\x65\x72\x74\x20\x49\x6E\x63\x31\x19\x30\x17\x06\x03\x55\x04\x0B\x13\x10\x77\x77\x77\x2E\x64\x69\x67\x69\x63\x65\x72\x74\x2E\x63\x6F\x6D\x31\x24\x30\x22\x06\x03\x55\x04\x03\x13\x1B\x44\x69\x67\x69\x43\x65\x72\x74\x20\x41\x73\x73\x75\x72\x65\x64\x20\x49\x44\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xAD\x0E\x15\xCE\xE4\x43\x80\x5C\xB1\x87\xF3\xB7\x60\xF9\x71\x12\xA5\xAE\xDC\x26\x94\x88\xAA\xF4\xCE\xF5\x20\x39\x28\x58\x60\x0C\xF8\x80\xDA\xA9\x15\x95\x32\x61\x3C\xB5\xB1\x28\x84\x8A\x8A\xDC\x9F\x0A\x0C\x83\x17\x7A\x8F\x90\xAC\x8A\xE7\x79\x53\x5C\x31\x84\x2A\xF6\x0F\x98\x32\x36\x76\xCC\xDE\xDD\x3C\xA8\xA2\xEF\x6A\xFB\x21\xF2\x52\x61\xDF\x9F\x20\xD7\x1F\xE2\xB1\xD9\xFE\x18\x64\xD2\x12\x5B\x5F\xF9\x58\x18\x35\xBC\x47\xCD\xA1\x36\xF9\x6B\x7F\xD4\xB0\x38\x3E\xC1\x1B\xC3\x8C\x33\xD9\xD8\x2F\x18\xFE\x28\x0F\xB3\xA7\x83\xD6\xC3\x6E\x44\xC0\x61\x35\x96\x16\xFE\x59\x9C\x8B\x76\x6D\xD7\xF1\xA2\x4B\x0D\x2B\xFF\x0B\x72\xDA\x9E\x60\xD0\x8E\x90\x35\xC6\x78\x55\x87\x20\xA1\xCF\xE5\x6D\x0A\xC8\x49\x7C\x31\x98\x33\x6C\x22\xE9\x87\xD0\x32\x5A\xA2\xBA\x13\x82\x11\xED\x39\x17\x9D\x99\x3A\x72\xA1\xE6\xFA\xA4\xD9\xD5\x17\x31\x75\xAE\x85\x7D\x22\xAE\x3F\x01\x46\x86\xF6\x28\x79\xC8\xB1\xDA\xE4\x57\x17\xC4\x7E\x1C\x0E\xB0\xB4\x92\xA6\x56\xB3\xBD\xB2\x97\xED\xAA\xA7\xF0\xB7\xC5\xA8\x3F\x95\x16\xD0\xFF\xA1\x96\xEB\x08\x5F\x18\x77\x4F\x02\x03\x01\x00\x01\xA3\x63\x30\x61\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x86\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x45\xEB\xA2\xAF\xF4\x92\xCB\x82\x31\x2D\x51\x8B\xA7\xA7\x21\x9D\xF3\x6D\xC8\x0F\x30\x1F\x06\x03\x55\x1D\x23\x04\x18\x30\x16\x80\x14\x45\xEB\xA2\xAF\xF4\x92\xCB\x82\x31\x2D\x51\x8B\xA7\xA7\x21\x9D\xF3\x6D\xC8\x0F\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\xA2\x0E\xBC\xDF\xE2\xED\xF0\xE3\x72\x73\x7A\x64\x94\xBF\xF7\x72\x66\xD8\x32\xE4\x42\x75\x62\xAE\x87\xEB\xF2\xD5\xD9\xDE\x56\xB3\x9F\xCC\xCE\x14\x28\xB9\x0D\x97\x60\x5C\x12\x4C\x58\xE4\xD3\x3D\x83\x49\x45\x58\x97\x35\x69\x1A\xA8\x47\xEA\x56\xC6\x79\xAB\x12\xD8\x67\x81\x84\xDF\x7F\x09\x3C\x94\xE6\xB8\x26\x2C\x20\xBD\x3D\xB3\x28\x89\xF7\x5F\xFF\x22\xE2\x97\x84\x1F\xE9\x65\xEF\x87\xE0\xDF\xC1\x67\x49\xB3\x5D\xEB\xB2\x09\x2A\xEB\x26\xED\x78\xBE\x7D\x3F\x2B\xF3\xB7\x26\x35\x6D\x5F\x89\x01\xB6\x49\x5B\x9F\x01\x05\x9B\xAB\x3D\x25\xC1\xCC\xB6\x7F\xC2\xF1\x6F\x86\xC6\xFA\x64\x68\xEB\x81\x2D\x94\xEB\x42\xB7\xFA\x8C\x1E\xDD\x62\xF1\xBE\x50\x67\xB7\x6C\xBD\xF3\xF1\x1F\x6B\x0C\x36\x07\x16\x7F\x37\x7C\xA9\x5B\x6D\x7A\xF1\x12\x46\x60\x83\xD7\x27\x04\xBE\x4B\xCE\x97\xBE\xC3\x67\x2A\x68\x11\xDF\x80\xE7\x0C\x33\x66\xBF\x13\x0D\x14\x6E\xF3\x7F\x1F\x63\x10\x1E\xFA\x8D\x1B\x25\x6D\x6C\x8F\xA5\xB7\x61\x01\xB1\xD2\xA3\x26\xA1\x10\x71\x9D\xAD\xE2\xC3\xF9\xC3\x99\x51\xB7\x2B\x07\x08\xCE\x2E\xE6\x50\xB2\xA7\xFA\x0A\x45\x2F\xA2\xF0\xF2", ["CN=DigiCert Global Root CA,OU=www.digicert.com,O=DigiCert Inc,C=US"] = "\x30\x82\x03\xAF\x30\x82\x02\x97\xA0\x03\x02\x01\x02\x02\x10\x08\x3B\xE0\x56\x90\x42\x46\xB1\xA1\x75\x6A\xC9\x59\x91\xC7\x4A\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x61\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x15\x30\x13\x06\x03\x55\x04\x0A\x13\x0C\x44\x69\x67\x69\x43\x65\x72\x74\x20\x49\x6E\x63\x31\x19\x30\x17\x06\x03\x55\x04\x0B\x13\x10\x77\x77\x77\x2E\x64\x69\x67\x69\x63\x65\x72\x74\x2E\x63\x6F\x6D\x31\x20\x30\x1E\x06\x03\x55\x04\x03\x13\x17\x44\x69\x67\x69\x43\x65\x72\x74\x20\x47\x6C\x6F\x62\x61\x6C\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x1E\x17\x0D\x30\x36\x31\x31\x31\x30\x30\x30\x30\x30\x30\x30\x5A\x17\x0D\x33\x31\x31\x31\x31\x30\x30\x30\x30\x30\x30\x30\x5A\x30\x61\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x15\x30\x13\x06\x03\x55\x04\x0A\x13\x0C\x44\x69\x67\x69\x43\x65\x72\x74\x20\x49\x6E\x63\x31\x19\x30\x17\x06\x03\x55\x04\x0B\x13\x10\x77\x77\x77\x2E\x64\x69\x67\x69\x63\x65\x72\x74\x2E\x63\x6F\x6D\x31\x20\x30\x1E\x06\x03\x55\x04\x03\x13\x17\x44\x69\x67\x69\x43\x65\x72\x74\x20\x47\x6C\x6F\x62\x61\x6C\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xE2\x3B\xE1\x11\x72\xDE\xA8\xA4\xD3\xA3\x57\xAA\x50\xA2\x8F\x0B\x77\x90\xC9\xA2\xA5\xEE\x12\xCE\x96\x5B\x01\x09\x20\xCC\x01\x93\xA7\x4E\x30\xB7\x53\xF7\x43\xC4\x69\x00\x57\x9D\xE2\x8D\x22\xDD\x87\x06\x40\x00\x81\x09\xCE\xCE\x1B\x83\xBF\xDF\xCD\x3B\x71\x46\xE2\xD6\x66\xC7\x05\xB3\x76\x27\x16\x8F\x7B\x9E\x1E\x95\x7D\xEE\xB7\x48\xA3\x08\xDA\xD6\xAF\x7A\x0C\x39\x06\x65\x7F\x4A\x5D\x1F\xBC\x17\xF8\xAB\xBE\xEE\x28\xD7\x74\x7F\x7A\x78\x99\x59\x85\x68\x6E\x5C\x23\x32\x4B\xBF\x4E\xC0\xE8\x5A\x6D\xE3\x70\xBF\x77\x10\xBF\xFC\x01\xF6\x85\xD9\xA8\x44\x10\x58\x32\xA9\x75\x18\xD5\xD1\xA2\xBE\x47\xE2\x27\x6A\xF4\x9A\x33\xF8\x49\x08\x60\x8B\xD4\x5F\xB4\x3A\x84\xBF\xA1\xAA\x4A\x4C\x7D\x3E\xCF\x4F\x5F\x6C\x76\x5E\xA0\x4B\x37\x91\x9E\xDC\x22\xE6\x6D\xCE\x14\x1A\x8E\x6A\xCB\xFE\xCD\xB3\x14\x64\x17\xC7\x5B\x29\x9E\x32\xBF\xF2\xEE\xFA\xD3\x0B\x42\xD4\xAB\xB7\x41\x32\xDA\x0C\xD4\xEF\xF8\x81\xD5\xBB\x8D\x58\x3F\xB5\x1B\xE8\x49\x28\xA2\x70\xDA\x31\x04\xDD\xF7\xB2\x16\xF2\x4C\x0A\x4E\x07\xA8\xED\x4A\x3D\x5E\xB5\x7F\xA3\x90\xC3\xAF\x27\x02\x03\x01\x00\x01\xA3\x63\x30\x61\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x86\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x03\xDE\x50\x35\x56\xD1\x4C\xBB\x66\xF0\xA3\xE2\x1B\x1B\xC3\x97\xB2\x3D\xD1\x55\x30\x1F\x06\x03\x55\x1D\x23\x04\x18\x30\x16\x80\x14\x03\xDE\x50\x35\x56\xD1\x4C\xBB\x66\xF0\xA3\xE2\x1B\x1B\xC3\x97\xB2\x3D\xD1\x55\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\xCB\x9C\x37\xAA\x48\x13\x12\x0A\xFA\xDD\x44\x9C\x4F\x52\xB0\xF4\xDF\xAE\x04\xF5\x79\x79\x08\xA3\x24\x18\xFC\x4B\x2B\x84\xC0\x2D\xB9\xD5\xC7\xFE\xF4\xC1\x1F\x58\xCB\xB8\x6D\x9C\x7A\x74\xE7\x98\x29\xAB\x11\xB5\xE3\x70\xA0\xA1\xCD\x4C\x88\x99\x93\x8C\x91\x70\xE2\xAB\x0F\x1C\xBE\x93\xA9\xFF\x63\xD5\xE4\x07\x60\xD3\xA3\xBF\x9D\x5B\x09\xF1\xD5\x8E\xE3\x53\xF4\x8E\x63\xFA\x3F\xA7\xDB\xB4\x66\xDF\x62\x66\xD6\xD1\x6E\x41\x8D\xF2\x2D\xB5\xEA\x77\x4A\x9F\x9D\x58\xE2\x2B\x59\xC0\x40\x23\xED\x2D\x28\x82\x45\x3E\x79\x54\x92\x26\x98\xE0\x80\x48\xA8\x37\xEF\xF0\xD6\x79\x60\x16\xDE\xAC\xE8\x0E\xCD\x6E\xAC\x44\x17\x38\x2F\x49\xDA\xE1\x45\x3E\x2A\xB9\x36\x53\xCF\x3A\x50\x06\xF7\x2E\xE8\xC4\x57\x49\x6C\x61\x21\x18\xD5\x04\xAD\x78\x3C\x2C\x3A\x80\x6B\xA7\xEB\xAF\x15\x14\xE9\xD8\x89\xC1\xB9\x38\x6C\xE2\x91\x6C\x8A\xFF\x64\xB9\x77\x25\x57\x30\xC0\x1B\x24\xA3\xE1\xDC\xE9\xDF\x47\x7C\xB5\xB4\x24\x08\x05\x30\xEC\x2D\xBD\x0B\xBF\x45\xBF\x50\xB9\xA9\xF3\xEB\x98\x01\x12\xAD\xC8\x88\xC6\x98\x34\x5F\x8D\x0A\x3C\xC6\xE9\xD5\x95\x95\x6D\xDE", ["CN=DigiCert High Assurance EV Root CA,OU=www.digicert.com,O=DigiCert Inc,C=US"] = "\x30\x82\x03\xC5\x30\x82\x02\xAD\xA0\x03\x02\x01\x02\x02\x10\x02\xAC\x5C\x26\x6A\x0B\x40\x9B\x8F\x0B\x79\xF2\xAE\x46\x25\x77\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x6C\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x15\x30\x13\x06\x03\x55\x04\x0A\x13\x0C\x44\x69\x67\x69\x43\x65\x72\x74\x20\x49\x6E\x63\x31\x19\x30\x17\x06\x03\x55\x04\x0B\x13\x10\x77\x77\x77\x2E\x64\x69\x67\x69\x63\x65\x72\x74\x2E\x63\x6F\x6D\x31\x2B\x30\x29\x06\x03\x55\x04\x03\x13\x22\x44\x69\x67\x69\x43\x65\x72\x74\x20\x48\x69\x67\x68\x20\x41\x73\x73\x75\x72\x61\x6E\x63\x65\x20\x45\x56\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x1E\x17\x0D\x30\x36\x31\x31\x31\x30\x30\x30\x30\x30\x30\x30\x5A\x17\x0D\x33\x31\x31\x31\x31\x30\x30\x30\x30\x30\x30\x30\x5A\x30\x6C\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x15\x30\x13\x06\x03\x55\x04\x0A\x13\x0C\x44\x69\x67\x69\x43\x65\x72\x74\x20\x49\x6E\x63\x31\x19\x30\x17\x06\x03\x55\x04\x0B\x13\x10\x77\x77\x77\x2E\x64\x69\x67\x69\x63\x65\x72\x74\x2E\x63\x6F\x6D\x31\x2B\x30\x29\x06\x03\x55\x04\x03\x13\x22\x44\x69\x67\x69\x43\x65\x72\x74\x20\x48\x69\x67\x68\x20\x41\x73\x73\x75\x72\x61\x6E\x63\x65\x20\x45\x56\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xC6\xCC\xE5\x73\xE6\xFB\xD4\xBB\xE5\x2D\x2D\x32\xA6\xDF\xE5\x81\x3F\xC9\xCD\x25\x49\xB6\x71\x2A\xC3\xD5\x94\x34\x67\xA2\x0A\x1C\xB0\x5F\x69\xA6\x40\xB1\xC4\xB7\xB2\x8F\xD0\x98\xA4\xA9\x41\x59\x3A\xD3\xDC\x94\xD6\x3C\xDB\x74\x38\xA4\x4A\xCC\x4D\x25\x82\xF7\x4A\xA5\x53\x12\x38\xEE\xF3\x49\x6D\x71\x91\x7E\x63\xB6\xAB\xA6\x5F\xC3\xA4\x84\xF8\x4F\x62\x51\xBE\xF8\xC5\xEC\xDB\x38\x92\xE3\x06\xE5\x08\x91\x0C\xC4\x28\x41\x55\xFB\xCB\x5A\x89\x15\x7E\x71\xE8\x35\xBF\x4D\x72\x09\x3D\xBE\x3A\x38\x50\x5B\x77\x31\x1B\x8D\xB3\xC7\x24\x45\x9A\xA7\xAC\x6D\x00\x14\x5A\x04\xB7\xBA\x13\xEB\x51\x0A\x98\x41\x41\x22\x4E\x65\x61\x87\x81\x41\x50\xA6\x79\x5C\x89\xDE\x19\x4A\x57\xD5\x2E\xE6\x5D\x1C\x53\x2C\x7E\x98\xCD\x1A\x06\x16\xA4\x68\x73\xD0\x34\x04\x13\x5C\xA1\x71\xD3\x5A\x7C\x55\xDB\x5E\x64\xE1\x37\x87\x30\x56\x04\xE5\x11\xB4\x29\x80\x12\xF1\x79\x39\x88\xA2\x02\x11\x7C\x27\x66\xB7\x88\xB7\x78\xF2\xCA\x0A\xA8\x38\xAB\x0A\x64\xC2\xBF\x66\x5D\x95\x84\xC1\xA1\x25\x1E\x87\x5D\x1A\x50\x0B\x20\x12\xCC\x41\xBB\x6E\x0B\x51\x38\xB8\x4B\xCB\x02\x03\x01\x00\x01\xA3\x63\x30\x61\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x86\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\xB1\x3E\xC3\x69\x03\xF8\xBF\x47\x01\xD4\x98\x26\x1A\x08\x02\xEF\x63\x64\x2B\xC3\x30\x1F\x06\x03\x55\x1D\x23\x04\x18\x30\x16\x80\x14\xB1\x3E\xC3\x69\x03\xF8\xBF\x47\x01\xD4\x98\x26\x1A\x08\x02\xEF\x63\x64\x2B\xC3\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x1C\x1A\x06\x97\xDC\xD7\x9C\x9F\x3C\x88\x66\x06\x08\x57\x21\xDB\x21\x47\xF8\x2A\x67\xAA\xBF\x18\x32\x76\x40\x10\x57\xC1\x8A\xF3\x7A\xD9\x11\x65\x8E\x35\xFA\x9E\xFC\x45\xB5\x9E\xD9\x4C\x31\x4B\xB8\x91\xE8\x43\x2C\x8E\xB3\x78\xCE\xDB\xE3\x53\x79\x71\xD6\xE5\x21\x94\x01\xDA\x55\x87\x9A\x24\x64\xF6\x8A\x66\xCC\xDE\x9C\x37\xCD\xA8\x34\xB1\x69\x9B\x23\xC8\x9E\x78\x22\x2B\x70\x43\xE3\x55\x47\x31\x61\x19\xEF\x58\xC5\x85\x2F\x4E\x30\xF6\xA0\x31\x16\x23\xC8\xE7\xE2\x65\x16\x33\xCB\xBF\x1A\x1B\xA0\x3D\xF8\xCA\x5E\x8B\x31\x8B\x60\x08\x89\x2D\x0C\x06\x5C\x52\xB7\xC4\xF9\x0A\x98\xD1\x15\x5F\x9F\x12\xBE\x7C\x36\x63\x38\xBD\x44\xA4\x7F\xE4\x26\x2B\x0A\xC4\x97\x69\x0D\xE9\x8C\xE2\xC0\x10\x57\xB8\xC8\x76\x12\x91\x55\xF2\x48\x69\xD8\xBC\x2A\x02\x5B\x0F\x44\xD4\x20\x31\xDB\xF4\xBA\x70\x26\x5D\x90\x60\x9E\xBC\x4B\x17\x09\x2F\xB4\xCB\x1E\x43\x68\xC9\x07\x27\xC1\xD2\x5C\xF7\xEA\x21\xB9\x68\x12\x9C\x3C\x9C\xBF\x9E\xFC\x80\x5C\x9B\x63\xCD\xEC\x47\xAA\x25\x27\x67\xA0\x37\xF3\x00\x82\x7D\x54\xD7\xA9\xF8\xE9\x2E\x13\xA3\x77\xE8\x1F\x4A", - ["CN=Class 2 Primary CA,O=Certplus,C=FR"] = "\x30\x82\x03\x92\x30\x82\x02\x7A\xA0\x03\x02\x01\x02\x02\x11\x00\x85\xBD\x4B\xF3\xD8\xDA\xE3\x69\xF6\x94\xD7\x5F\xC3\xA5\x44\x23\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x3D\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x46\x52\x31\x11\x30\x0F\x06\x03\x55\x04\x0A\x13\x08\x43\x65\x72\x74\x70\x6C\x75\x73\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x43\x6C\x61\x73\x73\x20\x32\x20\x50\x72\x69\x6D\x61\x72\x79\x20\x43\x41\x30\x1E\x17\x0D\x39\x39\x30\x37\x30\x37\x31\x37\x30\x35\x30\x30\x5A\x17\x0D\x31\x39\x30\x37\x30\x36\x32\x33\x35\x39\x35\x39\x5A\x30\x3D\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x46\x52\x31\x11\x30\x0F\x06\x03\x55\x04\x0A\x13\x08\x43\x65\x72\x74\x70\x6C\x75\x73\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x43\x6C\x61\x73\x73\x20\x32\x20\x50\x72\x69\x6D\x61\x72\x79\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xDC\x50\x96\xD0\x12\xF8\x35\xD2\x08\x78\x7A\xB6\x52\x70\xFD\x6F\xEE\xCF\xB9\x11\xCB\x5D\x77\xE1\xEC\xE9\x7E\x04\x8D\xD6\xCC\x6F\x73\x43\x57\x60\xAC\x33\x0A\x44\xEC\x03\x5F\x1C\x80\x24\x91\xE5\xA8\x91\x56\x12\x82\xF7\xE0\x2B\xF4\xDB\xAE\x61\x2E\x89\x10\x8D\x6B\x6C\xBA\xB3\x02\xBD\xD5\x36\xC5\x48\x37\x23\xE2\xF0\x5A\x37\x52\x33\x17\x12\xE2\xD1\x60\x4D\xBE\x2F\x41\x11\xE3\xF6\x17\x25\x0C\x8B\x91\xC0\x1B\x99\x7B\x99\x56\x0D\xAF\xEE\xD2\xBC\x47\x57\xE3\x79\x49\x7B\x34\x89\x27\x24\x84\xDE\xB1\xEC\xE9\x58\x4E\xFE\x4E\xDF\x5A\xBE\x41\xAD\xAC\x08\xC5\x18\x0E\xEF\xD2\x53\xEE\x6C\xD0\x9D\x12\x01\x13\x8D\xDC\x80\x62\xF7\x95\xA9\x44\x88\x4A\x71\x4E\x60\x55\x9E\xDB\x23\x19\x79\x56\x07\x0C\x3F\x63\x0B\x5C\xB0\xE2\xBE\x7E\x15\xFC\x94\x33\x58\x41\x38\x74\xC4\xE1\x8F\x8B\xDF\x26\xAC\x1F\xB5\x8B\x3B\xB7\x43\x59\x6B\xB0\x24\xA6\x6D\x90\x8B\xC4\x72\xEA\x5D\x33\x98\xB7\xCB\xDE\x5E\x7B\xEF\x94\xF1\x1B\x3E\xCA\xC9\x21\xC1\xC5\x98\x02\xAA\xA2\xF6\x5B\x77\x9B\xF5\x7E\x96\x55\x34\x1C\x67\x69\xC0\xF1\x42\xE3\x47\xAC\xFC\x28\x1C\x66\x55\x02\x03\x01\x00\x01\xA3\x81\x8C\x30\x81\x89\x30\x0F\x06\x03\x55\x1D\x13\x04\x08\x30\x06\x01\x01\xFF\x02\x01\x0A\x30\x0B\x06\x03\x55\x1D\x0F\x04\x04\x03\x02\x01\x06\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\xE3\x73\x2D\xDF\xCB\x0E\x28\x0C\xDE\xDD\xB3\xA4\xCA\x79\xB8\x8E\xBB\xE8\x30\x89\x30\x11\x06\x09\x60\x86\x48\x01\x86\xF8\x42\x01\x01\x04\x04\x03\x02\x01\x06\x30\x37\x06\x03\x55\x1D\x1F\x04\x30\x30\x2E\x30\x2C\xA0\x2A\xA0\x28\x86\x26\x68\x74\x74\x70\x3A\x2F\x2F\x77\x77\x77\x2E\x63\x65\x72\x74\x70\x6C\x75\x73\x2E\x63\x6F\x6D\x2F\x43\x52\x4C\x2F\x63\x6C\x61\x73\x73\x32\x2E\x63\x72\x6C\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\xA7\x54\xCF\x88\x44\x19\xCB\xDF\xD4\x7F\x00\xDF\x56\x33\x62\xB5\xF7\x51\x01\x90\xEB\xC3\x3F\xD1\x88\x44\xE9\x24\x5D\xEF\xE7\x14\xBD\x20\xB7\x9A\x3C\x00\xFE\x6D\x9F\xDB\x90\xDC\xD7\xF4\x62\xD6\x8B\x70\x5D\xE7\xE5\x04\x48\xA9\x68\x7C\xC9\xF1\x42\xF3\x6C\x7F\xC5\x7A\x7C\x1D\x51\x88\xBA\xD2\x0A\x3E\x27\x5D\xDE\x2D\x51\x4E\xD3\x13\x64\x69\xE4\x2E\xE3\xD3\xE7\x9B\x09\x99\xA6\xE0\x95\x9B\xCE\x1A\xD7\x7F\xBE\x3C\xCE\x52\xB3\x11\x15\xC1\x0F\x17\xCD\x03\xBB\x9C\x25\x15\xBA\xA2\x76\x89\xFC\x06\xF1\x18\xD0\x93\x4B\x0E\x7C\x82\xB7\xA5\xF4\xF6\x5F\xFE\xED\x40\xA6\x9D\x84\x74\x39\xB9\xDC\x1E\x85\x16\xDA\x29\x1B\x86\x23\x00\xC9\xBB\x89\x7E\x6E\x80\x88\x1E\x2F\x14\xB4\x03\x24\xA8\x32\x6F\x03\x9A\x47\x2C\x30\xBE\x56\xC6\xA7\x42\x02\x70\x1B\xEA\x40\xD8\xBA\x05\x03\x70\x07\xA4\x96\xFF\xFD\x48\x33\x0A\xE1\xDC\xA5\x81\x90\x9B\x4D\xDD\x7D\xE7\xE7\xB2\xCD\x5C\xC8\x6A\x95\xF8\xA5\xF6\x8D\xC4\x5D\x78\x08\xBE\x7B\x06\xD6\x49\xCF\x19\x36\x50\x23\x2E\x08\xE6\x9E\x05\x4D\x47\x18\xD5\x16\xE9\xB1\xD6\xB6\x10\xD5\xBB\x97\xBF\xA2\x8E\xB4\x54", ["CN=DST Root CA X3,O=Digital Signature Trust Co."] = "\x30\x82\x03\x4A\x30\x82\x02\x32\xA0\x03\x02\x01\x02\x02\x10\x44\xAF\xB0\x80\xD6\xA3\x27\xBA\x89\x30\x39\x86\x2E\xF8\x40\x6B\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x3F\x31\x24\x30\x22\x06\x03\x55\x04\x0A\x13\x1B\x44\x69\x67\x69\x74\x61\x6C\x20\x53\x69\x67\x6E\x61\x74\x75\x72\x65\x20\x54\x72\x75\x73\x74\x20\x43\x6F\x2E\x31\x17\x30\x15\x06\x03\x55\x04\x03\x13\x0E\x44\x53\x54\x20\x52\x6F\x6F\x74\x20\x43\x41\x20\x58\x33\x30\x1E\x17\x0D\x30\x30\x30\x39\x33\x30\x32\x31\x31\x32\x31\x39\x5A\x17\x0D\x32\x31\x30\x39\x33\x30\x31\x34\x30\x31\x31\x35\x5A\x30\x3F\x31\x24\x30\x22\x06\x03\x55\x04\x0A\x13\x1B\x44\x69\x67\x69\x74\x61\x6C\x20\x53\x69\x67\x6E\x61\x74\x75\x72\x65\x20\x54\x72\x75\x73\x74\x20\x43\x6F\x2E\x31\x17\x30\x15\x06\x03\x55\x04\x03\x13\x0E\x44\x53\x54\x20\x52\x6F\x6F\x74\x20\x43\x41\x20\x58\x33\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xDF\xAF\xE9\x97\x50\x08\x83\x57\xB4\xCC\x62\x65\xF6\x90\x82\xEC\xC7\xD3\x2C\x6B\x30\xCA\x5B\xEC\xD9\xC3\x7D\xC7\x40\xC1\x18\x14\x8B\xE0\xE8\x33\x76\x49\x2A\xE3\x3F\x21\x49\x93\xAC\x4E\x0E\xAF\x3E\x48\xCB\x65\xEE\xFC\xD3\x21\x0F\x65\xD2\x2A\xD9\x32\x8F\x8C\xE5\xF7\x77\xB0\x12\x7B\xB5\x95\xC0\x89\xA3\xA9\xBA\xED\x73\x2E\x7A\x0C\x06\x32\x83\xA2\x7E\x8A\x14\x30\xCD\x11\xA0\xE1\x2A\x38\xB9\x79\x0A\x31\xFD\x50\xBD\x80\x65\xDF\xB7\x51\x63\x83\xC8\xE2\x88\x61\xEA\x4B\x61\x81\xEC\x52\x6B\xB9\xA2\xE2\x4B\x1A\x28\x9F\x48\xA3\x9E\x0C\xDA\x09\x8E\x3E\x17\x2E\x1E\xDD\x20\xDF\x5B\xC6\x2A\x8A\xAB\x2E\xBD\x70\xAD\xC5\x0B\x1A\x25\x90\x74\x72\xC5\x7B\x6A\xAB\x34\xD6\x30\x89\xFF\xE5\x68\x13\x7B\x54\x0B\xC8\xD6\xAE\xEC\x5A\x9C\x92\x1E\x3D\x64\xB3\x8C\xC6\xDF\xBF\xC9\x41\x70\xEC\x16\x72\xD5\x26\xEC\x38\x55\x39\x43\xD0\xFC\xFD\x18\x5C\x40\xF1\x97\xEB\xD5\x9A\x9B\x8D\x1D\xBA\xDA\x25\xB9\xC6\xD8\xDF\xC1\x15\x02\x3A\xAB\xDA\x6E\xF1\x3E\x2E\xF5\x5C\x08\x9C\x3C\xD6\x83\x69\xE4\x10\x9B\x19\x2A\xB6\x29\x57\xE3\xE5\x3D\x9B\x9F\xF0\x02\x5D\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\xC4\xA7\xB1\xA4\x7B\x2C\x71\xFA\xDB\xE1\x4B\x90\x75\xFF\xC4\x15\x60\x85\x89\x10\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\xA3\x1A\x2C\x9B\x17\x00\x5C\xA9\x1E\xEE\x28\x66\x37\x3A\xBF\x83\xC7\x3F\x4B\xC3\x09\xA0\x95\x20\x5D\xE3\xD9\x59\x44\xD2\x3E\x0D\x3E\xBD\x8A\x4B\xA0\x74\x1F\xCE\x10\x82\x9C\x74\x1A\x1D\x7E\x98\x1A\xDD\xCB\x13\x4B\xB3\x20\x44\xE4\x91\xE9\xCC\xFC\x7D\xA5\xDB\x6A\xE5\xFE\xE6\xFD\xE0\x4E\xDD\xB7\x00\x3A\xB5\x70\x49\xAF\xF2\xE5\xEB\x02\xF1\xD1\x02\x8B\x19\xCB\x94\x3A\x5E\x48\xC4\x18\x1E\x58\x19\x5F\x1E\x02\x5A\xF0\x0C\xF1\xB1\xAD\xA9\xDC\x59\x86\x8B\x6E\xE9\x91\xF5\x86\xCA\xFA\xB9\x66\x33\xAA\x59\x5B\xCE\xE2\xA7\x16\x73\x47\xCB\x2B\xCC\x99\xB0\x37\x48\xCF\xE3\x56\x4B\xF5\xCF\x0F\x0C\x72\x32\x87\xC6\xF0\x44\xBB\x53\x72\x6D\x43\xF5\x26\x48\x9A\x52\x67\xB7\x58\xAB\xFE\x67\x76\x71\x78\xDB\x0D\xA2\x56\x14\x13\x39\x24\x31\x85\xA2\xA8\x02\x5A\x30\x47\xE1\xDD\x50\x07\xBC\x02\x09\x90\x00\xEB\x64\x63\x60\x9B\x16\xBC\x88\xC9\x12\xE6\xD2\x7D\x91\x8B\xF9\x3D\x32\x8D\x65\xB4\xE9\x7C\xB1\x57\x76\xEA\xC5\xB6\x28\x39\xBF\x15\x65\x1C\xC8\xF6\x77\x96\x6A\x0A\x8D\x77\x0B\xD8\x91\x0B\x04\x8E\x07\xDB\x29\xB6\x0A\xEE\x9D\x82\x35\x35\x10", ["CN=SwissSign Gold CA - G2,O=SwissSign AG,C=CH"] = "\x30\x82\x05\xBA\x30\x82\x03\xA2\xA0\x03\x02\x01\x02\x02\x09\x00\xBB\x40\x1C\x43\xF5\x5E\x4F\xB0\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x45\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x43\x48\x31\x15\x30\x13\x06\x03\x55\x04\x0A\x13\x0C\x53\x77\x69\x73\x73\x53\x69\x67\x6E\x20\x41\x47\x31\x1F\x30\x1D\x06\x03\x55\x04\x03\x13\x16\x53\x77\x69\x73\x73\x53\x69\x67\x6E\x20\x47\x6F\x6C\x64\x20\x43\x41\x20\x2D\x20\x47\x32\x30\x1E\x17\x0D\x30\x36\x31\x30\x32\x35\x30\x38\x33\x30\x33\x35\x5A\x17\x0D\x33\x36\x31\x30\x32\x35\x30\x38\x33\x30\x33\x35\x5A\x30\x45\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x43\x48\x31\x15\x30\x13\x06\x03\x55\x04\x0A\x13\x0C\x53\x77\x69\x73\x73\x53\x69\x67\x6E\x20\x41\x47\x31\x1F\x30\x1D\x06\x03\x55\x04\x03\x13\x16\x53\x77\x69\x73\x73\x53\x69\x67\x6E\x20\x47\x6F\x6C\x64\x20\x43\x41\x20\x2D\x20\x47\x32\x30\x82\x02\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x02\x0F\x00\x30\x82\x02\x0A\x02\x82\x02\x01\x00\xAF\xE4\xEE\x7E\x8B\x24\x0E\x12\x6E\xA9\x50\x2D\x16\x44\x3B\x92\x92\x5C\xCA\xB8\x5D\x84\x92\x42\x13\x2A\xBC\x65\x57\x82\x40\x3E\x57\x24\xCD\x50\x8B\x25\x2A\xB7\x6F\xFC\xEF\xA2\xD0\xC0\x1F\x02\x24\x4A\x13\x96\x8F\x23\x13\xE6\x28\x58\x00\xA3\x47\xC7\x06\xA7\x84\x23\x2B\xBB\xBD\x96\x2B\x7F\x55\xCC\x8B\xC1\x57\x1F\x0E\x62\x65\x0F\xDD\x3D\x56\x8A\x73\xDA\xAE\x7E\x6D\xBA\x81\x1C\x7E\x42\x8C\x20\x35\xD9\x43\x4D\x84\xFA\x84\xDB\x52\x2C\xF3\x0E\x27\x77\x0B\x6B\xBF\x11\x2F\x72\x78\x9F\x2E\xD8\x3E\xE6\x18\x37\x5A\x2A\x72\xF9\xDA\x62\x90\x92\x95\xCA\x1F\x9C\xE9\xB3\x3C\x2B\xCB\xF3\x01\x13\xBF\x5A\xCF\xC1\xB5\x0A\x60\xBD\xDD\xB5\x99\x64\x53\xB8\xA0\x96\xB3\x6F\xE2\x26\x77\x91\x8C\xE0\x62\x10\x02\x9F\x34\x0F\xA4\xD5\x92\x33\x51\xDE\xBE\x8D\xBA\x84\x7A\x60\x3C\x6A\xDB\x9F\x2B\xEC\xDE\xDE\x01\x3F\x6E\x4D\xE5\x50\x86\xCB\xB4\xAF\xED\x44\x40\xC5\xCA\x5A\x8C\xDA\xD2\x2B\x7C\xA8\xEE\xBE\xA6\xE5\x0A\xAA\x0E\xA5\xDF\x05\x52\xB7\x55\xC7\x22\x5D\x32\x6A\x97\x97\x63\x13\xDB\xC9\xDB\x79\x36\x7B\x85\x3A\x4A\xC5\x52\x89\xF9\x24\xE7\x9D\x77\xA9\x82\xFF\x55\x1C\xA5\x71\x69\x2B\xD1\x02\x24\xF2\xB3\x26\xD4\x6B\xDA\x04\x55\xE5\xC1\x0A\xC7\x6D\x30\x37\x90\x2A\xE4\x9E\x14\x33\x5E\x16\x17\x55\xC5\x5B\xB5\xCB\x34\x89\x92\xF1\x9D\x26\x8F\xA1\x07\xD4\xC6\xB2\x78\x50\xDB\x0C\x0C\x0B\x7C\x0B\x8C\x41\xD7\xB9\xE9\xDD\x8C\x88\xF7\xA3\x4D\xB2\x32\xCC\xD8\x17\xDA\xCD\xB7\xCE\x66\x9D\xD4\xFD\x5E\xFF\xBD\x97\x3E\x29\x75\xE7\x7E\xA7\x62\x58\xAF\x25\x34\xA5\x41\xC7\x3D\xBC\x0D\x50\xCA\x03\x03\x0F\x08\x5A\x1F\x95\x73\x78\x62\xBF\xAF\x72\x14\x69\x0E\xA5\xE5\x03\x0E\x78\x8E\x26\x28\x42\xF0\x07\x0B\x62\x20\x10\x67\x39\x46\xFA\xA9\x03\xCC\x04\x38\x7A\x66\xEF\x20\x83\xB5\x8C\x4A\x56\x8E\x91\x00\xFC\x8E\x5C\x82\xDE\x88\xA0\xC3\xE2\x68\x6E\x7D\x8D\xEF\x3C\xDD\x65\xF4\x5D\xAC\x51\xEF\x24\x80\xAE\xAA\x56\x97\x6F\xF9\xAD\x7D\xDA\x61\x3F\x98\x77\x3C\xA5\x91\xB6\x1C\x8C\x26\xDA\x65\xA2\x09\x6D\xC1\xE2\x54\xE3\xB9\xCA\x4C\x4C\x80\x8F\x77\x7B\x60\x9A\x1E\xDF\xB6\xF2\x48\x1E\x0E\xBA\x4E\x54\x6D\x98\xE0\xE1\xA2\x1A\xA2\x77\x50\xCF\xC4\x63\x92\xEC\x47\x19\x9D\xEB\xE6\x6B\xCE\xC1\x02\x03\x01\x00\x01\xA3\x81\xAC\x30\x81\xA9\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x5B\x25\x7B\x96\xA4\x65\x51\x7E\xB8\x39\xF3\xC0\x78\x66\x5E\xE8\x3A\xE7\xF0\xEE\x30\x1F\x06\x03\x55\x1D\x23\x04\x18\x30\x16\x80\x14\x5B\x25\x7B\x96\xA4\x65\x51\x7E\xB8\x39\xF3\xC0\x78\x66\x5E\xE8\x3A\xE7\xF0\xEE\x30\x46\x06\x03\x55\x1D\x20\x04\x3F\x30\x3D\x30\x3B\x06\x09\x60\x85\x74\x01\x59\x01\x02\x01\x01\x30\x2E\x30\x2C\x06\x08\x2B\x06\x01\x05\x05\x07\x02\x01\x16\x20\x68\x74\x74\x70\x3A\x2F\x2F\x72\x65\x70\x6F\x73\x69\x74\x6F\x72\x79\x2E\x73\x77\x69\x73\x73\x73\x69\x67\x6E\x2E\x63\x6F\x6D\x2F\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x02\x01\x00\x27\xBA\xE3\x94\x7C\xF1\xAE\xC0\xDE\x17\xE6\xE5\xD8\xD5\xF5\x54\xB0\x83\xF4\xBB\xCD\x5E\x05\x7B\x4F\x9F\x75\x66\xAF\x3C\xE8\x56\x7E\xFC\x72\x78\x38\x03\xD9\x2B\x62\x1B\x00\xB9\xF8\xE9\x60\xCD\xCC\xCE\x51\x8A\xC7\x50\x31\x6E\xE1\x4A\x7E\x18\x2F\x69\x59\xB6\x3D\x64\x81\x2B\xE3\x83\x84\xE6\x22\x87\x8E\x7D\xE0\xEE\x02\x99\x61\xB8\x1E\xF4\xB8\x2B\x88\x12\x16\x84\xC2\x31\x93\x38\x96\x31\xA6\xB9\x3B\x53\x3F\xC3\x24\x93\x56\x5B\x69\x92\xEC\xC5\xC1\xBB\x38\x00\xE3\xEC\x17\xA9\xB8\xDC\xC7\x7C\x01\x83\x9F\x32\x47\xBA\x52\x22\x34\x1D\x32\x7A\x09\x56\xA7\x7C\x25\x36\xA9\x3D\x4B\xDA\xC0\x82\x6F\x0A\xBB\x12\xC8\x87\x4B\x27\x11\xF9\x1E\x2D\xC7\x93\x3F\x9E\xDB\x5F\x26\x6B\x52\xD9\x2E\x8A\xF1\x14\xC6\x44\x8D\x15\xA9\xB7\xBF\xBD\xDE\xA6\x1A\xEE\xAE\x2D\xFB\x48\x77\x17\xFE\xBB\xEC\xAF\x18\xF5\x2A\x51\xF0\x39\x84\x97\x95\x6C\x6E\x1B\xC3\x2B\xC4\x74\x60\x79\x25\xB0\x0A\x27\xDF\xDF\x5E\xD2\x39\xCF\x45\x7D\x42\x4B\xDF\xB3\x2C\x1E\xC5\xC6\x5D\xCA\x55\x3A\xA0\x9C\x69\x9A\x8F\xDA\xEF\xB2\xB0\x3C\x9F\x87\x6C\x12\x2B\x65\x70\x15\x52\x31\x1A\x24\xCF\x6F\x31\x23\x50\x1F\x8C\x4F\x8F\x23\xC3\x74\x41\x63\x1C\x55\xA8\x14\xDD\x3E\xE0\x51\x50\xCF\xF1\x1B\x30\x56\x0E\x92\xB0\x82\x85\xD8\x83\xCB\x22\x64\xBC\x2D\xB8\x25\xD5\x54\xA2\xB8\x06\xEA\xAD\x92\xA4\x24\xA0\xC1\x86\xB5\x4A\x13\x6A\x47\xCF\x2E\x0B\x56\x95\x54\xCB\xCE\x9A\xDB\x6A\xB4\xA6\xB2\xDB\x41\x08\x86\x27\x77\xF7\x6A\xA0\x42\x6C\x0B\x38\xCE\xD7\x75\x50\x32\x92\xC2\xDF\x2B\x30\x22\x48\xD0\xD5\x41\x38\x25\x5D\xA4\xE9\x5D\x9F\xC6\x94\x75\xD0\x45\xFD\x30\x97\x43\x8F\x90\xAB\x0A\xC7\x86\x73\x60\x4A\x69\x2D\xDE\xA5\x78\xD7\x06\xDA\x6A\x9E\x4B\x3E\x77\x3A\x20\x13\x22\x01\xD0\xBF\x68\x9E\x63\x60\x6B\x35\x4D\x0B\x6D\xBA\xA1\x3D\xC0\x93\xE0\x7F\x23\xB3\x55\xAD\x72\x25\x4E\x46\xF9\xD2\x16\xEF\xB0\x64\xC1\x01\x9E\xE9\xCA\xA0\x6A\x98\x0E\xCF\xD8\x60\xF2\x2F\x49\xB8\xE4\x42\xE1\x38\x35\x16\xF4\xC8\x6E\x4F\xF7\x81\x56\xE8\xBA\xA3\xBE\x23\xAF\xAE\xFD\x6F\x03\xE0\x02\x3B\x30\x76\xFA\x1B\x6D\x41\xCF\x01\xB1\xE9\xB8\xC9\x66\xF4\xDB\x26\xF3\x3A\xA4\x74\xF2\x49\x24\x5B\xC9\xB0\xD0\x57\xC1\xFA\x3E\x7A\xE1\x97\xC9", ["CN=SwissSign Silver CA - G2,O=SwissSign AG,C=CH"] = "\x30\x82\x05\xBD\x30\x82\x03\xA5\xA0\x03\x02\x01\x02\x02\x08\x4F\x1B\xD4\x2F\x54\xBB\x2F\x4B\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x47\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x43\x48\x31\x15\x30\x13\x06\x03\x55\x04\x0A\x13\x0C\x53\x77\x69\x73\x73\x53\x69\x67\x6E\x20\x41\x47\x31\x21\x30\x1F\x06\x03\x55\x04\x03\x13\x18\x53\x77\x69\x73\x73\x53\x69\x67\x6E\x20\x53\x69\x6C\x76\x65\x72\x20\x43\x41\x20\x2D\x20\x47\x32\x30\x1E\x17\x0D\x30\x36\x31\x30\x32\x35\x30\x38\x33\x32\x34\x36\x5A\x17\x0D\x33\x36\x31\x30\x32\x35\x30\x38\x33\x32\x34\x36\x5A\x30\x47\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x43\x48\x31\x15\x30\x13\x06\x03\x55\x04\x0A\x13\x0C\x53\x77\x69\x73\x73\x53\x69\x67\x6E\x20\x41\x47\x31\x21\x30\x1F\x06\x03\x55\x04\x03\x13\x18\x53\x77\x69\x73\x73\x53\x69\x67\x6E\x20\x53\x69\x6C\x76\x65\x72\x20\x43\x41\x20\x2D\x20\x47\x32\x30\x82\x02\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x02\x0F\x00\x30\x82\x02\x0A\x02\x82\x02\x01\x00\xC4\xF1\x87\x7F\xD3\x78\x31\xF7\x38\xC9\xF8\xC3\x99\x43\xBC\xC7\xF7\xBC\x37\xE7\x4E\x71\xBA\x4B\x8F\xA5\x73\x1D\x5C\x6E\x98\xAE\x03\x57\xAE\x38\x37\x43\x2F\x17\x3D\x1F\xC8\xCE\x68\x10\xC1\x78\xAE\x19\x03\x2B\x10\xFA\x2C\x79\x83\xF6\xE8\xB9\x68\xB9\x55\xF2\x04\x44\xA7\x39\xF9\xFC\x04\x8B\x1E\xF1\xA2\x4D\x27\xF9\x61\x7B\xBA\xB7\xE5\xA2\x13\xB6\xEB\x61\x3E\xD0\x6C\xD1\xE6\xFB\xFA\x5E\xED\x1D\xB4\x9E\xA0\x35\x5B\xA1\x92\xCB\xF0\x49\x92\xFE\x85\x0A\x05\x3E\xE6\xD9\x0B\xE2\x4F\xBB\xDC\x95\x37\xFC\x91\xE9\x32\x35\x22\xD1\x1F\x3A\x4E\x27\x85\x9D\xB0\x15\x94\x32\xDA\x61\x0D\x47\x4D\x60\x42\xAE\x92\x47\xE8\x83\x5A\x50\x58\xE9\x8A\x8B\xB9\x5D\xA1\xDC\xDD\x99\x4A\x1F\x36\x67\xBB\x48\xE4\x83\xB6\x37\xEB\x48\x3A\xAF\x0F\x67\x8F\x17\x07\xE8\x04\xCA\xEF\x6A\x31\x87\xD4\xC0\xB6\xF9\x94\x71\x7B\x67\x64\xB8\xB6\x91\x4A\x42\x7B\x65\x2E\x30\x6A\x0C\xF5\x90\xEE\x95\xE6\xF2\xCD\x82\xEC\xD9\xA1\x4A\xEC\xF6\xB2\x4B\xE5\x45\x85\xE6\x6D\x78\x93\x04\x2E\x9C\x82\x6D\x36\xA9\xC4\x31\x64\x1F\x86\x83\x0B\x2A\xF4\x35\x0A\x78\xC9\x55\xCF\x41\xB0\x47\xE9\x30\x9F\x99\xBE\x61\xA8\x06\x84\xB9\x28\x7A\x5F\x38\xD9\x1B\xA9\x38\xB0\x83\x7F\x73\xC1\xC3\x3B\x48\x2A\x82\x0F\x21\x9B\xB8\xCC\xA8\x35\xC3\x84\x1B\x83\xB3\x3E\xBE\xA4\x95\x69\x01\x3A\x89\x00\x78\x04\xD9\xC9\xF4\x99\x19\xAB\x56\x7E\x5B\x8B\x86\x39\x15\x91\xA4\x10\x2C\x09\x32\x80\x60\xB3\x93\xC0\x2A\xB6\x18\x0B\x9D\x7E\x8D\x49\xF2\x10\x4A\x7F\xF9\xD5\x46\x2F\x19\x92\xA3\x99\xA7\x26\xAC\xBB\x8C\x3C\xE6\x0E\xBC\x47\x07\xDC\x73\x51\xF1\x70\x64\x2F\x08\xF9\xB4\x47\x1D\x30\x6C\x44\xEA\x29\x37\x85\x92\x68\x66\xBC\x83\x38\xFE\x7B\x39\x2E\xD3\x50\xF0\x1F\xFB\x5E\x60\xB6\xA9\xA6\xFA\x27\x41\xF1\x9B\x18\x72\xF2\xF5\x84\x74\x4A\xC9\x67\xC4\x54\xAE\x48\x64\xDF\x8C\xD1\x6E\xB0\x1D\xE1\x07\x8F\x08\x1E\x99\x9C\x71\xE9\x4C\xD8\xA5\xF7\x47\x12\x1F\x74\xD1\x51\x9E\x86\xF3\xC2\xA2\x23\x40\x0B\x73\xDB\x4B\xA6\xE7\x73\x06\x8C\xC1\xA0\xE9\xC1\x59\xAC\x46\xFA\xE6\x2F\xF8\xCF\x71\x9C\x46\x6D\xB9\xC4\x15\x8D\x38\x79\x03\x45\x48\xEF\xC4\x5D\xD7\x08\xEE\x87\x39\x22\x86\xB2\x0D\x0F\x58\x43\xF7\x71\xA9\x48\x2E\xFD\xEA\xD6\x1F\x02\x03\x01\x00\x01\xA3\x81\xAC\x30\x81\xA9\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x17\xA0\xCD\xC1\xE4\x41\xB6\x3A\x5B\x3B\xCB\x45\x9D\xBD\x1C\xC2\x98\xFA\x86\x58\x30\x1F\x06\x03\x55\x1D\x23\x04\x18\x30\x16\x80\x14\x17\xA0\xCD\xC1\xE4\x41\xB6\x3A\x5B\x3B\xCB\x45\x9D\xBD\x1C\xC2\x98\xFA\x86\x58\x30\x46\x06\x03\x55\x1D\x20\x04\x3F\x30\x3D\x30\x3B\x06\x09\x60\x85\x74\x01\x59\x01\x03\x01\x01\x30\x2E\x30\x2C\x06\x08\x2B\x06\x01\x05\x05\x07\x02\x01\x16\x20\x68\x74\x74\x70\x3A\x2F\x2F\x72\x65\x70\x6F\x73\x69\x74\x6F\x72\x79\x2E\x73\x77\x69\x73\x73\x73\x69\x67\x6E\x2E\x63\x6F\x6D\x2F\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x02\x01\x00\x73\xC6\x81\xE0\x27\xD2\x2D\x0F\xE0\x95\x30\xE2\x9A\x41\x7F\x50\x2C\x5F\x5F\x62\x61\xA9\x86\x6A\x69\x18\x0C\x74\x49\xD6\x5D\x84\xEA\x41\x52\x18\x6F\x58\xAD\x50\x56\x20\x6A\xC6\xBD\x28\x69\x58\x91\xDC\x91\x11\x35\xA9\x3A\x1D\xBC\x1A\xA5\x60\x9E\xD8\x1F\x7F\x45\x91\x69\xD9\x7E\xBB\x78\x72\xC1\x06\x0F\x2A\xCE\x8F\x85\x70\x61\xAC\xA0\xCD\x0B\xB8\x39\x29\x56\x84\x32\x4E\x86\xBB\x3D\xC4\x2A\xD9\xD7\x1F\x72\xEE\xFE\x51\xA1\x22\x41\xB1\x71\x02\x63\x1A\x82\xB0\x62\xAB\x5E\x57\x12\x1F\xDF\xCB\xDD\x75\xA0\xC0\x5D\x79\x90\x8C\x1B\xE0\x50\xE6\xDE\x31\xFE\x98\x7B\x70\x5F\xA5\x90\xD8\xAD\xF8\x02\xB6\x6F\xD3\x60\xDD\x40\x4B\x22\xC5\x3D\xAD\x3A\x7A\x9F\x1A\x1A\x47\x91\x79\x33\xBA\x82\xDC\x32\x69\x03\x96\x6E\x1F\x4B\xF0\x71\xFE\xE3\x67\x72\xA0\xB1\xBF\x5C\x8B\xE4\xFA\x99\x22\xC7\x84\xB9\x1B\x8D\x23\x97\x3F\xED\x25\xE0\xCF\x65\xBB\xF5\x61\x04\xEF\xDD\x1E\xB2\x5A\x41\x22\x5A\xA1\x9F\x5D\x2C\xE8\x5B\xC9\x6D\xA9\x0C\x0C\x78\xAA\x60\xC6\x56\x8F\x01\x5A\x0C\x68\xBC\x69\x19\x79\xC4\x1F\x7E\x97\x05\xBF\xC5\xE9\x24\x51\x5E\xD4\xD5\x4B\x53\xED\xD9\x23\x5A\x36\x03\x65\xA3\xC1\x03\xAD\x41\x30\xF3\x46\x1B\x85\x90\xAF\x65\xB5\xD5\xB1\xE4\x16\x5B\x78\x75\x1D\x97\x7A\x6D\x59\xA9\x2A\x8F\x7B\xDE\xC3\x87\x89\x10\x99\x49\x73\x78\xC8\x3D\xBD\x51\x35\x74\x2A\xD5\xF1\x7E\x69\x1B\x2A\xBB\x3B\xBD\x25\xB8\x9A\x5A\x3D\x72\x61\x90\x66\x87\xEE\x0C\xD6\x4D\xD4\x11\x74\x0B\x6A\xFE\x0B\x03\xFC\xA3\x55\x57\x89\xFE\x4A\xCB\xAE\x5B\x17\x05\xC8\xF2\x8D\x23\x31\x53\x38\xD2\x2D\x6A\x3F\x82\xB9\x8D\x08\x6A\xF7\x5E\x41\x74\x6E\xC3\x11\x7E\x07\xAC\x29\x60\x91\x3F\x38\xCA\x57\x10\x0D\xBD\x30\x2F\xC7\xA5\xE6\x41\xA0\xDA\xAE\x05\x87\x9A\xA0\xA4\x65\x6C\x4C\x09\x0C\x89\xBA\xB8\xD3\xB9\xC0\x93\x8A\x30\xFA\x8D\xE5\x9A\x6B\x15\x01\x4E\x67\xAA\xDA\x62\x56\x3E\x84\x08\x66\xD2\xC4\x36\x7D\xA7\x3E\x10\xFC\x88\xE0\xD4\x80\xE5\x00\xBD\xAA\xF3\x4E\x06\xA3\x7A\x6A\xF9\x62\x72\xE3\x09\x4F\xEB\x9B\x0E\x01\x23\xF1\x9F\xBB\x7C\xDC\xDC\x6C\x11\x97\x25\xB2\xF2\xB4\x63\x14\xD2\x06\x2A\x67\x8C\x83\xF5\xCE\xEA\x07\xD8\x9A\x6A\x1E\xEC\xE4\x0A\xBB\x2A\x4C\xEB\x09\x60\x39\xCE\xCA\x62\xD8\x2E\x6E", @@ -48,7 +47,6 @@ redef root_certs += { ["CN=COMODO ECC Certification Authority,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB"] = "\x30\x82\x02\x89\x30\x82\x02\x0F\xA0\x03\x02\x01\x02\x02\x10\x1F\x47\xAF\xAA\x62\x00\x70\x50\x54\x4C\x01\x9E\x9B\x63\x99\x2A\x30\x0A\x06\x08\x2A\x86\x48\xCE\x3D\x04\x03\x03\x30\x81\x85\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x47\x42\x31\x1B\x30\x19\x06\x03\x55\x04\x08\x13\x12\x47\x72\x65\x61\x74\x65\x72\x20\x4D\x61\x6E\x63\x68\x65\x73\x74\x65\x72\x31\x10\x30\x0E\x06\x03\x55\x04\x07\x13\x07\x53\x61\x6C\x66\x6F\x72\x64\x31\x1A\x30\x18\x06\x03\x55\x04\x0A\x13\x11\x43\x4F\x4D\x4F\x44\x4F\x20\x43\x41\x20\x4C\x69\x6D\x69\x74\x65\x64\x31\x2B\x30\x29\x06\x03\x55\x04\x03\x13\x22\x43\x4F\x4D\x4F\x44\x4F\x20\x45\x43\x43\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x30\x1E\x17\x0D\x30\x38\x30\x33\x30\x36\x30\x30\x30\x30\x30\x30\x5A\x17\x0D\x33\x38\x30\x31\x31\x38\x32\x33\x35\x39\x35\x39\x5A\x30\x81\x85\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x47\x42\x31\x1B\x30\x19\x06\x03\x55\x04\x08\x13\x12\x47\x72\x65\x61\x74\x65\x72\x20\x4D\x61\x6E\x63\x68\x65\x73\x74\x65\x72\x31\x10\x30\x0E\x06\x03\x55\x04\x07\x13\x07\x53\x61\x6C\x66\x6F\x72\x64\x31\x1A\x30\x18\x06\x03\x55\x04\x0A\x13\x11\x43\x4F\x4D\x4F\x44\x4F\x20\x43\x41\x20\x4C\x69\x6D\x69\x74\x65\x64\x31\x2B\x30\x29\x06\x03\x55\x04\x03\x13\x22\x43\x4F\x4D\x4F\x44\x4F\x20\x45\x43\x43\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x30\x76\x30\x10\x06\x07\x2A\x86\x48\xCE\x3D\x02\x01\x06\x05\x2B\x81\x04\x00\x22\x03\x62\x00\x04\x03\x47\x7B\x2F\x75\xC9\x82\x15\x85\xFB\x75\xE4\x91\x16\xD4\xAB\x62\x99\xF5\x3E\x52\x0B\x06\xCE\x41\x00\x7F\x97\xE1\x0A\x24\x3C\x1D\x01\x04\xEE\x3D\xD2\x8D\x09\x97\x0C\xE0\x75\xE4\xFA\xFB\x77\x8A\x2A\xF5\x03\x60\x4B\x36\x8B\x16\x23\x16\xAD\x09\x71\xF4\x4A\xF4\x28\x50\xB4\xFE\x88\x1C\x6E\x3F\x6C\x2F\x2F\x09\x59\x5B\xA5\x5B\x0B\x33\x99\xE2\xC3\x3D\x89\xF9\x6A\x2C\xEF\xB2\xD3\x06\xE9\xA3\x42\x30\x40\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x75\x71\xA7\x19\x48\x19\xBC\x9D\x9D\xEA\x41\x47\xDF\x94\xC4\x48\x77\x99\xD3\x79\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x0A\x06\x08\x2A\x86\x48\xCE\x3D\x04\x03\x03\x03\x68\x00\x30\x65\x02\x31\x00\xEF\x03\x5B\x7A\xAC\xB7\x78\x0A\x72\xB7\x88\xDF\xFF\xB5\x46\x14\x09\x0A\xFA\xA0\xE6\x7D\x08\xC6\x1A\x87\xBD\x18\xA8\x73\xBD\x26\xCA\x60\x0C\x9D\xCE\x99\x9F\xCF\x5C\x0F\x30\xE1\xBE\x14\x31\xEA\x02\x30\x14\xF4\x93\x3C\x49\xA7\x33\x7A\x90\x46\x47\xB3\x63\x7D\x13\x9B\x4E\xB7\x6F\x18\x37\x80\x53\xFE\xDD\x20\xE0\x35\x9A\x36\xD1\xC7\x01\xB9\xE6\xDC\xDD\xF3\xFF\x1D\x2C\x3A\x16\x57\xD9\x92\x39\xD6", ["CN=OISTE WISeKey Global Root GA CA,OU=OISTE Foundation Endorsed,OU=Copyright (c) 2005,O=WISeKey,C=CH"] = "\x30\x82\x03\xF1\x30\x82\x02\xD9\xA0\x03\x02\x01\x02\x02\x10\x41\x3D\x72\xC7\xF4\x6B\x1F\x81\x43\x7D\xF1\xD2\x28\x54\xDF\x9A\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x81\x8A\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x43\x48\x31\x10\x30\x0E\x06\x03\x55\x04\x0A\x13\x07\x57\x49\x53\x65\x4B\x65\x79\x31\x1B\x30\x19\x06\x03\x55\x04\x0B\x13\x12\x43\x6F\x70\x79\x72\x69\x67\x68\x74\x20\x28\x63\x29\x20\x32\x30\x30\x35\x31\x22\x30\x20\x06\x03\x55\x04\x0B\x13\x19\x4F\x49\x53\x54\x45\x20\x46\x6F\x75\x6E\x64\x61\x74\x69\x6F\x6E\x20\x45\x6E\x64\x6F\x72\x73\x65\x64\x31\x28\x30\x26\x06\x03\x55\x04\x03\x13\x1F\x4F\x49\x53\x54\x45\x20\x57\x49\x53\x65\x4B\x65\x79\x20\x47\x6C\x6F\x62\x61\x6C\x20\x52\x6F\x6F\x74\x20\x47\x41\x20\x43\x41\x30\x1E\x17\x0D\x30\x35\x31\x32\x31\x31\x31\x36\x30\x33\x34\x34\x5A\x17\x0D\x33\x37\x31\x32\x31\x31\x31\x36\x30\x39\x35\x31\x5A\x30\x81\x8A\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x43\x48\x31\x10\x30\x0E\x06\x03\x55\x04\x0A\x13\x07\x57\x49\x53\x65\x4B\x65\x79\x31\x1B\x30\x19\x06\x03\x55\x04\x0B\x13\x12\x43\x6F\x70\x79\x72\x69\x67\x68\x74\x20\x28\x63\x29\x20\x32\x30\x30\x35\x31\x22\x30\x20\x06\x03\x55\x04\x0B\x13\x19\x4F\x49\x53\x54\x45\x20\x46\x6F\x75\x6E\x64\x61\x74\x69\x6F\x6E\x20\x45\x6E\x64\x6F\x72\x73\x65\x64\x31\x28\x30\x26\x06\x03\x55\x04\x03\x13\x1F\x4F\x49\x53\x54\x45\x20\x57\x49\x53\x65\x4B\x65\x79\x20\x47\x6C\x6F\x62\x61\x6C\x20\x52\x6F\x6F\x74\x20\x47\x41\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xCB\x4F\xB3\x00\x9B\x3D\x36\xDD\xF9\xD1\x49\x6A\x6B\x10\x49\x1F\xEC\xD8\x2B\xB2\xC6\xF8\x32\x81\x29\x43\x95\x4C\x9A\x19\x23\x21\x15\x45\xDE\xE3\xC8\x1C\x51\x55\x5B\xAE\x93\xE8\x37\xFF\x2B\x6B\xE9\xD4\xEA\xBE\x2A\xDD\xA8\x51\x2B\xD7\x66\xC3\x61\x5C\x60\x02\xC8\xF5\xCE\x72\x7B\x3B\xB8\xF2\x4E\x65\x08\x9A\xCD\xA4\x6A\x19\xC1\x01\xBB\x73\xA6\xD7\xF6\xC3\xDD\xCD\xBC\xA4\x8B\xB5\x99\x61\xB8\x01\xA2\xA3\xD4\x4D\xD4\x05\x3D\x91\xAD\xF8\xB4\x08\x71\x64\xAF\x70\xF1\x1C\x6B\x7E\xF6\xC3\x77\x9D\x24\x73\x7B\xE4\x0C\x8C\xE1\xD9\x36\xE1\x99\x8B\x05\x99\x0B\xED\x45\x31\x09\xCA\xC2\x00\xDB\xF7\x72\xA0\x96\xAA\x95\x87\xD0\x8E\xC7\xB6\x61\x73\x0D\x76\x66\x8C\xDC\x1B\xB4\x63\xA2\x9F\x7F\x93\x13\x30\xF1\xA1\x27\xDB\xD9\xFF\x2C\x55\x88\x91\xA0\xE0\x4F\x07\xB0\x28\x56\x8C\x18\x1B\x97\x44\x8E\x89\xDD\xE0\x17\x6E\xE7\x2A\xEF\x8F\x39\x0A\x31\x84\x82\xD8\x40\x14\x49\x2E\x7A\x41\xE4\xA7\xFE\xE3\x64\xCC\xC1\x59\x71\x4B\x2C\x21\xA7\x5B\x7D\xE0\x1D\xD1\x2E\x81\x9B\xC3\xD8\x68\xF7\xBD\x96\x1B\xAC\x70\xB1\x16\x14\x0B\xDB\x60\xB9\x26\x01\x05\x02\x03\x01\x00\x01\xA3\x51\x30\x4F\x30\x0B\x06\x03\x55\x1D\x0F\x04\x04\x03\x02\x01\x86\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\xB3\x03\x7E\xAE\x36\xBC\xB0\x79\xD1\xDC\x94\x26\xB6\x11\xBE\x21\xB2\x69\x86\x94\x30\x10\x06\x09\x2B\x06\x01\x04\x01\x82\x37\x15\x01\x04\x03\x02\x01\x00\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x4B\xA1\xFF\x0B\x87\x6E\xB3\xF9\xC1\x43\xB1\x48\xF3\x28\xC0\x1D\x2E\xC9\x09\x41\xFA\x94\x00\x1C\xA4\xA4\xAB\x49\x4F\x8F\x3D\x1E\xEF\x4D\x6F\xBD\xBC\xA4\xF6\xF2\x26\x30\xC9\x10\xCA\x1D\x88\xFB\x74\x19\x1F\x85\x45\xBD\xB0\x6C\x51\xF9\x36\x7E\xDB\xF5\x4C\x32\x3A\x41\x4F\x5B\x47\xCF\xE8\x0B\x2D\xB6\xC4\x19\x9D\x74\xC5\x47\xC6\x3B\x6A\x0F\xAC\x14\xDB\x3C\xF4\x73\x9C\xA9\x05\xDF\x00\xDC\x74\x78\xFA\xF8\x35\x60\x59\x02\x13\x18\x7C\xBC\xFB\x4D\xB0\x20\x6D\x43\xBB\x60\x30\x7A\x67\x33\x5C\xC5\x99\xD1\xF8\x2D\x39\x52\x73\xFB\x8C\xAA\x97\x25\x5C\x72\xD9\x08\x1E\xAB\x4E\x3C\xE3\x81\x31\x9F\x03\xA6\xFB\xC0\xFE\x29\x88\x55\xDA\x84\xD5\x50\x03\xB6\xE2\x84\xA3\xA6\x36\xAA\x11\x3A\x01\xE1\x18\x4B\xD6\x44\x68\xB3\x3D\xF9\x53\x74\x84\xB3\x46\x91\x46\x96\x00\xB7\x80\x2C\xB6\xE1\xE3\x10\xE2\xDB\xA2\xE7\x28\x8F\x01\x96\x62\x16\x3E\x00\xE3\x1C\xA5\x36\x81\x18\xA2\x4C\x52\x76\xC0\x11\xA3\x6E\xE6\x1D\xBA\xE3\x5A\xBE\x36\x53\xC5\x3E\x75\x8F\x86\x69\x29\x58\x53\xB5\x9C\xBB\x6F\x9F\x5C\xC5\x18\xEC\xDD\x2F\xE1\x98\xC9\xFC\xBE\xDF\x0A\x0D", ["CN=Certigna,O=Dhimyotis,C=FR"] = "\x30\x82\x03\xA8\x30\x82\x02\x90\xA0\x03\x02\x01\x02\x02\x09\x00\xFE\xDC\xE3\x01\x0F\xC9\x48\xFF\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x34\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x46\x52\x31\x12\x30\x10\x06\x03\x55\x04\x0A\x0C\x09\x44\x68\x69\x6D\x79\x6F\x74\x69\x73\x31\x11\x30\x0F\x06\x03\x55\x04\x03\x0C\x08\x43\x65\x72\x74\x69\x67\x6E\x61\x30\x1E\x17\x0D\x30\x37\x30\x36\x32\x39\x31\x35\x31\x33\x30\x35\x5A\x17\x0D\x32\x37\x30\x36\x32\x39\x31\x35\x31\x33\x30\x35\x5A\x30\x34\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x46\x52\x31\x12\x30\x10\x06\x03\x55\x04\x0A\x0C\x09\x44\x68\x69\x6D\x79\x6F\x74\x69\x73\x31\x11\x30\x0F\x06\x03\x55\x04\x03\x0C\x08\x43\x65\x72\x74\x69\x67\x6E\x61\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xC8\x68\xF1\xC9\xD6\xD6\xB3\x34\x75\x26\x82\x1E\xEC\xB4\xBE\xEA\x5C\xE1\x26\xED\x11\x47\x61\xE1\xA2\x7C\x16\x78\x40\x21\xE4\x60\x9E\x5A\xC8\x63\xE1\xC4\xB1\x96\x92\xFF\x18\x6D\x69\x23\xE1\x2B\x62\xF7\xDD\xE2\x36\x2F\x91\x07\xB9\x48\xCF\x0E\xEC\x79\xB6\x2C\xE7\x34\x4B\x70\x08\x25\xA3\x3C\x87\x1B\x19\xF2\x81\x07\x0F\x38\x90\x19\xD3\x11\xFE\x86\xB4\xF2\xD1\x5E\x1E\x1E\x96\xCD\x80\x6C\xCE\x3B\x31\x93\xB6\xF2\xA0\xD0\xA9\x95\x12\x7D\xA5\x9A\xCC\x6B\xC8\x84\x56\x8A\x33\xA9\xE7\x22\x15\x53\x16\xF0\xCC\x17\xEC\x57\x5F\xE9\xA2\x0A\x98\x09\xDE\xE3\x5F\x9C\x6F\xDC\x48\xE3\x85\x0B\x15\x5A\xA6\xBA\x9F\xAC\x48\xE3\x09\xB2\xF7\xF4\x32\xDE\x5E\x34\xBE\x1C\x78\x5D\x42\x5B\xCE\x0E\x22\x8F\x4D\x90\xD7\x7D\x32\x18\xB3\x0B\x2C\x6A\xBF\x8E\x3F\x14\x11\x89\x20\x0E\x77\x14\xB5\x3D\x94\x08\x87\xF7\x25\x1E\xD5\xB2\x60\x00\xEC\x6F\x2A\x28\x25\x6E\x2A\x3E\x18\x63\x17\x25\x3F\x3E\x44\x20\x16\xF6\x26\xC8\x25\xAE\x05\x4A\xB4\xE7\x63\x2C\xF3\x8C\x16\x53\x7E\x5C\xFB\x11\x1A\x08\xC1\x46\x62\x9F\x22\xB8\xF1\xC2\x8D\x69\xDC\xFA\x3A\x58\x06\xDF\x02\x03\x01\x00\x01\xA3\x81\xBC\x30\x81\xB9\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x1A\xED\xFE\x41\x39\x90\xB4\x24\x59\xBE\x01\xF2\x52\xD5\x45\xF6\x5A\x39\xDC\x11\x30\x64\x06\x03\x55\x1D\x23\x04\x5D\x30\x5B\x80\x14\x1A\xED\xFE\x41\x39\x90\xB4\x24\x59\xBE\x01\xF2\x52\xD5\x45\xF6\x5A\x39\xDC\x11\xA1\x38\xA4\x36\x30\x34\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x46\x52\x31\x12\x30\x10\x06\x03\x55\x04\x0A\x0C\x09\x44\x68\x69\x6D\x79\x6F\x74\x69\x73\x31\x11\x30\x0F\x06\x03\x55\x04\x03\x0C\x08\x43\x65\x72\x74\x69\x67\x6E\x61\x82\x09\x00\xFE\xDC\xE3\x01\x0F\xC9\x48\xFF\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x11\x06\x09\x60\x86\x48\x01\x86\xF8\x42\x01\x01\x04\x04\x03\x02\x00\x07\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x85\x03\x1E\x92\x71\xF6\x42\xAF\xE1\xA3\x61\x9E\xEB\xF3\xC0\x0F\xF2\xA5\xD4\xDA\x95\xE6\xD6\xBE\x68\x36\x3D\x7E\x6E\x1F\x4C\x8A\xEF\xD1\x0F\x21\x6D\x5E\xA5\x52\x63\xCE\x12\xF8\xEF\x2A\xDA\x6F\xEB\x37\xFE\x13\x02\xC7\xCB\x3B\x3E\x22\x6B\xDA\x61\x2E\x7F\xD4\x72\x3D\xDD\x30\xE1\x1E\x4C\x40\x19\x8C\x0F\xD7\x9C\xD1\x83\x30\x7B\x98\x59\xDC\x7D\xC6\xB9\x0C\x29\x4C\xA1\x33\xA2\xEB\x67\x3A\x65\x84\xD3\x96\xE2\xED\x76\x45\x70\x8F\xB5\x2B\xDE\xF9\x23\xD6\x49\x6E\x3C\x14\xB5\xC6\x9F\x35\x1E\x50\xD0\xC1\x8F\x6A\x70\x44\x02\x62\xCB\xAE\x1D\x68\x41\xA7\xAA\x57\xE8\x53\xAA\x07\xD2\x06\xF6\xD5\x14\x06\x0B\x91\x03\x75\x2C\x6C\x72\xB5\x61\x95\x9A\x0D\x8B\xB9\x0D\xE7\xF5\xDF\x54\xCD\xDE\xE6\xD8\xD6\x09\x08\x97\x63\xE5\xC1\x2E\xB0\xB7\x44\x26\xC0\x26\xC0\xAF\x55\x30\x9E\x3B\xD5\x36\x2A\x19\x04\xF4\x5C\x1E\xFF\xCF\x2C\xB7\xFF\xD0\xFD\x87\x40\x11\xD5\x11\x23\xBB\x48\xC0\x21\xA9\xA4\x28\x2D\xFD\x15\xF8\xB0\x4E\x2B\xF4\x30\x5B\x21\xFC\x11\x91\x34\xBE\x41\xEF\x7B\x9D\x97\x75\xFF\x97\x95\xC0\x96\x58\x2F\xEA\xBB\x46\xD7\xBB\xE4\xD9\x2E", - ["CN=Deutsche Telekom Root CA 2,OU=T-TeleSec Trust Center,O=Deutsche Telekom AG,C=DE"] = "\x30\x82\x03\x9F\x30\x82\x02\x87\xA0\x03\x02\x01\x02\x02\x01\x26\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x71\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x44\x45\x31\x1C\x30\x1A\x06\x03\x55\x04\x0A\x13\x13\x44\x65\x75\x74\x73\x63\x68\x65\x20\x54\x65\x6C\x65\x6B\x6F\x6D\x20\x41\x47\x31\x1F\x30\x1D\x06\x03\x55\x04\x0B\x13\x16\x54\x2D\x54\x65\x6C\x65\x53\x65\x63\x20\x54\x72\x75\x73\x74\x20\x43\x65\x6E\x74\x65\x72\x31\x23\x30\x21\x06\x03\x55\x04\x03\x13\x1A\x44\x65\x75\x74\x73\x63\x68\x65\x20\x54\x65\x6C\x65\x6B\x6F\x6D\x20\x52\x6F\x6F\x74\x20\x43\x41\x20\x32\x30\x1E\x17\x0D\x39\x39\x30\x37\x30\x39\x31\x32\x31\x31\x30\x30\x5A\x17\x0D\x31\x39\x30\x37\x30\x39\x32\x33\x35\x39\x30\x30\x5A\x30\x71\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x44\x45\x31\x1C\x30\x1A\x06\x03\x55\x04\x0A\x13\x13\x44\x65\x75\x74\x73\x63\x68\x65\x20\x54\x65\x6C\x65\x6B\x6F\x6D\x20\x41\x47\x31\x1F\x30\x1D\x06\x03\x55\x04\x0B\x13\x16\x54\x2D\x54\x65\x6C\x65\x53\x65\x63\x20\x54\x72\x75\x73\x74\x20\x43\x65\x6E\x74\x65\x72\x31\x23\x30\x21\x06\x03\x55\x04\x03\x13\x1A\x44\x65\x75\x74\x73\x63\x68\x65\x20\x54\x65\x6C\x65\x6B\x6F\x6D\x20\x52\x6F\x6F\x74\x20\x43\x41\x20\x32\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xAB\x0B\xA3\x35\xE0\x8B\x29\x14\xB1\x14\x85\xAF\x3C\x10\xE4\x39\x6F\x35\x5D\x4A\xAE\xDD\xEA\x61\x8D\x95\x49\xF4\x6F\x64\xA3\x1A\x60\x66\xA4\xA9\x40\x22\x84\xD9\xD4\xA5\xE5\x78\x93\x0E\x68\x01\xAD\xB9\x4D\x5C\x3A\xCE\xD3\xB8\xA8\x42\x40\xDF\xCF\xA3\xBA\x82\x59\x6A\x92\x1B\xAC\x1C\x9A\xDA\x08\x2B\x25\x27\xF9\x69\x23\x47\xF1\xE0\xEB\x2C\x7A\x9B\xF5\x13\x02\xD0\x7E\x34\x7C\xC2\x9E\x3C\x00\x59\xAB\xF5\xDA\x0C\xF5\x32\x3C\x2B\xAC\x50\xDA\xD6\xC3\xDE\x83\x94\xCA\xA8\x0C\x99\x32\x0E\x08\x48\x56\x5B\x6A\xFB\xDA\xE1\x58\x58\x01\x49\x5F\x72\x41\x3C\x15\x06\x01\x8E\x5D\xAD\xAA\xB8\x93\xB4\xCD\x9E\xEB\xA7\xE8\x6A\x2D\x52\x34\xDB\x3A\xEF\x5C\x75\x51\xDA\xDB\xF3\x31\xF9\xEE\x71\x98\x32\xC4\x54\x15\x44\x0C\xF9\x9B\x55\xED\xAD\xDF\x18\x08\xA0\xA3\x86\x8A\x49\xEE\x53\x05\x8F\x19\x4C\xD5\xDE\x58\x79\x9B\xD2\x6A\x1C\x42\xAB\xC5\xD5\xA7\xCF\x68\x0F\x96\xE4\xE1\x61\x98\x76\x61\xC8\x91\x7C\xD6\x3E\x00\xE2\x91\x50\x87\xE1\x9D\x0A\xE6\xAD\x97\xD2\x1D\xC6\x3A\x7D\xCB\xBC\xDA\x03\x34\xD5\x8E\x5B\x01\xF5\x6A\x07\xB7\x16\xB6\x6E\x4A\x7F\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x31\xC3\x79\x1B\xBA\xF5\x53\xD7\x17\xE0\x89\x7A\x2D\x17\x6C\x0A\xB3\x2B\x9D\x33\x30\x0F\x06\x03\x55\x1D\x13\x04\x08\x30\x06\x01\x01\xFF\x02\x01\x05\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x94\x64\x59\xAD\x39\x64\xE7\x29\xEB\x13\xFE\x5A\xC3\x8B\x13\x57\xC8\x04\x24\xF0\x74\x77\xC0\x60\xE3\x67\xFB\xE9\x89\xA6\x83\xBF\x96\x82\x7C\x6E\xD4\xC3\x3D\xEF\x9E\x80\x6E\xBB\x29\xB4\x98\x7A\xB1\x3B\x54\xEB\x39\x17\x47\x7E\x1A\x8E\x0B\xFC\x1F\x31\x59\x31\x04\xB2\xCE\x17\xF3\x2C\xC7\x62\x36\x55\xE2\x22\xD8\x89\x55\xB4\x98\x48\xAA\x64\xFA\xD6\x1C\x36\xD8\x44\x78\x5A\x5A\x23\x3A\x57\x97\xF5\x7A\x30\x4F\xAE\x9F\x6A\x4C\x4B\x2B\x8E\xA0\x03\xE3\x3E\xE0\xA9\xD4\xD2\x7B\xD2\xB3\xA8\xE2\x72\x3C\xAD\x9E\xFF\x80\x59\xE4\x9B\x45\xB4\xF6\x3B\xB0\xCD\x39\x19\x98\x32\xE5\xEA\x21\x61\x90\xE4\x31\x21\x8E\x34\xB1\xF7\x2F\x35\x4A\x85\x10\xDA\xE7\x8A\x37\x21\xBE\x59\x63\xE0\xF2\x85\x88\x31\x53\xD4\x54\x14\x85\x70\x79\xF4\x2E\x06\x77\x27\x75\x2F\x1F\xB8\x8A\xF9\xFE\xC5\xBA\xD8\x36\xE4\x83\xEC\xE7\x65\xB7\xBF\x63\x5A\xF3\x46\xAF\x81\x94\x37\xD4\x41\x8C\xD6\x23\xD6\x1E\xCF\xF5\x68\x1B\x44\x63\xA2\x5A\xBA\xA7\x35\x59\xA1\xE5\x70\x05\x9B\x0E\x23\x57\x99\x94\x0A\x6D\xBA\x39\x63\x28\x86\x92\xF3\x18\x84\xD8\xFB\xD1\xCF\x05\x56\x64\x57", ["CN=Cybertrust Global Root,O=Cybertrust\, Inc"] = "\x30\x82\x03\xA1\x30\x82\x02\x89\xA0\x03\x02\x01\x02\x02\x0B\x04\x00\x00\x00\x00\x01\x0F\x85\xAA\x2D\x48\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x3B\x31\x18\x30\x16\x06\x03\x55\x04\x0A\x13\x0F\x43\x79\x62\x65\x72\x74\x72\x75\x73\x74\x2C\x20\x49\x6E\x63\x31\x1F\x30\x1D\x06\x03\x55\x04\x03\x13\x16\x43\x79\x62\x65\x72\x74\x72\x75\x73\x74\x20\x47\x6C\x6F\x62\x61\x6C\x20\x52\x6F\x6F\x74\x30\x1E\x17\x0D\x30\x36\x31\x32\x31\x35\x30\x38\x30\x30\x30\x30\x5A\x17\x0D\x32\x31\x31\x32\x31\x35\x30\x38\x30\x30\x30\x30\x5A\x30\x3B\x31\x18\x30\x16\x06\x03\x55\x04\x0A\x13\x0F\x43\x79\x62\x65\x72\x74\x72\x75\x73\x74\x2C\x20\x49\x6E\x63\x31\x1F\x30\x1D\x06\x03\x55\x04\x03\x13\x16\x43\x79\x62\x65\x72\x74\x72\x75\x73\x74\x20\x47\x6C\x6F\x62\x61\x6C\x20\x52\x6F\x6F\x74\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xF8\xC8\xBC\xBD\x14\x50\x66\x13\xFF\xF0\xD3\x79\xEC\x23\xF2\xB7\x1A\xC7\x8E\x85\xF1\x12\x73\xA6\x19\xAA\x10\xDB\x9C\xA2\x65\x74\x5A\x77\x3E\x51\x7D\x56\xF6\xDC\x23\xB6\xD4\xED\x5F\x58\xB1\x37\x4D\xD5\x49\x0E\x6E\xF5\x6A\x87\xD6\xD2\x8C\xD2\x27\xC6\xE2\xFF\x36\x9F\x98\x65\xA0\x13\x4E\xC6\x2A\x64\x9B\xD5\x90\x12\xCF\x14\x06\xF4\x3B\xE3\xD4\x28\xBE\xE8\x0E\xF8\xAB\x4E\x48\x94\x6D\x8E\x95\x31\x10\x5C\xED\xA2\x2D\xBD\xD5\x3A\x6D\xB2\x1C\xBB\x60\xC0\x46\x4B\x01\xF5\x49\xAE\x7E\x46\x8A\xD0\x74\x8D\xA1\x0C\x02\xCE\xEE\xFC\xE7\x8F\xB8\x6B\x66\xF3\x7F\x44\x00\xBF\x66\x25\x14\x2B\xDD\x10\x30\x1D\x07\x96\x3F\x4D\xF6\x6B\xB8\x8F\xB7\x7B\x0C\xA5\x38\xEB\xDE\x47\xDB\xD5\x5D\x39\xFC\x88\xA7\xF3\xD7\x2A\x74\xF1\xE8\x5A\xA2\x3B\x9F\x50\xBA\xA6\x8C\x45\x35\xC2\x50\x65\x95\xDC\x63\x82\xEF\xDD\xBF\x77\x4D\x9C\x62\xC9\x63\x73\x16\xD0\x29\x0F\x49\xA9\x48\xF0\xB3\xAA\xB7\x6C\xC5\xA7\x30\x39\x40\x5D\xAE\xC4\xE2\x5D\x26\x53\xF0\xCE\x1C\x23\x08\x61\xA8\x94\x19\xBA\x04\x62\x40\xEC\x1F\x38\x70\x77\x12\x06\x71\xA7\x30\x18\x5D\x25\x27\xA5\x02\x03\x01\x00\x01\xA3\x81\xA5\x30\x81\xA2\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\xB6\x08\x7B\x0D\x7A\xCC\xAC\x20\x4C\x86\x56\x32\x5E\xCF\xAB\x6E\x85\x2D\x70\x57\x30\x3F\x06\x03\x55\x1D\x1F\x04\x38\x30\x36\x30\x34\xA0\x32\xA0\x30\x86\x2E\x68\x74\x74\x70\x3A\x2F\x2F\x77\x77\x77\x32\x2E\x70\x75\x62\x6C\x69\x63\x2D\x74\x72\x75\x73\x74\x2E\x63\x6F\x6D\x2F\x63\x72\x6C\x2F\x63\x74\x2F\x63\x74\x72\x6F\x6F\x74\x2E\x63\x72\x6C\x30\x1F\x06\x03\x55\x1D\x23\x04\x18\x30\x16\x80\x14\xB6\x08\x7B\x0D\x7A\xCC\xAC\x20\x4C\x86\x56\x32\x5E\xCF\xAB\x6E\x85\x2D\x70\x57\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x56\xEF\x0A\x23\xA0\x54\x4E\x95\x97\xC9\xF8\x89\xDA\x45\xC1\xD4\xA3\x00\x25\xF4\x1F\x13\xAB\xB7\xA3\x85\x58\x69\xC2\x30\xAD\xD8\x15\x8A\x2D\xE3\xC9\xCD\x81\x5A\xF8\x73\x23\x5A\xA7\x7C\x05\xF3\xFD\x22\x3B\x0E\xD1\x06\xC4\xDB\x36\x4C\x73\x04\x8E\xE5\xB0\x22\xE4\xC5\xF3\x2E\xA5\xD9\x23\xE3\xB8\x4E\x4A\x20\xA7\x6E\x02\x24\x9F\x22\x60\x67\x7B\x8B\x1D\x72\x09\xC5\x31\x5C\xE9\x79\x9F\x80\x47\x3D\xAD\xA1\x0B\x07\x14\x3D\x47\xFF\x03\x69\x1A\x0C\x0B\x44\xE7\x63\x25\xA7\x7F\xB2\xC9\xB8\x76\x84\xED\x23\xF6\x7D\x07\xAB\x45\x7E\xD3\xDF\xB3\xBF\xE9\x8A\xB6\xCD\xA8\xA2\x67\x2B\x52\xD5\xB7\x65\xF0\x39\x4C\x63\xA0\x91\x79\x93\x52\x0F\x54\xDD\x83\xBB\x9F\xD1\x8F\xA7\x53\x73\xC3\xCB\xFF\x30\xEC\x7C\x04\xB8\xD8\x44\x1F\x93\x5F\x71\x09\x22\xB7\x6E\x3E\xEA\x1C\x03\x4E\x9D\x1A\x20\x61\xFB\x81\x37\xEC\x5E\xFC\x0A\x45\xAB\xD7\xE7\x17\x55\xD0\xA0\xEA\x60\x9B\xA6\xF6\xE3\x8C\x5B\x29\xC2\x06\x60\x14\x9D\x2D\x97\x4C\xA9\x93\x15\x9D\x61\xC4\x01\x5F\x48\xD6\x58\xBD\x56\x31\x12\x4E\x11\xC8\x21\xE0\xB3\x11\x91\x65\xDB\xB4\xA6\x88\x38\xCE\x55", ["OU=ePKI Root Certification Authority,O=Chunghwa Telecom Co.\, Ltd.,C=TW"] = "\x30\x82\x05\xB0\x30\x82\x03\x98\xA0\x03\x02\x01\x02\x02\x10\x15\xC8\xBD\x65\x47\x5C\xAF\xB8\x97\x00\x5E\xE4\x06\xD2\xBC\x9D\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x5E\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x54\x57\x31\x23\x30\x21\x06\x03\x55\x04\x0A\x0C\x1A\x43\x68\x75\x6E\x67\x68\x77\x61\x20\x54\x65\x6C\x65\x63\x6F\x6D\x20\x43\x6F\x2E\x2C\x20\x4C\x74\x64\x2E\x31\x2A\x30\x28\x06\x03\x55\x04\x0B\x0C\x21\x65\x50\x4B\x49\x20\x52\x6F\x6F\x74\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x30\x1E\x17\x0D\x30\x34\x31\x32\x32\x30\x30\x32\x33\x31\x32\x37\x5A\x17\x0D\x33\x34\x31\x32\x32\x30\x30\x32\x33\x31\x32\x37\x5A\x30\x5E\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x54\x57\x31\x23\x30\x21\x06\x03\x55\x04\x0A\x0C\x1A\x43\x68\x75\x6E\x67\x68\x77\x61\x20\x54\x65\x6C\x65\x63\x6F\x6D\x20\x43\x6F\x2E\x2C\x20\x4C\x74\x64\x2E\x31\x2A\x30\x28\x06\x03\x55\x04\x0B\x0C\x21\x65\x50\x4B\x49\x20\x52\x6F\x6F\x74\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x30\x82\x02\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x02\x0F\x00\x30\x82\x02\x0A\x02\x82\x02\x01\x00\xE1\x25\x0F\xEE\x8D\xDB\x88\x33\x75\x67\xCD\xAD\x1F\x7D\x3A\x4E\x6D\x9D\xD3\x2F\x14\xF3\x63\x74\xCB\x01\x21\x6A\x37\xEA\x84\x50\x07\x4B\x26\x5B\x09\x43\x6C\x21\x9E\x6A\xC8\xD5\x03\xF5\x60\x69\x8F\xCC\xF0\x22\xE4\x1F\xE7\xF7\x6A\x22\x31\xB7\x2C\x15\xF2\xE0\xFE\x00\x6A\x43\xFF\x87\x65\xC6\xB5\x1A\xC1\xA7\x4C\x6D\x22\x70\x21\x8A\x31\xF2\x97\x74\x89\x09\x12\x26\x1C\x9E\xCA\xD9\x12\xA2\x95\x3C\xDA\xE9\x67\xBF\x08\xA0\x64\xE3\xD6\x42\xB7\x45\xEF\x97\xF4\xF6\xF5\xD7\xB5\x4A\x15\x02\x58\x7D\x98\x58\x4B\x60\xBC\xCD\xD7\x0D\x9A\x13\x33\x53\xD1\x61\xF9\x7A\xD5\xD7\x78\xB3\x9A\x33\xF7\x00\x86\xCE\x1D\x4D\x94\x38\xAF\xA8\xEC\x78\x51\x70\x8A\x5C\x10\x83\x51\x21\xF7\x11\x3D\x34\x86\x5E\xE5\x48\xCD\x97\x81\x82\x35\x4C\x19\xEC\x65\xF6\x6B\xC5\x05\xA1\xEE\x47\x13\xD6\xB3\x21\x27\x94\x10\x0A\xD9\x24\x3B\xBA\xBE\x44\x13\x46\x30\x3F\x97\x3C\xD8\xD7\xD7\x6A\xEE\x3B\x38\xE3\x2B\xD4\x97\x0E\xB9\x1B\xE7\x07\x49\x7F\x37\x2A\xF9\x77\x78\xCF\x54\xED\x5B\x46\x9D\xA3\x80\x0E\x91\x43\xC1\xD6\x5B\x5F\x14\xBA\x9F\xA6\x8D\x24\x47\x40\x59\xBF\x72\x38\xB2\x36\x6C\x37\xFF\x99\xD1\x5D\x0E\x59\x0A\xAB\x69\xF7\xC0\xB2\x04\x45\x7A\x54\x00\xAE\xBE\x53\xF6\xB5\xE7\xE1\xF8\x3C\xA3\x31\xD2\xA9\xFE\x21\x52\x64\xC5\xA6\x67\xF0\x75\x07\x06\x94\x14\x81\x55\xC6\x27\xE4\x01\x8F\x17\xC1\x6A\x71\xD7\xBE\x4B\xFB\x94\x58\x7D\x7E\x11\x33\xB1\x42\xF7\x62\x6C\x18\xD6\xCF\x09\x68\x3E\x7F\x6C\xF6\x1E\x8F\x62\xAD\xA5\x63\xDB\x09\xA7\x1F\x22\x42\x41\x1E\x6F\x99\x8A\x3E\xD7\xF9\x3F\x40\x7A\x79\xB0\xA5\x01\x92\xD2\x9D\x3D\x08\x15\xA5\x10\x01\x2D\xB3\x32\x76\xA8\x95\x0D\xB3\x7A\x9A\xFB\x07\x10\x78\x11\x6F\xE1\x8F\xC7\xBA\x0F\x25\x1A\x74\x2A\xE5\x1C\x98\x41\x99\xDF\x21\x87\xE8\x95\x06\x6A\x0A\xB3\x6A\x47\x76\x65\xF6\x3A\xCF\x8F\x62\x17\x19\x7B\x0A\x28\xCD\x1A\xD2\x83\x1E\x21\xC7\x2C\xBF\xBE\xFF\x61\x68\xB7\x67\x1B\xBB\x78\x4D\x8D\xCE\x67\xE5\xE4\xC1\x8E\xB7\x23\x66\xE2\x9D\x90\x75\x34\x98\xA9\x36\x2B\x8A\x9A\x94\xB9\x9D\xEC\xCC\x8A\xB1\xF8\x25\x89\x5C\x5A\xB6\x2F\x8C\x1F\x6D\x79\x24\xA7\x52\x68\xC3\x84\x35\xE2\x66\x8D\x63\x0E\x25\x4D\xD5\x19\xB2\xE6\x79\x37\xA7\x22\x9D\x54\x31\x02\x03\x01\x00\x01\xA3\x6A\x30\x68\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x1E\x0C\xF7\xB6\x67\xF2\xE1\x92\x26\x09\x45\xC0\x55\x39\x2E\x77\x3F\x42\x4A\xA2\x30\x0C\x06\x03\x55\x1D\x13\x04\x05\x30\x03\x01\x01\xFF\x30\x39\x06\x04\x67\x2A\x07\x00\x04\x31\x30\x2F\x30\x2D\x02\x01\x00\x30\x09\x06\x05\x2B\x0E\x03\x02\x1A\x05\x00\x30\x07\x06\x05\x67\x2A\x03\x00\x00\x04\x14\x45\xB0\xC2\xC7\x0A\x56\x7C\xEE\x5B\x78\x0C\x95\xF9\x18\x53\xC1\xA6\x1C\xD8\x10\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x02\x01\x00\x09\xB3\x83\x53\x59\x01\x3E\x95\x49\xB9\xF1\x81\xBA\xF9\x76\x20\x23\xB5\x27\x60\x74\xD4\x6A\x99\x34\x5E\x6C\x00\x53\xD9\x9F\xF2\xA6\xB1\x24\x07\x44\x6A\x2A\xC6\xA5\x8E\x78\x12\xE8\x47\xD9\x58\x1B\x13\x2A\x5E\x79\x9B\x9F\x0A\x2A\x67\xA6\x25\x3F\x06\x69\x56\x73\xC3\x8A\x66\x48\xFB\x29\x81\x57\x74\x06\xCA\x9C\xEA\x28\xE8\x38\x67\x26\x2B\xF1\xD5\xB5\x3F\x65\x93\xF8\x36\x5D\x8E\x8D\x8D\x40\x20\x87\x19\xEA\xEF\x27\xC0\x3D\xB4\x39\x0F\x25\x7B\x68\x50\x74\x55\x9C\x0C\x59\x7D\x5A\x3D\x41\x94\x25\x52\x08\xE0\x47\x2C\x15\x31\x19\xD5\xBF\x07\x55\xC6\xBB\x12\xB5\x97\xF4\x5F\x83\x85\xBA\x71\xC1\xD9\x6C\x81\x11\x76\x0A\x0A\xB0\xBF\x82\x97\xF7\xEA\x3D\xFA\xFA\xEC\x2D\xA9\x28\x94\x3B\x56\xDD\xD2\x51\x2E\xAE\xC0\xBD\x08\x15\x8C\x77\x52\x34\x96\xD6\x9B\xAC\xD3\x1D\x8E\x61\x0F\x35\x7B\x9B\xAE\x39\x69\x0B\x62\x60\x40\x20\x36\x8F\xAF\xFB\x36\xEE\x2D\x08\x4A\x1D\xB8\xBF\x9B\x5C\xF8\xEA\xA5\x1B\xA0\x73\xA6\xD8\xF8\x6E\xE0\x33\x04\x5F\x68\xAA\x27\x87\xED\xD9\xC1\x90\x9C\xED\xBD\xE3\x6A\x35\xAF\x63\xDF\xAB\x18\xD9\xBA\xE6\xE9\x4A\xEA\x50\x8A\x0F\x61\x93\x1E\xE2\x2D\x19\xE2\x30\x94\x35\x92\x5D\x0E\xB6\x07\xAF\x19\x80\x8F\x47\x90\x51\x4B\x2E\x4D\xDD\x85\xE2\xD2\x0A\x52\x0A\x17\x9A\xFC\x1A\xB0\x50\x02\xE5\x01\xA3\x63\x37\x21\x4C\x44\xC4\x9B\x51\x99\x11\x0E\x73\x9C\x06\x8F\x54\x2E\xA7\x28\x5E\x44\x39\x87\x56\x2D\x37\xBD\x85\x44\x94\xE1\x0C\x4B\x2C\x9C\xC3\x92\x85\x34\x61\xCB\x0F\xB8\x9B\x4A\x43\x52\xFE\x34\x3A\x7D\xB8\xE9\x29\xDC\x76\xA9\xC8\x30\xF8\x14\x71\x80\xC6\x1E\x36\x48\x74\x22\x41\x5C\x87\x82\xE8\x18\x71\x8B\x41\x89\x44\xE7\x7E\x58\x5B\xA8\xB8\x8D\x13\xE9\xA7\x6C\xC3\x47\xED\xB3\x1A\x9D\x62\xAE\x8D\x82\xEA\x94\x9E\xDD\x59\x10\xC3\xAD\xDD\xE2\x4D\xE3\x31\xD5\xC7\xEC\xE8\xF2\xB0\xFE\x92\x1E\x16\x0A\x1A\xFC\xD9\xF3\xF8\x27\xB6\xC9\xBE\x1D\xB4\x6C\x64\x90\x7F\xF4\xE4\xC4\x5B\xD7\x37\xAE\x42\x0E\xDD\xA4\x1A\x6F\x7C\x88\x54\xC5\x16\x6E\xE1\x7A\x68\x2E\xF8\x3A\xBF\x0D\xA4\x3C\x89\x3B\x78\xA7\x4E\x63\x83\x04\x21\x08\x67\x8D\xF2\x82\x49\xD0\x5B\xFD\xB1\xCD\x0F\x83\x84\xD4\x3E\x20\x85\xF7\x4A\x3D\x2B\x9C\xFD\x2A\x0A\x09\x4D\xEA\x81\xF8\x11\x9C", ["OU=certSIGN ROOT CA,O=certSIGN,C=RO"] = "\x30\x82\x03\x38\x30\x82\x02\x20\xA0\x03\x02\x01\x02\x02\x06\x20\x06\x05\x16\x70\x02\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x3B\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x52\x4F\x31\x11\x30\x0F\x06\x03\x55\x04\x0A\x13\x08\x63\x65\x72\x74\x53\x49\x47\x4E\x31\x19\x30\x17\x06\x03\x55\x04\x0B\x13\x10\x63\x65\x72\x74\x53\x49\x47\x4E\x20\x52\x4F\x4F\x54\x20\x43\x41\x30\x1E\x17\x0D\x30\x36\x30\x37\x30\x34\x31\x37\x32\x30\x30\x34\x5A\x17\x0D\x33\x31\x30\x37\x30\x34\x31\x37\x32\x30\x30\x34\x5A\x30\x3B\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x52\x4F\x31\x11\x30\x0F\x06\x03\x55\x04\x0A\x13\x08\x63\x65\x72\x74\x53\x49\x47\x4E\x31\x19\x30\x17\x06\x03\x55\x04\x0B\x13\x10\x63\x65\x72\x74\x53\x49\x47\x4E\x20\x52\x4F\x4F\x54\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xB7\x33\xB9\x7E\xC8\x25\x4A\x8E\xB5\xDB\xB4\x28\x1B\xAA\x57\x90\xE8\xD1\x22\xD3\x64\xBA\xD3\x93\xE8\xD4\xAC\x86\x61\x40\x6A\x60\x57\x68\x54\x84\x4D\xBC\x6A\x54\x02\x05\xFF\xDF\x9B\x9A\x2A\xAE\x5D\x07\x8F\x4A\xC3\x28\x7F\xEF\xFB\x2B\xFA\x79\xF1\xC7\xAD\xF0\x10\x53\x24\x90\x8B\x66\xC9\xA8\x88\xAB\xAF\x5A\xA3\x00\xE9\xBE\xBA\x46\xEE\x5B\x73\x7B\x2C\x17\x82\x81\x5E\x62\x2C\xA1\x02\x65\xB3\xBD\xC5\x2B\x00\x7E\xC4\xFC\x03\x33\x57\x0D\xED\xE2\xFA\xCE\x5D\x45\xD6\x38\xCD\x35\xB6\xB2\xC1\xD0\x9C\x81\x4A\xAA\xE4\xB2\x01\x5C\x1D\x8F\x5F\x99\xC4\xB1\xAD\xDB\x88\x21\xEB\x90\x08\x82\x80\xF3\x30\xA3\x43\xE6\x90\x82\xAE\x55\x28\x49\xED\x5B\xD7\xA9\x10\x38\x0E\xFE\x8F\x4C\x5B\x9B\x46\xEA\x41\xF5\xB0\x08\x74\xC3\xD0\x88\x33\xB6\x7C\xD7\x74\xDF\xDC\x84\xD1\x43\x0E\x75\x39\xA1\x25\x40\x28\xEA\x78\xCB\x0E\x2C\x2E\x39\x9D\x8C\x8B\x6E\x16\x1C\x2F\x26\x82\x10\xE2\xE3\x65\x94\x0A\x04\xC0\x5E\xF7\x5D\x5B\xF8\x10\xE2\xD0\xBA\x7A\x4B\xFB\xDE\x37\x00\x00\x1A\x5B\x28\xE3\xD2\x9C\x73\x3E\x32\x87\x98\xA1\xC9\x51\x2F\xD7\xDE\xAC\x33\xB3\x4F\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\xC6\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\xE0\x8C\x9B\xDB\x25\x49\xB3\xF1\x7C\x86\xD6\xB2\x42\x87\x0B\xD0\x6B\xA0\xD9\xE4\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x3E\xD2\x1C\x89\x2E\x35\xFC\xF8\x75\xDD\xE6\x7F\x65\x88\xF4\x72\x4C\xC9\x2C\xD7\x32\x4E\xF3\xDD\x19\x79\x47\xBD\x8E\x3B\x5B\x93\x0F\x50\x49\x24\x13\x6B\x14\x06\x72\xEF\x09\xD3\xA1\xA1\xE3\x40\x84\xC9\xE7\x18\x32\x74\x3C\x48\x6E\x0F\x9F\x4B\xD4\xF7\x1E\xD3\x93\x86\x64\x54\x97\x63\x72\x50\xD5\x55\xCF\xFA\x20\x93\x02\xA2\x9B\xC3\x23\x93\x4E\x16\x55\x76\xA0\x70\x79\x6D\xCD\x21\x1F\xCF\x2F\x2D\xBC\x19\xE3\x88\x31\xF8\x59\x1A\x81\x09\xC8\x97\xA6\x74\xC7\x60\xC4\x5B\xCC\x57\x8E\xB2\x75\xFD\x1B\x02\x09\xDB\x59\x6F\x72\x93\x69\xF7\x31\x41\xD6\x88\x38\xBF\x87\xB2\xBD\x16\x79\xF9\xAA\xE4\xBE\x88\x25\xDD\x61\x27\x23\x1C\xB5\x31\x07\x04\x36\xB4\x1A\x90\xBD\xA0\x74\x71\x50\x89\x6D\xBC\x14\xE3\x0F\x86\xAE\xF1\xAB\x3E\xC7\xA0\x09\xCC\xA3\x48\xD1\xE0\xDB\x64\xE7\x92\xB5\xCF\xAF\x72\x43\x70\x8B\xF9\xC3\x84\x3C\x13\xAA\x7E\x92\x9B\x57\x53\x93\xFA\x70\xC2\x91\x0E\x31\xF9\x9B\x67\x5D\xE9\x96\x38\x5E\x5F\xB3\x73\x4E\x88\x15\x67\xDE\x9E\x76\x10\x62\x20\xBE\x55\x69\x95\x43\x00\x39\x4D\xF6\xEE\xB0\x5A\x4E\x49\x44\x54\x58\x5F\x42\x83", @@ -150,4 +148,5 @@ redef root_certs += { ["CN=emSign Root CA - C1,O=eMudhra Inc,OU=emSign PKI,C=US"] = "\x30\x82\x03\x73\x30\x82\x02\x5B\xA0\x03\x02\x01\x02\x02\x0B\x00\xAE\xCF\x00\xBA\xC4\xCF\x32\xF8\x43\xB2\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x0B\x05\x00\x30\x56\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x13\x30\x11\x06\x03\x55\x04\x0B\x13\x0A\x65\x6D\x53\x69\x67\x6E\x20\x50\x4B\x49\x31\x14\x30\x12\x06\x03\x55\x04\x0A\x13\x0B\x65\x4D\x75\x64\x68\x72\x61\x20\x49\x6E\x63\x31\x1C\x30\x1A\x06\x03\x55\x04\x03\x13\x13\x65\x6D\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x20\x2D\x20\x43\x31\x30\x1E\x17\x0D\x31\x38\x30\x32\x31\x38\x31\x38\x33\x30\x30\x30\x5A\x17\x0D\x34\x33\x30\x32\x31\x38\x31\x38\x33\x30\x30\x30\x5A\x30\x56\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x13\x30\x11\x06\x03\x55\x04\x0B\x13\x0A\x65\x6D\x53\x69\x67\x6E\x20\x50\x4B\x49\x31\x14\x30\x12\x06\x03\x55\x04\x0A\x13\x0B\x65\x4D\x75\x64\x68\x72\x61\x20\x49\x6E\x63\x31\x1C\x30\x1A\x06\x03\x55\x04\x03\x13\x13\x65\x6D\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x20\x2D\x20\x43\x31\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xCF\xEB\xA9\xB9\xF1\x99\x05\xCC\xD8\x28\x21\x4A\xF3\x73\x34\x51\x84\x56\x10\xF5\xA0\x4F\x2C\x12\xE3\xFA\x13\x9A\x27\xD0\xCF\xF9\x79\x1A\x74\x5F\x1D\x79\x39\xFC\x5B\xF8\x70\x8E\xE0\x92\x52\xF7\xE4\x25\xF9\x54\x83\xD9\x1D\xD3\xC8\x5A\x85\x3F\x5E\xC7\xB6\x07\xEE\x3E\xC0\xCE\x9A\xAF\xAC\x56\x42\x2A\x39\x25\x70\xD6\xBF\xB5\x7B\x36\xAD\xAC\xF6\x73\xDC\xCD\xD7\x1D\x8A\x83\xA5\xFB\x2B\x90\x15\x37\x6B\x1C\x26\x47\xDC\x3B\x29\x56\x93\x6A\xB3\xC1\x6A\x3A\x9D\x3D\xF5\xC1\x97\x38\x58\x05\x8B\x1C\x11\xE3\xE4\xB4\xB8\x5D\x85\x1D\x83\xFE\x78\x5F\x0B\x45\x68\x18\x48\xA5\x46\x73\x34\x3B\xFE\x0F\xC8\x76\xBB\xC7\x18\xF3\x05\xD1\x86\xF3\x85\xED\xE7\xB9\xD9\x32\xAD\x55\x88\xCE\xA6\xB6\x91\xB0\x4F\xAC\x7E\x15\x23\x96\xF6\x3F\xF0\x20\x34\x16\xDE\x0A\xC6\xC4\x04\x45\x79\x7F\xA7\xFD\xBE\xD2\xA9\xA5\xAF\x9C\xC5\x23\x2A\xF7\x3C\x21\x6C\xBD\xAF\x8F\x4E\xC5\x3A\xB2\xF3\x34\x12\xFC\xDF\x80\x1A\x49\xA4\xD4\xA9\x95\xF7\x9E\x89\x5E\xA2\x89\xAC\x94\xCB\xA8\x68\x9B\xAF\x8A\x65\x27\xCD\x89\xEE\xDD\x8C\xB5\x6B\x29\x70\x43\xA0\x69\x0B\xE4\xB9\x0F\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\xFE\xA1\xE0\x70\x1E\x2A\x03\x39\x52\x5A\x42\xBE\x5C\x91\x85\x7A\x18\xAA\x4D\xB5\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x0B\x05\x00\x03\x82\x01\x01\x00\xC2\x4A\x56\xFA\x15\x21\x7B\x28\xA2\xE9\xE5\x1D\xFB\xF8\x2D\xC4\x39\x96\x41\x4C\x3B\x27\x2C\xC4\x6C\x18\x15\x80\xC6\xAC\xAF\x47\x59\x2F\x26\x0B\xE3\x36\xB0\xEF\x3B\xFE\x43\x97\x49\x32\x99\x12\x15\x5B\xDF\x11\x29\xFF\xAB\x53\xF8\xBB\xC1\x78\x0F\xAC\x9C\x53\xAF\x57\xBD\x68\x8C\x3D\x69\x33\xF0\xA3\xA0\x23\x63\x3B\x64\x67\x22\x44\xAD\xD5\x71\xCB\x56\x2A\x78\x92\xA3\x4F\x12\x31\x36\x36\xE2\xDE\xFE\x00\xC4\xA3\x60\x0F\x27\xAD\xA0\xB0\x8A\xB5\x36\x7A\x52\xA1\xBD\x27\xF4\x20\x27\x62\xE8\x4D\x94\x24\x13\xE4\x0A\x04\xE9\x3C\xAB\x2E\xC8\x43\x09\x4A\xC6\x61\x04\xE5\x49\x34\x7E\xD3\xC4\xC8\xF5\x0F\xC0\xAA\xE9\xBA\x54\x5E\xF3\x63\x2B\x4F\x4F\x50\xD4\xFE\xB9\x7B\x99\x8C\x3D\xC0\x2E\xBC\x02\x2B\xD3\xC4\x40\xE4\x8A\x07\x31\x1E\x9B\xCE\x26\x99\x13\xFB\x11\xEA\x9A\x22\x0C\x11\x19\xC7\x5E\x1B\x81\x50\x30\xC8\x96\x12\x6E\xE7\xCB\x41\x7F\x91\x3B\xA2\x47\xB7\x54\x80\x1B\xDC\x00\xCC\x9A\x90\xEA\xC3\xC3\x50\x06\x62\x0C\x30\xC0\x15\x48\xA7\xA8\x59\x7C\xE1\xAE\x22\xA2\xE2\x0A\x7A\x0F\xFA\x62\xAB\x52\x4C\xE1\xF1\xDF\xCA\xBE\x83\x0D\x42", ["CN=emSign ECC Root CA - C3,O=eMudhra Inc,OU=emSign PKI,C=US"] = "\x30\x82\x02\x2B\x30\x82\x01\xB1\xA0\x03\x02\x01\x02\x02\x0A\x7B\x71\xB6\x82\x56\xB8\x12\x7C\x9C\xA8\x30\x0A\x06\x08\x2A\x86\x48\xCE\x3D\x04\x03\x03\x30\x5A\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x13\x30\x11\x06\x03\x55\x04\x0B\x13\x0A\x65\x6D\x53\x69\x67\x6E\x20\x50\x4B\x49\x31\x14\x30\x12\x06\x03\x55\x04\x0A\x13\x0B\x65\x4D\x75\x64\x68\x72\x61\x20\x49\x6E\x63\x31\x20\x30\x1E\x06\x03\x55\x04\x03\x13\x17\x65\x6D\x53\x69\x67\x6E\x20\x45\x43\x43\x20\x52\x6F\x6F\x74\x20\x43\x41\x20\x2D\x20\x43\x33\x30\x1E\x17\x0D\x31\x38\x30\x32\x31\x38\x31\x38\x33\x30\x30\x30\x5A\x17\x0D\x34\x33\x30\x32\x31\x38\x31\x38\x33\x30\x30\x30\x5A\x30\x5A\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x13\x30\x11\x06\x03\x55\x04\x0B\x13\x0A\x65\x6D\x53\x69\x67\x6E\x20\x50\x4B\x49\x31\x14\x30\x12\x06\x03\x55\x04\x0A\x13\x0B\x65\x4D\x75\x64\x68\x72\x61\x20\x49\x6E\x63\x31\x20\x30\x1E\x06\x03\x55\x04\x03\x13\x17\x65\x6D\x53\x69\x67\x6E\x20\x45\x43\x43\x20\x52\x6F\x6F\x74\x20\x43\x41\x20\x2D\x20\x43\x33\x30\x76\x30\x10\x06\x07\x2A\x86\x48\xCE\x3D\x02\x01\x06\x05\x2B\x81\x04\x00\x22\x03\x62\x00\x04\xFD\xA5\x61\xAE\x7B\x26\x10\x1D\xE9\xB7\x22\x30\xAE\x06\xF4\x81\xB3\xB1\x42\x71\x95\x39\xBC\xD3\x52\xE3\xAF\xAF\xF9\xF2\x97\x35\x92\x36\x46\x0E\x87\x95\x8D\xB9\x39\x5A\xE9\xBB\xDF\xD0\xFE\xC8\x07\x41\x3C\xBB\x55\x6F\x83\xA3\x6A\xFB\x62\xB0\x81\x89\x02\x70\x7D\x48\xC5\x4A\xE3\xE9\x22\x54\x22\x4D\x93\xBB\x42\x0C\xAF\x77\x9C\x23\xA6\x7D\xD7\x61\x11\xCE\x65\xC7\xF8\x7F\xFE\xF5\xF2\xA9\xA3\x42\x30\x40\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\xFB\x5A\x48\xD0\x80\x20\x40\xF2\xA8\xE9\x00\x07\x69\x19\x77\xA7\xE6\xC3\xF4\xCF\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x0A\x06\x08\x2A\x86\x48\xCE\x3D\x04\x03\x03\x03\x68\x00\x30\x65\x02\x31\x00\xB4\xD8\x2F\x02\x89\xFD\xB6\x4C\x62\xBA\x43\x4E\x13\x84\x72\xB5\xAE\xDD\x1C\xDE\xD6\xB5\xDC\x56\x8F\x58\x40\x5A\x2D\xDE\x20\x4C\x22\x83\xCA\x93\xA8\x7E\xEE\x12\x40\xC7\xD6\x87\x4F\xF8\xDF\x85\x02\x30\x1C\x14\x64\xE4\x7C\x96\x83\x11\x9C\xB0\xD1\x5A\x61\x4B\xA6\x0F\x49\xD3\x00\xFC\xA1\xFC\xE4\xA5\xFF\x7F\xAD\xD7\x30\xD0\xC7\x77\x7F\xBE\x81\x07\x55\x30\x50\x20\x14\xF5\x57\x38\x0A\xA8\x31\x51", ["CN=Hongkong Post Root CA 3,O=Hongkong Post,L=Hong Kong,ST=Hong Kong,C=HK"] = "\x30\x82\x05\xCF\x30\x82\x03\xB7\xA0\x03\x02\x01\x02\x02\x14\x08\x16\x5F\x8A\x4C\xA5\xEC\x00\xC9\x93\x40\xDF\xC4\xC6\xAE\x23\xB8\x1C\x5A\xA4\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x0B\x05\x00\x30\x6F\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x48\x4B\x31\x12\x30\x10\x06\x03\x55\x04\x08\x13\x09\x48\x6F\x6E\x67\x20\x4B\x6F\x6E\x67\x31\x12\x30\x10\x06\x03\x55\x04\x07\x13\x09\x48\x6F\x6E\x67\x20\x4B\x6F\x6E\x67\x31\x16\x30\x14\x06\x03\x55\x04\x0A\x13\x0D\x48\x6F\x6E\x67\x6B\x6F\x6E\x67\x20\x50\x6F\x73\x74\x31\x20\x30\x1E\x06\x03\x55\x04\x03\x13\x17\x48\x6F\x6E\x67\x6B\x6F\x6E\x67\x20\x50\x6F\x73\x74\x20\x52\x6F\x6F\x74\x20\x43\x41\x20\x33\x30\x1E\x17\x0D\x31\x37\x30\x36\x30\x33\x30\x32\x32\x39\x34\x36\x5A\x17\x0D\x34\x32\x30\x36\x30\x33\x30\x32\x32\x39\x34\x36\x5A\x30\x6F\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x48\x4B\x31\x12\x30\x10\x06\x03\x55\x04\x08\x13\x09\x48\x6F\x6E\x67\x20\x4B\x6F\x6E\x67\x31\x12\x30\x10\x06\x03\x55\x04\x07\x13\x09\x48\x6F\x6E\x67\x20\x4B\x6F\x6E\x67\x31\x16\x30\x14\x06\x03\x55\x04\x0A\x13\x0D\x48\x6F\x6E\x67\x6B\x6F\x6E\x67\x20\x50\x6F\x73\x74\x31\x20\x30\x1E\x06\x03\x55\x04\x03\x13\x17\x48\x6F\x6E\x67\x6B\x6F\x6E\x67\x20\x50\x6F\x73\x74\x20\x52\x6F\x6F\x74\x20\x43\x41\x20\x33\x30\x82\x02\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x02\x0F\x00\x30\x82\x02\x0A\x02\x82\x02\x01\x00\xB3\x88\xD7\xEA\xCE\x0F\x20\x4E\xBE\xE6\xD6\x03\x6D\xEE\x59\xFC\xC2\x57\xDF\x29\x68\xA1\x83\x0E\x3E\x68\xC7\x68\x58\x9C\x1C\x60\x4B\x89\x43\x0C\xB9\xD4\x15\xB2\xEE\xC1\x4E\x75\xE9\xB5\xA7\xEF\xE5\xE9\x35\x99\xE4\xCC\x1C\xE7\x4B\x5F\x8D\x33\x30\x20\x33\x53\xD9\xA6\xBB\xD5\x3E\x13\x8E\xE9\x1F\x87\x49\xAD\x50\x2D\x50\xCA\x18\xBE\x01\x58\xA2\x13\x70\x96\xBB\x89\x88\x56\x80\x5C\xF8\xBD\x2C\x3C\xE1\x4C\x57\x88\xBB\xD3\xB9\x95\xEF\xCB\xC7\xF6\xDA\x31\x74\x28\xA6\xE6\x54\x89\xF5\x41\x31\xCA\xE5\x26\x1A\xCD\x82\xE0\x70\xDA\x3B\x29\xBB\xD5\x03\xF5\x99\xBA\x55\xF5\x64\xD1\x60\x0E\xB3\x89\x49\xB8\x8A\x2F\x05\xD2\x84\x45\x28\x7C\x8F\x68\x50\x12\x78\xFC\x0B\xB5\x53\xCB\xC2\x98\x1C\x84\xA3\x9E\xB0\xBE\x23\xA4\xDA\xDC\xC8\x2B\x1E\xDA\x6E\x45\x1E\x89\x98\xDA\xF9\x00\x2E\x06\xE9\x0C\x3B\x70\xD5\x50\x25\x88\x99\xCB\xCD\x73\x60\xF7\xD5\xFF\x35\x67\xC5\xA1\xBC\x5E\xAB\xCD\x4A\xB8\x45\xEB\xC8\x68\x1E\x0D\x0D\x14\x46\x12\xE3\xD2\x64\x62\x8A\x42\x98\xBC\xB4\xC6\x08\x08\xF8\xFD\xA8\x4C\x64\x9C\x76\x01\xBD\x2F\xA9\x6C\x33\x0F\xD8\x3F\x28\xB8\x3C\x69\x01\x42\x86\x7E\x69\xC1\xC9\x06\xCA\xE5\x7A\x46\x65\xE9\xC2\xD6\x50\x41\x2E\x3F\xB7\xE4\xED\x6C\xD7\xBF\x26\x01\x11\xA2\x16\x29\x4A\x6B\x34\x06\x90\xEC\x13\xD2\xB6\xFB\x6A\x76\xD2\x3C\xED\xF0\xD6\x2D\xDD\xE1\x15\xEC\xA3\x9B\x2F\x2C\xC9\x3E\x2B\xE4\x69\x3B\xFF\x72\x25\xB1\x36\x86\x5B\xC7\x7F\x6B\x8B\x55\x1B\x4A\xC5\x20\x61\x3D\xAE\xCB\x50\xE1\x08\x3A\xBE\xB0\x8F\x63\x41\x53\x30\x08\x59\x3C\x98\x1D\x77\xBA\x63\x91\x7A\xCA\x10\x50\x60\xBF\xF0\xD7\xBC\x95\x87\x8F\x97\xC5\xFE\x97\x6A\x01\x94\xA3\x7C\x5B\x85\x1D\x2A\x39\x3A\xD0\x54\xA1\xD1\x39\x71\x9D\xFD\x21\xF9\xB5\x7B\xF0\xE2\xE0\x02\x8F\x6E\x96\x24\x25\x2C\xA0\x1E\x2C\xA8\xC4\x89\xA7\xEF\xED\x99\x06\x2F\xB6\x0A\x4C\x4F\xDB\xA2\xCC\x37\x1A\xAF\x47\x85\x2D\x8A\x5F\xC4\x34\x34\x4C\x00\xFD\x18\x93\x67\x13\xD1\x37\xE6\x48\xB4\x8B\x06\xC5\x57\x7B\x19\x86\x0A\x79\xCB\x00\xC9\x52\xAF\x42\xFF\x37\x8F\xE1\xA3\x1E\x7A\x3D\x50\xAB\x63\x06\xE7\x15\xB5\x3F\xB6\x45\x37\x94\x37\xB1\x7E\xF2\x48\xC3\x7F\xC5\x75\xFE\x97\x8D\x45\x8F\x1A\xA7\x1A\x72\x28\x1A\x40\x0F\x02\x03\x01\x00\x01\xA3\x63\x30\x61\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x1F\x06\x03\x55\x1D\x23\x04\x18\x30\x16\x80\x14\x17\x9D\xCD\x1E\x8B\xD6\x39\x2B\x70\xD3\x5C\xD4\xA0\xB8\x1F\xB0\x00\xFC\xC5\x61\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x17\x9D\xCD\x1E\x8B\xD6\x39\x2B\x70\xD3\x5C\xD4\xA0\xB8\x1F\xB0\x00\xFC\xC5\x61\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x0B\x05\x00\x03\x82\x02\x01\x00\x56\xD5\x7B\x6E\xE6\x22\x01\xD2\x42\x9B\x18\xD5\x0E\xD7\x66\x23\x5C\xE3\xFE\xA0\xC7\x92\xD2\xE9\x94\xAD\x4B\xA2\xC6\xEC\x12\x7C\x74\xD5\x48\xD2\x59\x14\x99\xC0\xEB\xB9\xD1\xEB\xF4\x48\x30\x5B\xAD\xA7\x57\x73\x99\xA9\xD3\xE5\xB7\xD1\x2E\x59\x24\x58\xDC\x68\x2E\x2E\x62\xD8\x6A\xE4\x70\x0B\x2D\x20\x50\x20\xA4\x32\x95\xD1\x00\x98\xBB\xD3\xFD\xF7\x32\xF2\x49\xAE\xC6\x7A\xE0\x47\xBE\x6E\xCE\xCB\xA3\x72\x3A\x2D\x69\x5D\xCB\xC8\xE8\x45\x39\xD4\xFA\x42\xC1\x11\x4C\x77\x5D\x92\xFB\x6A\xFF\x58\x44\xE5\xEB\x81\x9E\xAF\xA0\x99\xAD\xBE\xA9\x01\x66\xCB\x38\x1D\x3C\xDF\x43\x1F\xF4\x4D\x6E\xB4\xBA\x17\x46\xFC\x7D\xFD\x87\x81\x79\x6A\x0D\x33\x0F\xFA\x2F\xF8\x14\xB9\x80\xB3\x5D\x4D\xAA\x97\xE1\xF9\xE4\x18\xC5\xF8\xD5\x38\x8C\x26\x3C\xFD\xF2\x28\xE2\xEE\x5A\x49\x88\x2C\xDF\x79\x3D\x8E\x9E\x90\x3C\xBD\x41\x4A\x3A\xDD\x5B\xF6\x9A\xB4\xCE\x3F\x25\x30\x7F\x32\x7D\xA2\x03\x94\xD0\xDC\x7A\xA1\x52\xDE\x6E\x93\x8D\x18\x26\xFD\x55\xAC\xBD\x8F\x9B\xD2\xCF\xAF\xE7\x86\x2C\xCB\x1F\x09\x6F\xA3\x6F\xA9\x84\xD4\x73\xBF\x4D\xA1\x74\x1B\x4E\x23\x60\xF2\xCC\x0E\xAA\x7F\xA4\x9C\x4C\x25\xA8\xB2\x66\x3B\x38\xFF\xD9\x94\x30\xF6\x72\x84\xBE\x68\x55\x10\x0F\xC6\x73\x2C\x16\x69\x93\x07\xFE\xB1\x45\xED\xBB\xA2\x55\x6A\xB0\xDA\xB5\x4A\x02\x25\x27\x85\xD7\xB7\xB7\x86\x44\x16\x89\x6C\x80\x2B\x3E\x97\xA9\x9C\xD5\x7E\x55\x4C\xC6\xDE\x45\x10\x1C\xEA\xE9\x3B\x9F\x03\x53\xEE\xEE\x7A\x01\x02\x16\x78\xD4\xE8\xC2\xBE\x46\x76\x88\x13\x3F\x22\xBB\x48\x12\x1D\x52\x00\xB4\x02\x7E\x21\x1A\x1E\x9C\x25\xF4\xF3\x3D\x5E\x1E\xD2\x1C\xF9\xB3\x2D\xB6\xF7\x37\x5C\xC6\xCB\x21\x4E\xB0\xF7\x99\x47\x18\x85\xC1\x2B\xBA\x55\xAE\x06\xEA\xD0\x07\xB2\xDC\xAB\xD0\x82\x96\x75\xCE\xD2\x50\xFE\x99\xE7\xCF\x2F\x9F\xE7\x76\xD1\x61\x2A\xFB\x21\xBB\x31\xD0\xAA\x9F\x47\xA4\xB2\x22\xCA\x16\x3A\x50\x57\xC4\x5B\x43\x67\xC5\x65\x62\x03\x49\x01\xEB\x43\xD9\xD8\xF8\x9E\xAD\xCF\xB1\x63\x0E\x45\xF4\xA0\x5A\x2C\x9B\x2D\xC5\xA6\xC0\xAD\xA8\x47\xF4\x27\x4C\x38\x0D\x2E\x1B\x49\x3B\x52\xF4\xE8\x88\x83\x2B\x54\x28\xD4\xF2\x35\x52\xB4\x32\x83\x62\x69\x64\x0C\x91\x9C\x9F\x97\xEA\x74\x16\xFD\x1F\x11\x06\x9A\x9B\xF4", + ["CN=Entrust Root Certification Authority - G4,OU=(c) 2015 Entrust\, Inc. - for authorized use only,OU=See www.entrust.net/legal-terms,O=Entrust\, Inc.,C=US"] = "\x30\x82\x06\x4B\x30\x82\x04\x33\xA0\x03\x02\x01\x02\x02\x11\x00\xD9\xB5\x43\x7F\xAF\xA9\x39\x0F\x00\x00\x00\x00\x55\x65\xAD\x58\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x0B\x05\x00\x30\x81\xBE\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x16\x30\x14\x06\x03\x55\x04\x0A\x13\x0D\x45\x6E\x74\x72\x75\x73\x74\x2C\x20\x49\x6E\x63\x2E\x31\x28\x30\x26\x06\x03\x55\x04\x0B\x13\x1F\x53\x65\x65\x20\x77\x77\x77\x2E\x65\x6E\x74\x72\x75\x73\x74\x2E\x6E\x65\x74\x2F\x6C\x65\x67\x61\x6C\x2D\x74\x65\x72\x6D\x73\x31\x39\x30\x37\x06\x03\x55\x04\x0B\x13\x30\x28\x63\x29\x20\x32\x30\x31\x35\x20\x45\x6E\x74\x72\x75\x73\x74\x2C\x20\x49\x6E\x63\x2E\x20\x2D\x20\x66\x6F\x72\x20\x61\x75\x74\x68\x6F\x72\x69\x7A\x65\x64\x20\x75\x73\x65\x20\x6F\x6E\x6C\x79\x31\x32\x30\x30\x06\x03\x55\x04\x03\x13\x29\x45\x6E\x74\x72\x75\x73\x74\x20\x52\x6F\x6F\x74\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x20\x2D\x20\x47\x34\x30\x1E\x17\x0D\x31\x35\x30\x35\x32\x37\x31\x31\x31\x31\x31\x36\x5A\x17\x0D\x33\x37\x31\x32\x32\x37\x31\x31\x34\x31\x31\x36\x5A\x30\x81\xBE\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x16\x30\x14\x06\x03\x55\x04\x0A\x13\x0D\x45\x6E\x74\x72\x75\x73\x74\x2C\x20\x49\x6E\x63\x2E\x31\x28\x30\x26\x06\x03\x55\x04\x0B\x13\x1F\x53\x65\x65\x20\x77\x77\x77\x2E\x65\x6E\x74\x72\x75\x73\x74\x2E\x6E\x65\x74\x2F\x6C\x65\x67\x61\x6C\x2D\x74\x65\x72\x6D\x73\x31\x39\x30\x37\x06\x03\x55\x04\x0B\x13\x30\x28\x63\x29\x20\x32\x30\x31\x35\x20\x45\x6E\x74\x72\x75\x73\x74\x2C\x20\x49\x6E\x63\x2E\x20\x2D\x20\x66\x6F\x72\x20\x61\x75\x74\x68\x6F\x72\x69\x7A\x65\x64\x20\x75\x73\x65\x20\x6F\x6E\x6C\x79\x31\x32\x30\x30\x06\x03\x55\x04\x03\x13\x29\x45\x6E\x74\x72\x75\x73\x74\x20\x52\x6F\x6F\x74\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x20\x2D\x20\x47\x34\x30\x82\x02\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x02\x0F\x00\x30\x82\x02\x0A\x02\x82\x02\x01\x00\xB1\xEC\x2C\x42\xEE\xE2\xD1\x30\xFF\xA5\x92\x47\xE2\x2D\xC3\xBA\x64\x97\x6D\xCA\xF7\x0D\xB5\x59\xC1\xB3\xCB\xA8\x68\x19\xD8\xAF\x84\x6D\x30\x70\x5D\x7E\xF3\x2E\xD2\x53\x99\xE1\xFE\x1F\x5E\xD9\x48\xAF\x5D\x13\x8D\xDB\xFF\x63\x33\x4D\xD3\x00\x02\xBC\xC4\xF8\xD1\x06\x08\x94\x79\x58\x8A\x15\xDE\x29\xB3\xFD\xFD\xC4\x4F\xE8\xAA\xE2\xA0\x3B\x79\xCD\xBF\x6B\x43\x32\xDD\xD9\x74\x10\xB9\xF7\xF4\x68\xD4\xBB\xD0\x87\xD5\xAA\x4B\x8A\x2A\x6F\x2A\x04\xB5\xB2\xA6\xC7\xA0\x7A\xE6\x48\xAB\xD2\xD1\x59\xCC\xD6\x7E\x23\xE6\x97\x6C\xF0\x42\xE5\xDC\x51\x4B\x15\x41\xED\x49\x4A\xC9\xDE\x10\x97\xD6\x76\xC1\xEF\xA5\xB5\x36\x14\x97\x35\xD8\x78\x22\x35\x52\xEF\x43\xBD\xDB\x27\xDB\x61\x56\x82\x34\xDC\xCB\x88\x60\x0C\x0B\x5A\xE5\x2C\x01\xC6\x54\xAF\xD7\xAA\xC1\x10\x7B\xD2\x05\x5A\xB8\x40\x9E\x86\xA7\xC3\x90\x86\x02\x56\x52\x09\x7A\x9C\xD2\x27\x82\x53\x4A\x65\x52\x6A\xF5\x3C\xE7\xA8\xF2\x9C\xAF\x8B\xBD\xD3\x0E\xD4\xD4\x5E\x6E\x87\x9E\x6A\x3D\x45\x1D\xD1\x5D\x1B\xF4\xE9\x0A\xAC\x60\x99\xFB\x89\xB4\xFF\x98\x2C\xCF\x7C\x1D\xE9\x02\xAA\x04\x9A\x1E\xB8\xDC\x88\x6E\x25\xB3\x6C\x66\xF7\x3C\x90\xF3\x57\xC1\xB3\x2F\xF5\x6D\xF2\xFB\xCA\xA1\xF8\x29\x9D\x46\x8B\xB3\x6A\xF6\xE6\x67\x07\xBE\x2C\x67\x0A\x2A\x1F\x5A\xB2\x3E\x57\xC4\xD3\x21\x21\x63\x65\x52\x91\x1B\xB1\x99\x8E\x79\x7E\xE6\xEB\x8D\x00\xD9\x5A\xAA\xEA\x73\xE8\xA4\x82\x02\x47\x96\xFE\x5B\x8E\x54\x61\xA3\xEB\x2F\x4B\x30\xB0\x8B\x23\x75\x72\x7C\x21\x3C\xC8\xF6\xF1\x74\xD4\x1C\x7B\xA3\x05\x55\xEE\xBB\x4D\x3B\x32\xBE\x9A\x77\x66\x9E\xAC\x69\x90\x22\x07\x1F\x61\x3A\x96\xBE\xE5\x9A\x4F\xCC\x05\x3C\x28\x59\xD3\xC1\x0C\x54\xA8\x59\x61\xBD\xC8\x72\x4C\xE8\xDC\x9F\x87\x7F\xBD\x9C\x48\x36\x5E\x95\xA3\x0E\xB9\x38\x24\x55\xFC\x75\x66\xEB\x02\xE3\x08\x34\x29\x4A\xC6\xE3\x2B\x2F\x33\xA0\xDA\xA3\x86\xA5\x12\x97\xFD\x80\x2B\xDA\x14\x42\xE3\x92\xBD\x3E\xF2\x5D\x5E\x67\x74\x2E\x1C\x88\x47\x29\x34\x5F\xE2\x32\xA8\x9C\x25\x37\x8C\xBA\x98\x00\x97\x8B\x49\x96\x1E\xFD\x25\x8A\xAC\xDC\xDA\xD8\x5D\x74\x6E\x66\xB0\xFF\x44\xDF\xA1\x18\xC6\xBE\x48\x2F\x37\x94\x78\xF8\x95\x4A\x3F\x7F\x13\x5E\x5D\x59\xFD\x74\x86\x43\x63\x73\x49\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x9F\x38\xC4\x56\x23\xC3\x39\xE8\xA0\x71\x6C\xE8\x54\x4C\xE4\xE8\x3A\xB1\xBF\x67\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x0B\x05\x00\x03\x82\x02\x01\x00\x12\xE5\x42\xA6\x7B\x8B\x0F\x0C\xE4\x46\xA5\xB6\x60\x40\x87\x8C\x25\x7E\xAD\xB8\x68\x2E\x5B\xC6\x40\x76\x3C\x03\xF8\xC9\x59\xF4\xF3\xAB\x62\xCE\x10\x8D\xB4\x5A\x64\x8C\x68\xC0\xB0\x72\x43\x34\xD2\x1B\x0B\xF6\x2C\x53\xD2\xCA\x90\x4B\x86\x66\xFC\xAA\x83\x22\xF4\x8B\x1A\x6F\x26\x48\xAC\x76\x77\x08\xBF\xC5\x98\x5C\xF4\x26\x89\x9E\x7B\xC3\xB9\x64\x32\x01\x7F\xD3\xC3\xDD\x58\x6D\xEC\xB1\xAB\x84\x55\x74\x77\x84\x04\x27\x52\x6B\x86\x4C\xCE\xDD\xB9\x65\xFF\xD6\xC6\x5E\x9F\x9A\x10\x99\x4B\x75\x6A\xFE\x6A\xE9\x97\x20\xE4\xE4\x76\x7A\xC6\xD0\x24\xAA\x90\xCD\x20\x90\xBA\x47\x64\xFB\x7F\x07\xB3\x53\x78\xB5\x0A\x62\xF2\x73\x43\xCE\x41\x2B\x81\x6A\x2E\x85\x16\x94\x53\xD4\x6B\x5F\x72\x22\xAB\x51\x2D\x42\xD5\x00\x9C\x99\xBF\xDE\xBB\x94\x3B\x57\xFD\x9A\xF5\x86\xCB\x56\x3B\x5B\x88\x01\xE5\x7C\x28\x4B\x03\xF9\x49\x83\x7C\xB2\x7F\x7C\xE3\xED\x8E\xA1\x7F\x60\x53\x8E\x55\x9D\x50\x34\x12\x0F\xB7\x97\x7B\x6C\x87\x4A\x44\xE7\xF5\x6D\xEC\x80\x37\xF0\x58\x19\x6E\x4A\x68\x76\xF0\x1F\x92\xE4\xEA\xB5\x92\xD3\x61\x51\x10\x0B\xAD\xA7\xD9\x5F\xC7\x5F\xDC\x1F\xA3\x5C\x8C\xA1\x7E\x9B\xB7\x9E\xD3\x56\x6F\x66\x5E\x07\x96\x20\xED\x0B\x74\xFB\x66\x4E\x8B\x11\x15\xE9\x81\x49\x7E\x6F\xB0\xD4\x50\x7F\x22\xD7\x5F\x65\x02\x0D\xA6\xF4\x85\x1E\xD8\xAE\x06\x4B\x4A\xA7\xD2\x31\x66\xC2\xF8\xCE\xE5\x08\xA6\xA4\x02\x96\x44\x68\x57\xC4\xD5\x33\xCF\x19\x2F\x14\xC4\x94\x1C\x7B\xA4\xD9\xF0\x9F\x0E\xB1\x80\xE2\xD1\x9E\x11\x64\xA9\x88\x11\x3A\x76\x82\xE5\x62\xC2\x80\xD8\xA4\x83\xED\x93\xEF\x7C\x2F\x90\xB0\x32\x4C\x96\x15\x68\x48\x52\xD4\x99\x08\xC0\x24\xE8\x1C\xE3\xB3\xA5\x21\x0E\x92\xC0\x90\x1F\xCF\x20\x5F\xCA\x3B\x38\xC7\xB7\x6D\x3A\xF3\xE6\x44\xB8\x0E\x31\x6B\x88\x8E\x70\xEB\x9C\x17\x52\xA8\x41\x94\x2E\x87\xB6\xE7\xA6\x12\xC5\x75\xDF\x5B\xC0\x0A\x6E\x7B\xA4\xE4\x5E\x86\xF9\x36\x94\xDF\x77\xC3\xE9\x0D\xC0\x39\xF1\x79\xBB\x46\x8E\xAB\x43\x59\x27\xB7\x20\xBB\x23\xE9\x56\x40\x21\xEC\x31\x3D\x65\xAA\x43\xF2\x3D\xDF\x70\x44\xE1\xBA\x4D\x26\x10\x3B\x98\x9F\xF3\xC8\x8E\x1B\x38\x56\x21\x6A\x51\x93\xD3\x91\xCA\x46\xDA\x89\xB7\x3D\x53\x83\x2C\x08\x1F\x8B\x8F\x53\xDD\xFF\xAC\x1F", }; diff --git a/src/3rdparty b/src/3rdparty index afc87b64b1..0c732695b4 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit afc87b64b12b722511f7dc0c60c671f1393b009e +Subproject commit 0c732695b446255047ee91456cc9aebea831fe2b From 2129a808f6e2c9d9afada094fc215b560c6842e0 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Mon, 3 Feb 2020 13:25:39 -0800 Subject: [PATCH 45/53] Updating CHANGES and VERSION after messing up last commit message. --- CHANGES | 6 ++++++ VERSION | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 1e0565a9b4..6696cccb5a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,10 @@ +3.1.0-dev.531 | 2020-02-03 13:23:22 -0800 + + * Update sqlite to 3.31.1 (Johanna Amann, Corelight) + + * Update certificate list to state of NSS 3.49.2. (Johanna Amann, Corelight) + 3.1.0-dev.530 | 2020-02-03 13:21:16 -0800 * Disable Travis leak test (Jon Siwek, Corelight) diff --git a/VERSION b/VERSION index 0e558a147e..93dd47822c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.0-dev.530 +3.1.0-dev.531 From 6d2a7892437c71b5d4f5a6c67d6c7f41d59ab1cb Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 3 Feb 2020 18:01:17 -0800 Subject: [PATCH 46/53] Updating submodule(s). [nomail] --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index a7f0e850a4..87b63d810d 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit a7f0e850a42c3624b1460967ab9436e4d4f34644 +Subproject commit 87b63d810d345172084d15c9c7feb132688456fe From 532c66df51911833648c7cff44548e372af79293 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 4 Feb 2020 11:57:35 +0000 Subject: [PATCH 47/53] Updating submodule. [nomail] --- CHANGES | 4 ++++ VERSION | 2 +- aux/broker | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 558ce1d014..76b16857b1 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +3.1.0-dev.538 | 2020-02-04 11:57:35 +0000 + + * Updating submodules. + 3.1.0-dev.536 | 2020-02-03 15:44:10 -0700 * Change various functions to by-value std::string_view args (Jon Siwek, Corelight) diff --git a/VERSION b/VERSION index 53cbdde01b..da2b7eefdf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.0-dev.536 +3.1.0-dev.538 diff --git a/aux/broker b/aux/broker index b66c2167b5..ae9b6ffd85 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit b66c2167b5ec87473a5ba55daa955a026dec5473 +Subproject commit ae9b6ffd85b1097db2a7a6acbfd708165e3a2f10 From 0db61f309481905e459c5bbdc4f32c21f22baa98 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Sun, 2 Feb 2020 10:01:14 +0100 Subject: [PATCH 48/53] include cleanup The Zeek code base has very inconsistent #includes. Many sources included a few headers, and those headers included other headers, and in the end, nearly everything is included everywhere, so missing #includes were never noticed. Another side effect was a lot of header bloat which slows down the build. First step to fix it: in each source file, its own header should be included first to verify that each header's includes are correct, and none is missing. After adding the missing #includes, I replaced lots of #includes inside headers with class forward declarations. In most headers, object pointers are never referenced, so declaring the function prototypes with forward-declared classes is just fine. This patch speeds up the build by 19%, because each compilation unit gets smaller. Here are the "time" numbers for a fresh build (with a warm page cache but without ccache): Before this patch: 3144.94user 161.63system 3:02.87elapsed 1808%CPU (0avgtext+0avgdata 2168608maxresident)k 760inputs+12008400outputs (1511major+57747204minor)pagefaults 0swaps After this patch: 2565.17user 141.83system 2:25.46elapsed 1860%CPU (0avgtext+0avgdata 1489076maxresident)k 72576inputs+9130920outputs (1667major+49400430minor)pagefaults 0swaps --- src/Anon.cc | 3 +- src/Anon.h | 4 +-- src/Attr.cc | 2 ++ src/Attr.h | 1 + src/Base64.cc | 2 ++ src/Base64.h | 8 ++--- src/BroString.cc | 6 ++-- src/BroString.h | 8 ++--- src/Brofiler.cc | 7 ++++- src/Brofiler.h | 8 ++++- src/CCL.h | 3 +- src/CompHash.h | 2 +- src/Conn.cc | 5 ++- src/Conn.h | 9 +++--- src/DFA.cc | 3 +- src/DFA.h | 8 ++++- src/DNS_Mgr.cc | 4 ++- src/DNS_Mgr.h | 2 -- src/DbgBreakpoint.cc | 9 ++++-- src/DbgBreakpoint.h | 7 ++++- src/DbgDisplay.h | 2 +- src/DbgWatch.h | 3 +- src/Debug.cc | 11 ++++++- src/Debug.h | 14 ++++----- src/DebugCmds.cc | 9 ++++-- src/Desc.cc | 4 +-- src/Desc.h | 6 ++-- src/Discard.cc | 13 +++++--- src/Discard.h | 4 +-- src/EquivClass.cc | 1 + src/EquivClass.h | 2 +- src/Event.cc | 1 + src/Event.h | 4 +-- src/EventHandler.cc | 3 +- src/EventHandler.h | 5 ++- src/EventRegistry.cc | 1 + src/EventRegistry.h | 11 ++++--- src/Expr.cc | 3 ++ src/Expr.h | 8 ++--- src/File.cc | 4 ++- src/File.h | 8 +++-- src/Frag.cc | 5 +-- src/Frag.h | 12 ++++--- src/Frame.cc | 6 +++- src/Frame.h | 9 ++++-- src/Func.cc | 8 ++++- src/Func.h | 14 ++++++--- src/Hash.cc | 1 + src/Hash.h | 6 ++-- src/ID.cc | 5 +++ src/ID.h | 7 +++++ src/IP.cc | 3 +- src/IP.h | 7 +++-- src/IPAddr.h | 11 +++---- src/NFA.cc | 6 ++-- src/NFA.h | 6 ++-- src/Net.cc | 10 +++--- src/Net.h | 8 ++--- src/NetVar.cc | 4 ++- src/Notifier.cc | 4 ++- src/Notifier.h | 5 --- src/Obj.cc | 3 +- src/Obj.h | 3 +- src/OpaqueVal.cc | 3 ++ src/OpaqueVal.h | 8 +++-- src/Options.cc | 10 +++--- src/PacketDumper.cc | 10 ++---- src/PacketDumper.h | 2 ++ src/PacketFilter.cc | 1 + src/PacketFilter.h | 5 ++- src/PrefixTable.cc | 1 + src/PrefixTable.h | 10 ++++-- src/PriorityQueue.h | 2 +- src/Queue.h | 2 ++ src/RE.cc | 3 +- src/RE.h | 6 ++-- src/RandTest.cc | 3 +- src/RandTest.h | 2 +- src/Reassem.cc | 7 +++-- src/Reassem.h | 5 ++- src/Reporter.cc | 9 ++++-- src/Reporter.h | 3 +- src/Rule.cc | 2 ++ src/Rule.h | 14 ++++----- src/RuleAction.h | 8 +++-- src/RuleCondition.cc | 5 +++ src/RuleCondition.h | 7 ++--- src/RuleMatcher.cc | 14 +++++++-- src/RuleMatcher.h | 31 +++++++++++-------- src/Scope.cc | 4 ++- src/Scope.h | 2 -- src/SerializationFormat.cc | 5 +-- src/SerializationFormat.h | 2 +- src/Sessions.cc | 3 +- src/Sessions.h | 19 ++++++------ src/SmithWaterman.cc | 1 + src/Stats.cc | 6 +++- src/Stats.h | 6 ++++ src/Stmt.cc | 1 + src/Stmt.h | 7 ++++- src/Tag.h | 7 +++-- src/Timer.h | 3 +- src/Traverse.cc | 1 + src/Traverse.h | 12 +++---- src/Trigger.cc | 13 +++++++- src/Trigger.h | 17 ++++++++-- src/TunnelEncapsulation.h | 4 ++- src/Type.cc | 6 ++++ src/Type.h | 9 +++--- src/UID.cc | 4 +-- src/UID.h | 6 ++-- src/Val.cc | 4 ++- src/Val.h | 22 ++++++++----- src/Var.cc | 10 ++++-- src/Var.h | 12 ++++--- src/WeirdState.cc | 1 + src/analyzer/Analyzer.h | 12 +++++-- src/analyzer/Tag.h | 10 ++++-- src/analyzer/protocol/arp/ARP.cc | 1 + src/analyzer/protocol/ayiya/Plugin.cc | 5 ++- .../protocol/ayiya/ayiya-analyzer.pac | 3 ++ src/analyzer/protocol/ayiya/ayiya.pac | 6 ++++ .../protocol/bittorrent/BitTorrentTracker.h | 2 ++ src/analyzer/protocol/bittorrent/Plugin.cc | 5 ++- src/analyzer/protocol/conn-size/ConnSize.cc | 2 ++ src/analyzer/protocol/conn-size/Plugin.cc | 5 ++- src/analyzer/protocol/conn-size/functions.bif | 2 ++ src/analyzer/protocol/dhcp/Plugin.cc | 5 ++- src/analyzer/protocol/dnp3/DNP3.cc | 1 + src/analyzer/protocol/dnp3/Plugin.cc | 5 ++- src/analyzer/protocol/dns/DNS.cc | 3 +- src/analyzer/protocol/dns/Plugin.cc | 5 ++- src/analyzer/protocol/file/Plugin.cc | 5 ++- src/analyzer/protocol/finger/Plugin.cc | 4 +-- src/analyzer/protocol/ftp/FTP.cc | 3 +- src/analyzer/protocol/ftp/FTP.h | 3 +- src/analyzer/protocol/ftp/Plugin.cc | 5 ++- src/analyzer/protocol/ftp/functions.bif | 1 + src/analyzer/protocol/gnutella/Plugin.cc | 5 ++- src/analyzer/protocol/gtpv1/Plugin.cc | 5 ++- .../protocol/gtpv1/gtpv1-analyzer.pac | 3 ++ src/analyzer/protocol/gtpv1/gtpv1.pac | 3 ++ src/analyzer/protocol/http/Plugin.cc | 5 ++- src/analyzer/protocol/icmp/ICMP.cc | 6 +++- src/analyzer/protocol/icmp/ICMP.h | 3 ++ src/analyzer/protocol/icmp/Plugin.cc | 5 ++- src/analyzer/protocol/ident/Plugin.cc | 5 ++- src/analyzer/protocol/imap/Plugin.cc | 4 ++- src/analyzer/protocol/imap/imap.pac | 1 + src/analyzer/protocol/irc/Plugin.cc | 5 ++- src/analyzer/protocol/krb/KRB.cc | 3 +- src/analyzer/protocol/krb/Plugin.cc | 3 +- src/analyzer/protocol/krb/krb-padata.pac | 1 + src/analyzer/protocol/login/Login.cc | 3 +- src/analyzer/protocol/login/NVT.cc | 3 +- src/analyzer/protocol/login/Plugin.cc | 5 ++- src/analyzer/protocol/login/RSH.cc | 3 +- src/analyzer/protocol/login/Rlogin.cc | 3 +- src/analyzer/protocol/login/functions.bif | 2 ++ src/analyzer/protocol/mime/MIME.cc | 4 +-- src/analyzer/protocol/mime/MIME.h | 5 ++- src/analyzer/protocol/modbus/Plugin.cc | 5 ++- src/analyzer/protocol/mqtt/Plugin.cc | 4 +-- src/analyzer/protocol/mysql/Plugin.cc | 4 +-- src/analyzer/protocol/ncp/Plugin.cc | 5 ++- src/analyzer/protocol/netbios/NetbiosSSN.cc | 3 +- src/analyzer/protocol/netbios/Plugin.cc | 5 ++- src/analyzer/protocol/netbios/functions.bif | 3 ++ src/analyzer/protocol/ntp/Plugin.cc | 4 +-- src/analyzer/protocol/pia/PIA.cc | 4 +++ src/analyzer/protocol/pia/PIA.h | 1 + src/analyzer/protocol/pia/Plugin.cc | 5 ++- src/analyzer/protocol/pop3/POP3.cc | 8 ++--- src/analyzer/protocol/pop3/Plugin.cc | 5 ++- src/analyzer/protocol/radius/Plugin.cc | 5 ++- src/analyzer/protocol/rdp/Plugin.cc | 4 +-- src/analyzer/protocol/rdp/rdp-analyzer.pac | 1 + src/analyzer/protocol/rfb/Plugin.cc | 4 +-- src/analyzer/protocol/rpc/MOUNT.cc | 6 ++-- src/analyzer/protocol/rpc/MOUNT.h | 2 -- src/analyzer/protocol/rpc/NFS.cc | 8 ++--- src/analyzer/protocol/rpc/NFS.h | 3 +- src/analyzer/protocol/rpc/Plugin.cc | 5 ++- src/analyzer/protocol/rpc/Portmap.cc | 6 ++-- src/analyzer/protocol/rpc/RPC.cc | 11 ++++--- src/analyzer/protocol/rpc/RPC.h | 2 +- src/analyzer/protocol/rpc/XDR.cc | 7 ++--- src/analyzer/protocol/sip/Plugin.cc | 5 ++- src/analyzer/protocol/smtp/Plugin.cc | 5 ++- src/analyzer/protocol/snmp/Plugin.cc | 4 +-- src/analyzer/protocol/snmp/SNMP.cc | 1 + src/analyzer/protocol/snmp/snmp.pac | 1 + src/analyzer/protocol/socks/Plugin.cc | 5 ++- src/analyzer/protocol/socks/socks.pac | 1 + src/analyzer/protocol/ssh/Plugin.cc | 3 +- src/analyzer/protocol/ssl/Plugin.cc | 5 ++- src/analyzer/protocol/ssl/functions.bif | 1 + src/analyzer/protocol/ssl/ssl.pac | 1 + src/analyzer/protocol/ssl/tls-handshake.pac | 1 + .../protocol/stepping-stone/Plugin.cc | 5 ++- .../protocol/stepping-stone/SteppingStone.cc | 4 +-- src/analyzer/protocol/syslog/Plugin.cc | 5 ++- src/analyzer/protocol/tcp/ContentLine.cc | 5 ++- src/analyzer/protocol/tcp/Plugin.cc | 5 ++- src/analyzer/protocol/tcp/TCP.cc | 16 +++++++--- src/analyzer/protocol/tcp/TCP.h | 1 - src/analyzer/protocol/tcp/TCP_Endpoint.cc | 1 + src/analyzer/protocol/tcp/TCP_Endpoint.h | 1 + src/analyzer/protocol/tcp/TCP_Reassembler.cc | 10 +++--- src/analyzer/protocol/tcp/TCP_Reassembler.h | 6 +++- src/analyzer/protocol/tcp/functions.bif | 2 ++ src/analyzer/protocol/teredo/Plugin.cc | 5 ++- src/analyzer/protocol/teredo/Teredo.cc | 1 + src/analyzer/protocol/udp/Plugin.cc | 5 ++- src/analyzer/protocol/vxlan/Plugin.cc | 4 +-- src/analyzer/protocol/vxlan/VXLAN.cc | 6 ++++ src/analyzer/protocol/vxlan/VXLAN.h | 2 -- src/analyzer/protocol/xmpp/Plugin.cc | 3 +- src/analyzer/protocol/xmpp/xmpp.pac | 1 + src/analyzer/protocol/zip/Plugin.cc | 5 ++- src/binpac_bro.h | 2 -- src/bro-bif.h | 2 ++ src/broker/Data.cc | 2 ++ src/broker/Data.h | 9 +++--- src/broker/Manager.cc | 5 ++- src/broker/Manager.h | 13 ++++---- src/broker/Store.cc | 1 + src/broker/Store.h | 2 +- src/file_analysis/Component.h | 5 ++- src/file_analysis/File.cc | 7 +++-- src/file_analysis/File.h | 11 ++++--- src/file_analysis/FileReassembler.h | 1 - src/file_analysis/FileTimer.cc | 3 +- src/file_analysis/FileTimer.h | 5 ++- src/file_analysis/Manager.cc | 6 ++-- src/file_analysis/Manager.h | 26 +++++++++------- src/file_analysis/Tag.h | 10 ++++-- .../analyzer/data_event/Plugin.cc | 4 +-- src/file_analysis/analyzer/entropy/Plugin.cc | 4 +-- src/file_analysis/analyzer/extract/Plugin.cc | 4 +-- src/file_analysis/analyzer/hash/Plugin.cc | 4 +-- src/file_analysis/analyzer/pe/Plugin.cc | 4 +-- src/file_analysis/analyzer/unified2/Plugin.cc | 4 +-- src/file_analysis/analyzer/x509/OCSP.cc | 1 + src/file_analysis/analyzer/x509/Plugin.cc | 5 ++- src/file_analysis/analyzer/x509/X509Common.cc | 1 + src/file_analysis/analyzer/x509/X509Common.h | 2 ++ src/file_analysis/file_analysis.bif | 2 ++ src/input/Manager.cc | 9 ++++-- src/input/Manager.h | 10 +++--- src/input/ReaderFrontend.cc | 2 -- src/input/ReaderFrontend.h | 3 +- src/input/Tag.h | 10 ++++-- src/iosource/BPF_Program.cc | 3 +- src/iosource/BPF_Program.h | 2 +- src/iosource/Component.h | 4 +-- src/iosource/IOSource.h | 6 ---- src/iosource/Manager.cc | 4 ++- src/iosource/Manager.h | 3 ++ src/iosource/Packet.cc | 2 ++ src/iosource/Packet.h | 10 ++++-- src/iosource/PktDumper.cc | 4 +-- src/iosource/PktDumper.h | 5 +-- src/iosource/PktSrc.cc | 8 ++--- src/iosource/PktSrc.h | 7 +++-- src/iosource/pcap/Plugin.cc | 4 +-- src/iosource/pcap/Source.cc | 3 +- src/iosource/pcap/Source.h | 6 ++++ src/logging/Manager.cc | 9 ++++-- src/logging/Manager.h | 3 +- src/logging/Tag.h | 10 ++++-- src/logging/WriterFrontend.h | 2 -- src/logging/writers/ascii/Ascii.h | 1 + src/logging/writers/none/None.cc | 2 ++ src/logging/writers/sqlite/SQLite.h | 1 + src/main.cc | 1 + src/parse.y | 5 +++ src/plugin/ComponentManager.h | 4 +-- src/plugin/Manager.cc | 1 + src/plugin/Plugin.cc | 6 +++- src/plugin/Plugin.h | 4 +-- src/probabilistic/BitVector.cc | 5 ++- src/probabilistic/BitVector.h | 6 ++-- src/probabilistic/BloomFilter.cc | 6 ++-- src/probabilistic/BloomFilter.h | 4 ++- src/probabilistic/CardinalityCounter.cc | 7 +++-- src/probabilistic/CardinalityCounter.h | 8 +++-- src/probabilistic/CounterVector.cc | 6 ++-- src/probabilistic/CounterVector.h | 4 ++- src/probabilistic/Hasher.cc | 6 +++- src/probabilistic/Hasher.h | 5 +-- src/probabilistic/Topk.cc | 4 ++- src/re-parse.y | 1 + src/re-scan.l | 1 + src/rule-parse.y | 2 ++ src/rule-scan.l | 1 + src/scan.l | 3 ++ src/supervisor/Supervisor.cc | 5 ++- src/threading/BasicThread.h | 3 +- src/threading/Formatter.cc | 4 +-- src/threading/Formatter.h | 14 +++++++-- src/threading/Manager.cc | 3 +- src/threading/Manager.h | 6 ++-- src/threading/formatters/Ascii.cc | 6 ++-- src/threading/formatters/JSON.cc | 8 +++-- src/util.cc | 3 +- src/zeek.bif | 1 + src/zeekygen/Configuration.cc | 1 - src/zeekygen/Configuration.h | 5 ++- src/zeekygen/IdentifierInfo.cc | 1 + src/zeekygen/IdentifierInfo.h | 6 ++-- src/zeekygen/Manager.cc | 4 +++ src/zeekygen/Manager.h | 11 +++---- src/zeekygen/PackageInfo.h | 2 ++ src/zeekygen/ScriptInfo.cc | 2 ++ src/zeekygen/ScriptInfo.h | 3 +- src/zeekygen/Target.cc | 3 ++ src/zeekygen/Target.h | 10 +++--- src/zeekygen/utils.cc | 2 ++ src/zeekygen/utils.h | 6 ++-- src/zeekygen/zeekygen.bif | 3 ++ testing/btest/plugins/file-plugin/src/Foo.cc | 2 ++ .../btest/plugins/file-plugin/src/Plugin.cc | 5 +-- .../btest/plugins/hooks-plugin/src/Plugin.cc | 1 + .../logging-hooks-plugin/src/Plugin.cc | 1 + .../btest/plugins/pktdumper-plugin/src/Foo.cc | 5 +-- .../plugins/pktdumper-plugin/src/Plugin.cc | 2 ++ .../btest/plugins/pktsrc-plugin/src/Foo.cc | 8 +++-- .../btest/plugins/pktsrc-plugin/src/Plugin.cc | 1 + .../plugins/protocol-plugin/src/Plugin.cc | 1 + .../reporter-hook-plugin/src/Plugin.cc | 1 + testing/btest/plugins/writer-plugin/src/Foo.h | 1 + 332 files changed, 1047 insertions(+), 606 deletions(-) diff --git a/src/Anon.cc b/src/Anon.cc index 6d2e7a6173..583d11016b 100644 --- a/src/Anon.cc +++ b/src/Anon.cc @@ -1,3 +1,5 @@ +#include "Anon.h" + #include #include #include @@ -5,7 +7,6 @@ #include "util.h" #include "net_util.h" -#include "Anon.h" #include "Val.h" #include "NetVar.h" diff --git a/src/Anon.h b/src/Anon.h index f7cbaa24f3..cce4ab1922 100644 --- a/src/Anon.h +++ b/src/Anon.h @@ -11,11 +11,11 @@ #pragma once #include -#include #include #include "Reporter.h" -#include "net_util.h" + +using std::map; // TODO: Anon.h may not be the right place to put these functions ... diff --git a/src/Attr.cc b/src/Attr.cc index d63f72a3fd..c98af3586c 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -4,6 +4,8 @@ #include "Attr.h" #include "Expr.h" +#include "Desc.h" +#include "Val.h" #include "threading/SerialTypes.h" const char* attr_name(attr_tag t) diff --git a/src/Attr.h b/src/Attr.h index 53d60eda27..9f8f6c90d2 100644 --- a/src/Attr.h +++ b/src/Attr.h @@ -3,6 +3,7 @@ #pragma once #include "Obj.h" +#include "BroList.h" class Expr; diff --git a/src/Base64.cc b/src/Base64.cc index 53ddfecef6..11455ef150 100644 --- a/src/Base64.cc +++ b/src/Base64.cc @@ -1,5 +1,7 @@ #include "zeek-config.h" #include "Base64.h" +#include "BroString.h" + #include int Base64Converter::default_base64_table[256]; diff --git a/src/Base64.h b/src/Base64.h index 8c8146c3c0..c829203851 100644 --- a/src/Base64.h +++ b/src/Base64.h @@ -1,14 +1,10 @@ #pragma once -#include -#include -#include - -#include "util.h" -#include "BroString.h" #include "Reporter.h" #include "Conn.h" +class BroString; + // Maybe we should have a base class for generic decoders? class Base64Converter { public: diff --git a/src/BroString.cc b/src/BroString.cc index 2938410a9f..788bbf6d75 100644 --- a/src/BroString.cc +++ b/src/BroString.cc @@ -1,13 +1,13 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek-config.h" +#include "BroString.h" #include +#include #include -#include - -#include "BroString.h" +#include "Val.h" #include "Var.h" #include "Reporter.h" diff --git a/src/BroString.h b/src/BroString.h index 0361c0a927..df9f216ecf 100644 --- a/src/BroString.h +++ b/src/BroString.h @@ -2,13 +2,13 @@ #pragma once +#include "util.h" + #include #include -#include -#include -#include +#include -#include "util.h" +#include typedef u_char* byte_vec; diff --git a/src/Brofiler.cc b/src/Brofiler.cc index 5603040786..c406182747 100644 --- a/src/Brofiler.cc +++ b/src/Brofiler.cc @@ -1,3 +1,5 @@ +#include "Brofiler.h" + #include #include #include @@ -5,7 +7,10 @@ #include #include #include -#include "Brofiler.h" + +#include "Stmt.h" +#include "Desc.h" +#include "Reporter.h" #include "util.h" Brofiler::Brofiler() diff --git a/src/Brofiler.h b/src/Brofiler.h index 504b6e324f..ee8d528baf 100644 --- a/src/Brofiler.h +++ b/src/Brofiler.h @@ -3,8 +3,14 @@ #include #include #include -#include +#include +using std::list; +using std::map; +using std::pair; +using std::string; + +class Stmt; /** * A simple class for managing stats of Bro script coverage across Bro runs. diff --git a/src/CCL.h b/src/CCL.h index 28f41e9922..51fa901be8 100644 --- a/src/CCL.h +++ b/src/CCL.h @@ -2,8 +2,9 @@ #pragma once +#include "util.h" // for ptr_compat_int + #include -#include "List.h" typedef std::vector int_list; diff --git a/src/CompHash.h b/src/CompHash.h index 073d4b049f..ef0f0d14c3 100644 --- a/src/CompHash.h +++ b/src/CompHash.h @@ -2,10 +2,10 @@ #pragma once -#include "Hash.h" #include "Type.h" class ListVal; +class HashKey; class CompositeHash { public: diff --git a/src/Conn.cc b/src/Conn.cc index 1f96af5a7d..650105ddd3 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -2,15 +2,18 @@ #include "zeek-config.h" +#include "Conn.h" + #include +#include "Desc.h" #include "Net.h" #include "NetVar.h" -#include "Conn.h" #include "Event.h" #include "Sessions.h" #include "Reporter.h" #include "Timer.h" +#include "iosource/IOSource.h" #include "analyzer/protocol/pia/PIA.h" #include "binpac.h" #include "TunnelEncapsulation.h" diff --git a/src/Conn.h b/src/Conn.h index a7ea9d3820..562511d8b9 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -7,17 +7,15 @@ #include #include "Dict.h" -#include "Val.h" #include "Timer.h" -#include "RuleMatcher.h" +#include "Rule.h" #include "IPAddr.h" -#include "TunnelEncapsulation.h" #include "UID.h" #include "WeirdState.h" +#include "iosource/Packet.h" #include "analyzer/Tag.h" #include "analyzer/Analyzer.h" -#include "iosource/Packet.h" class Connection; class ConnectionTimer; @@ -26,6 +24,9 @@ class LoginConn; class RuleHdrTest; class Specific_RE_Matcher; class RuleEndpointState; +class EncapsulationStack; +class Val; +class RecordVal; namespace analyzer { class TransportLayerAnalyzer; } diff --git a/src/DFA.cc b/src/DFA.cc index 23aecbad60..e99e772fcc 100644 --- a/src/DFA.cc +++ b/src/DFA.cc @@ -2,8 +2,9 @@ #include "zeek-config.h" -#include "EquivClass.h" #include "DFA.h" +#include "EquivClass.h" +#include "Desc.h" #include "digest.h" unsigned int DFA_State::transition_counter = 0; diff --git a/src/DFA.h b/src/DFA.h index a3b79920f6..332cdaff8e 100644 --- a/src/DFA.h +++ b/src/DFA.h @@ -3,9 +3,15 @@ #pragma once -#include +#include "RE.h" // for typedef AcceptingSet +#include "Obj.h" + +#include #include +#include +#include // for u_char + class DFA_State; // Transitions to the uncomputed state indicate that we haven't yet diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 93eba847cc..90b42db748 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -2,6 +2,8 @@ #include "zeek-config.h" +#include "DNS_Mgr.h" + #include #include #ifdef TIME_WITH_SYS_TIME @@ -29,9 +31,9 @@ #include -#include "DNS_Mgr.h" #include "Event.h" #include "Net.h" +#include "Val.h" #include "Var.h" #include "Reporter.h" #include "iosource/Manager.h" diff --git a/src/DNS_Mgr.h b/src/DNS_Mgr.h index 6b5975f876..3c816f1afb 100644 --- a/src/DNS_Mgr.h +++ b/src/DNS_Mgr.h @@ -7,9 +7,7 @@ #include #include -#include "util.h" #include "List.h" -#include "Dict.h" #include "EventHandler.h" #include "iosource/IOSource.h" #include "IPAddr.h" diff --git a/src/DbgBreakpoint.cc b/src/DbgBreakpoint.cc index 50f5295fc5..2ca288ffa6 100644 --- a/src/DbgBreakpoint.cc +++ b/src/DbgBreakpoint.cc @@ -2,17 +2,22 @@ #include "zeek-config.h" +#include "DbgBreakpoint.h" + #include +#include "Desc.h" #include "ID.h" #include "Queue.h" #include "Debug.h" #include "Scope.h" +#include "Frame.h" #include "Func.h" +#include "Val.h" #include "Stmt.h" -#include "DbgBreakpoint.h" #include "Timer.h" - +#include "Reporter.h" +#include "module_util.h" // BreakpointTimer used for time-based breakpoints class BreakpointTimer : public Timer { diff --git a/src/DbgBreakpoint.h b/src/DbgBreakpoint.h index 6f762e3c00..be5d27d6dd 100644 --- a/src/DbgBreakpoint.h +++ b/src/DbgBreakpoint.h @@ -2,7 +2,12 @@ #pragma once -#include "Debug.h" +#include + +using std::string; + +struct ParseLocationRec; +class Stmt; enum BreakCode { bcNoHit, bcHit, bcHitAndDelete }; class DbgBreakpoint { diff --git a/src/DbgDisplay.h b/src/DbgDisplay.h index 2af0107261..cd159ad3af 100644 --- a/src/DbgDisplay.h +++ b/src/DbgDisplay.h @@ -2,7 +2,7 @@ #pragma once -#include "Debug.h" +class Expr; // Automatic displays: display these at each stoppage. class DbgDisplay { diff --git a/src/DbgWatch.h b/src/DbgWatch.h index fa96c083bc..2dfb8ea605 100644 --- a/src/DbgWatch.h +++ b/src/DbgWatch.h @@ -2,7 +2,8 @@ #pragma once -#include "Debug.h" +class BroObj; +class Expr; class DbgWatch { public: diff --git a/src/Debug.cc b/src/Debug.cc index 946fca7bd4..e0f6ca33f5 100644 --- a/src/Debug.cc +++ b/src/Debug.cc @@ -2,6 +2,8 @@ #include "zeek-config.h" +#include "Debug.h" + #include #include #include @@ -11,13 +13,20 @@ using namespace std; #include "util.h" -#include "Debug.h" #include "DebugCmds.h" #include "DbgBreakpoint.h" +#include "ID.h" +#include "Expr.h" #include "Stmt.h" +#include "Frame.h" #include "Func.h" #include "Scope.h" #include "PolicyFile.h" +#include "Desc.h" +#include "Reporter.h" +#include "Val.h" +#include "module_util.h" +#include "input.h" #ifdef HAVE_READLINE #include diff --git a/src/Debug.h b/src/Debug.h index 42fd3bd42a..45bb8ed470 100644 --- a/src/Debug.h +++ b/src/Debug.h @@ -2,10 +2,15 @@ #pragma once +#include "Obj.h" +#include "Queue.h" +#include "StmtEnums.h" + #include #include #include +class Val; class Stmt; // This needs to be defined before we do the includes that come after it. @@ -17,17 +22,10 @@ struct ParseLocationRec { int line; }; -#include "Expr.h" -#include "Var.h" -#include "Frame.h" -#include "Queue.h" -#include "Dict.h" -#include "StmtEnums.h" -#include "DbgBreakpoint.h" - class StmtLocMapping; typedef PQueue Filemap; // mapping for a single file +class Frame; class DbgBreakpoint; class DbgWatch; class DbgDisplay; diff --git a/src/DebugCmds.cc b/src/DebugCmds.cc index 6ae6a56f40..0632d24a9a 100644 --- a/src/DebugCmds.cc +++ b/src/DebugCmds.cc @@ -2,6 +2,7 @@ // implementation of most commands. #include "zeek-config.h" +#include "DebugCmds.h" #include @@ -9,14 +10,18 @@ #include #include -#include "Debug.h" -#include "DebugCmds.h" #include "DebugCmdInfoConstants.cc" +#include "Debug.h" +#include "Desc.h" #include "DbgBreakpoint.h" +#include "ID.h" +#include "Frame.h" #include "Func.h" #include "Stmt.h" #include "Scope.h" +#include "Reporter.h" #include "PolicyFile.h" +#include "Val.h" #include "util.h" // diff --git a/src/Desc.cc b/src/Desc.cc index 7baddeab0a..4511f59614 100644 --- a/src/Desc.cc +++ b/src/Desc.cc @@ -1,15 +1,15 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek-config.h" +#include "Desc.h" #include +#include #include #include -#include "Desc.h" #include "File.h" #include "Reporter.h" - #include "ConvertUTF.h" #define DEFAULT_SIZE 128 diff --git a/src/Desc.h b/src/Desc.h index ff59aba43c..99ccffea0f 100644 --- a/src/Desc.h +++ b/src/Desc.h @@ -2,12 +2,14 @@ #pragma once -#include +#include "BroString.h" // for byte_vec +#include "util.h" // for bro_int_t + #include #include #include -#include "BroString.h" +#include // for u_char typedef enum { DESC_READABLE, diff --git a/src/Discard.cc b/src/Discard.cc index f84e901143..cb279ca580 100644 --- a/src/Discard.cc +++ b/src/Discard.cc @@ -1,13 +1,18 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include - #include "zeek-config.h" -#include "Net.h" -#include "Var.h" #include "Discard.h" +#include + +#include "Net.h" +#include "Func.h" +#include "Var.h" +#include "Val.h" +#include "IP.h" +#include "Reporter.h" // for InterpreterException + Discarder::Discarder() { check_ip = internal_func("discarder_check_ip"); diff --git a/src/Discard.h b/src/Discard.h index 28a041cf05..fb154dcf92 100644 --- a/src/Discard.h +++ b/src/Discard.h @@ -2,14 +2,14 @@ #pragma once -#include "IP.h" -#include "Func.h" +#include // for u_char struct ip; struct tcphdr; struct udphdr; struct icmp; +class IP_Hdr; class Val; class RecordType; class Func; diff --git a/src/EquivClass.cc b/src/EquivClass.cc index 60cddbf486..c067137227 100644 --- a/src/EquivClass.cc +++ b/src/EquivClass.cc @@ -3,6 +3,7 @@ #include "zeek-config.h" #include "EquivClass.h" +#include "CCL.h" EquivClass::EquivClass(int arg_size) { diff --git a/src/EquivClass.h b/src/EquivClass.h index 5441442002..8285a8d866 100644 --- a/src/EquivClass.h +++ b/src/EquivClass.h @@ -4,7 +4,7 @@ #include -#include "CCL.h" +class CCL; class EquivClass { public: diff --git a/src/Event.cc b/src/Event.cc index 8e8db10ed9..98b4f5455c 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -3,6 +3,7 @@ #include "zeek-config.h" #include "Event.h" +#include "Desc.h" #include "Func.h" #include "NetVar.h" #include "Trigger.h" diff --git a/src/Event.h b/src/Event.h index 411d48b011..3d367ef034 100644 --- a/src/Event.h +++ b/src/Event.h @@ -2,10 +2,8 @@ #pragma once -#include "EventRegistry.h" - -#include "analyzer/Tag.h" #include "analyzer/Analyzer.h" +#include "Val.h" class EventMgr; diff --git a/src/EventHandler.cc b/src/EventHandler.cc index 8c1b9f28f2..9928df9a8b 100644 --- a/src/EventHandler.cc +++ b/src/EventHandler.cc @@ -1,5 +1,6 @@ -#include "Event.h" #include "EventHandler.h" +#include "Event.h" +#include "Desc.h" #include "Func.h" #include "Scope.h" #include "NetVar.h" diff --git a/src/EventHandler.h b/src/EventHandler.h index 4978eb3064..ed01bf18a3 100644 --- a/src/EventHandler.h +++ b/src/EventHandler.h @@ -2,11 +2,10 @@ #pragma once -#include +#include "BroList.h" + #include #include -#include "List.h" -#include "BroList.h" class Func; class FuncType; diff --git a/src/EventRegistry.cc b/src/EventRegistry.cc index a7ad0636f4..957f90efe0 100644 --- a/src/EventRegistry.cc +++ b/src/EventRegistry.cc @@ -1,4 +1,5 @@ #include "EventRegistry.h" +#include "EventHandler.h" #include "RE.h" #include "Reporter.h" diff --git a/src/EventRegistry.h b/src/EventRegistry.h index a8c1eb1049..e49366e0d3 100644 --- a/src/EventRegistry.h +++ b/src/EventRegistry.h @@ -4,11 +4,14 @@ #include #include +#include -#include "Func.h" -#include "List.h" -#include "Dict.h" -#include "EventHandler.h" +using std::string; +using std::vector; + +class EventHandler; +class EventHandlerPtr; +class RE_Matcher; // The registry keeps track of all events that we provide or handle. class EventRegistry { diff --git a/src/Expr.cc b/src/Expr.cc index 1049f12929..b21029d945 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -4,6 +4,7 @@ #include "Expr.h" #include "Event.h" +#include "Desc.h" #include "Frame.h" #include "Func.h" #include "RE.h" @@ -15,6 +16,8 @@ #include "Trigger.h" #include "IPAddr.h" #include "digest.h" +#include "module_util.h" +#include "DebugLogger.h" #include "broker/Data.h" diff --git a/src/Expr.h b/src/Expr.h index 34aaec4847..52bc32b743 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -5,16 +5,18 @@ // BRO expressions. #include "BroList.h" -#include "ID.h" #include "Timer.h" +#include "Type.h" #include "Val.h" -#include "Debug.h" #include "EventHandler.h" #include "TraverseTypes.h" #include +#include #include +using std::string; + typedef enum { EXPR_ANY = -1, EXPR_NAME, EXPR_CONST, @@ -873,8 +875,6 @@ protected: int num_fields; }; -class EventHandler; - class ScheduleTimer : public Timer { public: ScheduleTimer(EventHandlerPtr event, val_list* args, double t, diff --git a/src/File.cc b/src/File.cc index 2a48f14ef9..14ced1d32c 100644 --- a/src/File.cc +++ b/src/File.cc @@ -1,6 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek-config.h" +#include "File.h" #include #ifdef TIME_WITH_SYS_TIME @@ -20,13 +21,14 @@ #include -#include "File.h" +#include "Attr.h" #include "Type.h" #include "Expr.h" #include "NetVar.h" #include "Net.h" #include "Event.h" #include "Reporter.h" +#include "Desc.h" std::list> BroFile::open_files; diff --git a/src/File.h b/src/File.h index 6433cd7dfd..c5a358a23a 100644 --- a/src/File.h +++ b/src/File.h @@ -2,19 +2,21 @@ #pragma once -#include -#include "util.h" #include "Obj.h" -#include "Attr.h" #include +#include #include +#include + # ifdef NEED_KRB5_H # include # endif // NEED_KRB5_H +class Attributes; class BroType; +class RecordVal; class BroFile : public BroObj { public: diff --git a/src/Frag.cc b/src/Frag.cc index a39f2ea33e..c09ffe9834 100644 --- a/src/Frag.cc +++ b/src/Frag.cc @@ -2,11 +2,12 @@ #include "zeek-config.h" -#include "util.h" -#include "Hash.h" #include "Frag.h" +#include "Hash.h" +#include "IP.h" #include "NetVar.h" #include "Sessions.h" +#include "Reporter.h" #define MIN_ACCEPTABLE_FRAG_SIZE 64 #define MAX_ACCEPTABLE_FRAG_SIZE 64000 diff --git a/src/Frag.h b/src/Frag.h index 1c5d83d89b..2a8b21db6c 100644 --- a/src/Frag.h +++ b/src/Frag.h @@ -2,16 +2,18 @@ #pragma once -#include - -#include "util.h" -#include "IP.h" -#include "Net.h" +#include "util.h" // for bro_uint_t +#include "IPAddr.h" #include "Reassem.h" #include "Timer.h" +#include + +#include // for u_char + class HashKey; class NetSessions; +class IP_Hdr; class FragReassembler; class FragTimer; diff --git a/src/Frame.cc b/src/Frame.cc index 369e28db47..a8e6ac1e65 100644 --- a/src/Frame.cc +++ b/src/Frame.cc @@ -1,9 +1,13 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include "Frame.h" + #include #include "broker/Data.h" -#include "Frame.h" +#include "Func.h" +#include "Desc.h" +#include "IntrusivePtr.h" #include "Trigger.h" vector g_frame_stack; diff --git a/src/Frame.h b/src/Frame.h index 21f411ba1f..fabe2a9e19 100644 --- a/src/Frame.h +++ b/src/Frame.h @@ -2,17 +2,20 @@ #pragma once +#include "BroList.h" // for typedef val_list +#include "Val.h" + #include -#include #include +#include +#include #include #include -#include "Val.h" - namespace trigger { class Trigger; } class CallExpr; +class BroFunc; class Frame : public BroObj { public: diff --git a/src/Func.cc b/src/Func.cc index e0f4958660..759b132a9a 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -1,6 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek-config.h" +#include "Func.h" #include #include @@ -32,12 +33,14 @@ #include #include "Base64.h" +#include "Debug.h" +#include "Desc.h" +#include "Expr.h" #include "Stmt.h" #include "Scope.h" #include "Net.h" #include "NetVar.h" #include "File.h" -#include "Func.h" #include "Frame.h" #include "Var.h" #include "analyzer/protocol/login/Login.h" @@ -47,6 +50,9 @@ #include "Traverse.h" #include "Reporter.h" #include "plugin/Manager.h" +#include "module_util.h" +#include "iosource/PktSrc.h" +#include "iosource/PktDumper.h" extern RETSIGTYPE sig_handler(int signo); diff --git a/src/Func.h b/src/Func.h index 802c4509da..e70980d1de 100644 --- a/src/Func.h +++ b/src/Func.h @@ -2,16 +2,21 @@ #pragma once +#include "BroList.h" +#include "Obj.h" +#include "Type.h" /* for function_flavor */ +#include "TraverseTypes.h" + #include #include +#include +#include #include #include -#include "BroList.h" -#include "Obj.h" -#include "Debug.h" -#include "Frame.h" +using std::string; +using std::vector; class Val; class ListExpr; @@ -20,6 +25,7 @@ class Stmt; class Frame; class ID; class CallExpr; +class Scope; class Func : public BroObj { public: diff --git a/src/Hash.cc b/src/Hash.cc index 57ef6659b9..c20e928dd5 100644 --- a/src/Hash.cc +++ b/src/Hash.cc @@ -19,6 +19,7 @@ #include "Hash.h" #include "Reporter.h" +#include "BroString.h" #include "siphash24.h" diff --git a/src/Hash.h b/src/Hash.h index 0a5655f049..9a5c99c60e 100644 --- a/src/Hash.h +++ b/src/Hash.h @@ -2,12 +2,14 @@ #pragma once +#include "util.h" // for bro_int_t + #include -#include "BroString.h" - #define UHASH_KEY_SIZE 36 +class BroString; + typedef uint64_t hash_t; typedef enum { diff --git a/src/ID.cc b/src/ID.cc index 6029cb9862..793c7bc7ff 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -3,6 +3,7 @@ #include "zeek-config.h" #include "ID.h" +#include "Desc.h" #include "Expr.h" #include "Dict.h" #include "EventRegistry.h" @@ -11,7 +12,11 @@ #include "File.h" #include "Scope.h" #include "Traverse.h" +#include "Val.h" #include "zeekygen/Manager.h" +#include "zeekygen/IdentifierInfo.h" +#include "zeekygen/ScriptInfo.h" +#include "module_util.h" ID::ID(const char* arg_name, IDScope arg_scope, bool arg_is_export) { diff --git a/src/ID.h b/src/ID.h index b3e36ebd4f..f9d7f01e40 100644 --- a/src/ID.h +++ b/src/ID.h @@ -2,14 +2,21 @@ #pragma once +#include "Obj.h" #include "Type.h" #include "Attr.h" #include "Notifier.h" #include "TraverseTypes.h" + +#include #include +#include class Val; +class Expr; class Func; +class BroType; +class Attributes; typedef enum { INIT_NONE, INIT_FULL, INIT_EXTRA, INIT_REMOVE, } init_class; typedef enum { SCOPE_FUNCTION, SCOPE_MODULE, SCOPE_GLOBAL } IDScope; diff --git a/src/IP.cc b/src/IP.cc index 2d220bae23..bc7c90262a 100644 --- a/src/IP.cc +++ b/src/IP.cc @@ -1,10 +1,11 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include "IP.h" + #include #include #include -#include "IP.h" #include "Type.h" #include "Val.h" #include "Var.h" diff --git a/src/IP.h b/src/IP.h index a6017fdd30..985f23c0b4 100644 --- a/src/IP.h +++ b/src/IP.h @@ -3,15 +3,16 @@ #pragma once #include "zeek-config.h" -#include "net_util.h" #include "IPAddr.h" #include "Reporter.h" -#include "Val.h" -#include "Type.h" #include + +#include // for u_char #include #include +using std::vector; + #ifdef ENABLE_MOBILE_IPV6 #ifndef IPPROTO_MOBILITY diff --git a/src/IPAddr.h b/src/IPAddr.h index a834fec851..238ba10407 100644 --- a/src/IPAddr.h +++ b/src/IPAddr.h @@ -2,17 +2,16 @@ #pragma once +#include "BroString.h" +#include "Hash.h" +#include "threading/SerialTypes.h" + #include #include #include #include -#include "BroString.h" -#include "Hash.h" -#include "util.h" -#include "Type.h" -#include "threading/SerialTypes.h" - +using std::string; struct ConnID; namespace analyzer { class ExpectedConn; } diff --git a/src/NFA.cc b/src/NFA.cc index 55fdb09439..f22b4e0b21 100644 --- a/src/NFA.cc +++ b/src/NFA.cc @@ -2,10 +2,12 @@ #include "zeek-config.h" -#include - #include "NFA.h" +#include "Desc.h" #include "EquivClass.h" +#include "IntSet.h" + +#include static int nfa_state_id = 0; diff --git a/src/NFA.h b/src/NFA.h index 2982323afe..6b6827acc2 100644 --- a/src/NFA.h +++ b/src/NFA.h @@ -2,9 +2,11 @@ #pragma once -#include "RE.h" -#include "IntSet.h" +#include "Obj.h" +#include "List.h" +class CCL; +class Func; class NFA_State; class EquivClass; diff --git a/src/Net.cc b/src/Net.cc index 77b7ad01fc..1b35907a93 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -1,6 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek-config.h" +#include "Net.h" #include #ifdef TIME_WITH_SYS_TIME @@ -19,13 +20,16 @@ #include #include +extern "C" { +#include "setsignal.h" +}; + #include "NetVar.h" #include "Sessions.h" #include "Event.h" #include "Timer.h" #include "Var.h" #include "Reporter.h" -#include "Net.h" #include "Anon.h" #include "PacketDumper.h" #include "iosource/Manager.h" @@ -34,10 +38,6 @@ #include "plugin/Manager.h" #include "broker/Manager.h" -extern "C" { -#include "setsignal.h" -}; - extern "C" { extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *); } diff --git a/src/Net.h b/src/Net.h index 655bab8c8e..25901a78dd 100644 --- a/src/Net.h +++ b/src/Net.h @@ -2,14 +2,14 @@ #pragma once +#include #include #include #include -#include "net_util.h" -#include "util.h" -#include "List.h" -#include "Func.h" +#include // for ino_t + +using std::string; namespace iosource { class IOSource; diff --git a/src/NetVar.cc b/src/NetVar.cc index c3f8b34340..53a5e6c35c 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -2,8 +2,10 @@ #include "zeek-config.h" -#include "Var.h" #include "NetVar.h" +#include "Var.h" +#include "EventHandler.h" +#include "Val.h" RecordType* conn_id; RecordType* endpoint; diff --git a/src/Notifier.cc b/src/Notifier.cc index 265c574b2a..0acd7e6cec 100644 --- a/src/Notifier.cc +++ b/src/Notifier.cc @@ -1,7 +1,9 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include "DebugLogger.h" #include "Notifier.h" +#include "DebugLogger.h" + +#include notifier::Registry notifier::registry; diff --git a/src/Notifier.h b/src/Notifier.h index e85345fa81..a3ae5188b7 100644 --- a/src/Notifier.h +++ b/src/Notifier.h @@ -7,12 +7,7 @@ #pragma once -#include #include -#include - -#include "util.h" -#include "DebugLogger.h" namespace notifier { diff --git a/src/Obj.cc b/src/Obj.cc index c2e47f50b8..91a9349d46 100644 --- a/src/Obj.cc +++ b/src/Obj.cc @@ -1,10 +1,11 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek-config.h" +#include "Obj.h" #include -#include "Obj.h" +#include "Desc.h" #include "Func.h" #include "File.h" #include "plugin/Manager.h" diff --git a/src/Obj.h b/src/Obj.h index 0147a21a8b..0c96013620 100644 --- a/src/Obj.h +++ b/src/Obj.h @@ -4,8 +4,7 @@ #include -#include "input.h" -#include "Desc.h" +class ODesc; class Location { public: diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index 033c4e6fe6..b522539469 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -5,9 +5,12 @@ #include "OpaqueVal.h" #include "NetVar.h" #include "Reporter.h" +#include "Desc.h" +#include "Var.h" #include "probabilistic/BloomFilter.h" #include "probabilistic/CardinalityCounter.h" +#include #include // Helper to retrieve a broker value out of a broker::vector at a specified diff --git a/src/OpaqueVal.h b/src/OpaqueVal.h index faff969cbf..7d216e7ff9 100644 --- a/src/OpaqueVal.h +++ b/src/OpaqueVal.h @@ -2,14 +2,16 @@ #pragma once -#include -#include - #include "RandTest.h" #include "Val.h" #include "digest.h" #include "paraglob/paraglob.h" +#include + +#include // for u_char + +namespace broker { class data; } class OpaqueVal; /** diff --git a/src/Options.cc b/src/Options.cc index 27fc648ce3..c4f864189a 100644 --- a/src/Options.cc +++ b/src/Options.cc @@ -1,9 +1,13 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include - #include "zeek-config.h" +#include "Options.h" + +#include + +#include + #ifdef HAVE_GETOPT_H #include #endif @@ -11,8 +15,6 @@ #include "bsd-getopt-long.h" #include "logging/writers/ascii/Ascii.h" -#include "Options.h" - void zeek::Options::filter_supervisor_options() { pcap_filter = {}; diff --git a/src/PacketDumper.cc b/src/PacketDumper.cc index 0d64c89290..ffb2a037d9 100644 --- a/src/PacketDumper.cc +++ b/src/PacketDumper.cc @@ -1,14 +1,10 @@ // See the file "COPYING" in the main distribution directory for copyright. - #include "zeek-config.h" - -#include -#include - -#include "Event.h" -#include "Net.h" #include "PacketDumper.h" +#include "Reporter.h" +#include "util.h" +#include "iosource/PktDumper.h" PacketDumper::PacketDumper(pcap_dumper_t* arg_pkt_dump) { diff --git a/src/PacketDumper.h b/src/PacketDumper.h index 38a31b920d..421985150e 100644 --- a/src/PacketDumper.h +++ b/src/PacketDumper.h @@ -4,6 +4,8 @@ #include +#include // for u_char + class PacketDumper { public: explicit PacketDumper(pcap_dumper_t* pkt_dump); diff --git a/src/PacketFilter.cc b/src/PacketFilter.cc index f506d1778e..480326d070 100644 --- a/src/PacketFilter.cc +++ b/src/PacketFilter.cc @@ -1,4 +1,5 @@ #include "PacketFilter.h" +#include "IP.h" void PacketFilter::DeleteFilter(void* data) { diff --git a/src/PacketFilter.h b/src/PacketFilter.h index 384ce503c5..4729a4cd5e 100644 --- a/src/PacketFilter.h +++ b/src/PacketFilter.h @@ -2,9 +2,12 @@ #pragma once -#include "IP.h" +#include "IPAddr.h" #include "PrefixTable.h" +class IP_Hdr; +class Val; + class PacketFilter { public: explicit PacketFilter(bool arg_default); diff --git a/src/PrefixTable.cc b/src/PrefixTable.cc index 007e08349c..ee2280757a 100644 --- a/src/PrefixTable.cc +++ b/src/PrefixTable.cc @@ -1,5 +1,6 @@ #include "PrefixTable.h" #include "Reporter.h" +#include "Val.h" prefix_t* PrefixTable::MakePrefix(const IPAddr& addr, int width) { diff --git a/src/PrefixTable.h b/src/PrefixTable.h index e22a89fff8..21b956756a 100644 --- a/src/PrefixTable.h +++ b/src/PrefixTable.h @@ -1,13 +1,19 @@ #pragma once -#include "Val.h" -#include "net_util.h" #include "IPAddr.h" extern "C" { #include "patricia.h" } +#include + +using std::list; +using std::tuple; + +class Val; +class SubNetVal; + class PrefixTable { private: struct iterator { diff --git a/src/PriorityQueue.h b/src/PriorityQueue.h index 0368396836..665a83aa07 100644 --- a/src/PriorityQueue.h +++ b/src/PriorityQueue.h @@ -3,7 +3,7 @@ #pragma once #include -#include "util.h" +#include class PriorityQueue; diff --git a/src/Queue.h b/src/Queue.h index 72adbd572a..1e5447ef27 100644 --- a/src/Queue.h +++ b/src/Queue.h @@ -2,6 +2,8 @@ #pragma once +#include + // Queue.h -- // Interface for class Queue, current implementation is as an // array of ent's. This implementation was chosen to optimize diff --git a/src/RE.cc b/src/RE.cc index be92ec6e41..7ed075a0be 100644 --- a/src/RE.cc +++ b/src/RE.cc @@ -1,15 +1,16 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek-config.h" +#include "RE.h" #include #include -#include "RE.h" #include "DFA.h" #include "CCL.h" #include "EquivClass.h" #include "Reporter.h" +#include "BroString.h" CCL* curr_ccl = 0; diff --git a/src/RE.h b/src/RE.h index ace38f2767..f1a2317659 100644 --- a/src/RE.h +++ b/src/RE.h @@ -2,9 +2,7 @@ #pragma once -#include "Obj.h" -#include "Dict.h" -#include "BroString.h" +#include "List.h" #include "CCL.h" #include "EquivClass.h" @@ -12,6 +10,7 @@ #include #include +#include // for u_char #include typedef int (*cce_func)(int); @@ -21,6 +20,7 @@ class DFA_Machine; class Specific_RE_Matcher; class RE_Matcher; class DFA_State; +class BroString; extern int case_insensitive; extern CCL* curr_ccl; diff --git a/src/RandTest.cc b/src/RandTest.cc index 94e76500b5..396cd4ebde 100644 --- a/src/RandTest.cc +++ b/src/RandTest.cc @@ -12,9 +12,10 @@ Modified for Bro by Seth Hall - July 2010 */ -#include #include "RandTest.h" +#include + #define log2of10 3.32192809488736234787 /* RT_LOG2 -- Calculate log to the base 2 */ static double rt_log2(double x) diff --git a/src/RandTest.h b/src/RandTest.h index 32c2f38ee3..a030008eeb 100644 --- a/src/RandTest.h +++ b/src/RandTest.h @@ -1,6 +1,6 @@ #pragma once -#include "util.h" +#include #define RT_MONTEN 6 /* Bytes used as Monte Carlo co-ordinates. This should be no more diff --git a/src/Reassem.cc b/src/Reassem.cc index 898276d0e4..658c6fe395 100644 --- a/src/Reassem.cc +++ b/src/Reassem.cc @@ -1,10 +1,13 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include "zeek-config.h" +#include "Reassem.h" + #include -#include "zeek-config.h" +#include "Desc.h" -#include "Reassem.h" +using std::min; uint64_t Reassembler::total_size = 0; uint64_t Reassembler::sizes[REASSEM_NUM]; diff --git a/src/Reassem.h b/src/Reassem.h index 5c056bfd74..1baf93bc84 100644 --- a/src/Reassem.h +++ b/src/Reassem.h @@ -5,7 +5,10 @@ #include #include "Obj.h" -#include "IPAddr.h" + +#include +#include +#include // for u_char // Whenever subclassing the Reassembler class // you should add to this for known subclasses. diff --git a/src/Reporter.cc b/src/Reporter.cc index 5b1de82eff..7b7b6a6675 100644 --- a/src/Reporter.cc +++ b/src/Reporter.cc @@ -2,18 +2,23 @@ // See the file "COPYING" in the main distribution directory for copyright. // +#include "zeek-config.h" +#include "Reporter.h" + #include #include -#include "zeek-config.h" -#include "Reporter.h" +#include "Desc.h" #include "Event.h" +#include "Expr.h" #include "NetVar.h" #include "Net.h" #include "Conn.h" #include "Timer.h" +#include "EventHandler.h" #include "plugin/Plugin.h" #include "plugin/Manager.h" +#include "input.h" #include "file_analysis/File.h" #ifdef SYSLOG_INT diff --git a/src/Reporter.h b/src/Reporter.h index 29b7c5d586..6e364dce8f 100644 --- a/src/Reporter.h +++ b/src/Reporter.h @@ -11,8 +11,6 @@ #include #include -#include "util.h" -#include "EventHandler.h" #include "IPAddr.h" namespace analyzer { class Analyzer; } @@ -20,6 +18,7 @@ namespace file_analysis { class File; } class Connection; class Location; class Reporter; +class EventHandlerPtr; // One cannot raise this exception directly, go through the // Reporter's methods instead. diff --git a/src/Rule.cc b/src/Rule.cc index f626e7a7b7..ade2ca3f7c 100644 --- a/src/Rule.cc +++ b/src/Rule.cc @@ -1,6 +1,8 @@ #include "zeek-config.h" #include "Rule.h" +#include "RuleAction.h" +#include "RuleCondition.h" #include "RuleMatcher.h" // Start at one as we want search for this within a list, diff --git a/src/Rule.h b/src/Rule.h index 663174f28f..5d18ae8d84 100644 --- a/src/Rule.h +++ b/src/Rule.h @@ -1,17 +1,17 @@ #pragma once -#include -#include - -#include "Obj.h" #include "List.h" -#include "Dict.h" -#include "util.h" +#include "Obj.h" + +#include +#include + +#include +#include class RuleCondition; class RuleAction; class RuleHdrTest; - class Rule; typedef PList rule_list; diff --git a/src/RuleAction.h b/src/RuleAction.h index a20442952b..8ffc097fc6 100644 --- a/src/RuleAction.h +++ b/src/RuleAction.h @@ -1,11 +1,15 @@ #pragma once -#include "BroString.h" -#include "List.h" #include "util.h" #include "analyzer/Tag.h" +#include + +#include // for u_char + +using std::string; + class Rule; class RuleEndpointState; diff --git a/src/RuleCondition.cc b/src/RuleCondition.cc index efaa856ecd..ff42b135b9 100644 --- a/src/RuleCondition.cc +++ b/src/RuleCondition.cc @@ -1,8 +1,13 @@ #include "zeek-config.h" #include "RuleCondition.h" +#include "RuleMatcher.h" #include "analyzer/protocol/tcp/TCP.h" +#include "Reporter.h" #include "Scope.h" +#include "Func.h" +#include "Val.h" +#include "Var.h" // for internal_type() static inline bool is_established(const analyzer::tcp::TCP_Endpoint* e) { diff --git a/src/RuleCondition.h b/src/RuleCondition.h index ee0a30dda4..d8f0b684b2 100644 --- a/src/RuleCondition.h +++ b/src/RuleCondition.h @@ -1,10 +1,9 @@ #pragma once -#include "BroString.h" -#include "Func.h" -#include "List.h" -#include "util.h" +#include // for u_char +#include // for u_char +class ID; class Rule; class RuleEndpointState; diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc index 747b4ce16e..d6cdae8bd2 100644 --- a/src/RuleMatcher.cc +++ b/src/RuleMatcher.cc @@ -1,15 +1,23 @@ + +#include "zeek-config.h" +#include "RuleMatcher.h" + #include #include -#include "zeek-config.h" - +#include "RuleAction.h" +#include "RuleCondition.h" +#include "ID.h" +#include "IntSet.h" +#include "IP.h" #include "analyzer/Analyzer.h" -#include "RuleMatcher.h" #include "DFA.h" +#include "DebugLogger.h" #include "NetVar.h" #include "Scope.h" #include "File.h" #include "Reporter.h" +#include "module_util.h" // FIXME: Things that are not fully implemented/working yet: // diff --git a/src/RuleMatcher.h b/src/RuleMatcher.h index 9ba087d837..0650081000 100644 --- a/src/RuleMatcher.h +++ b/src/RuleMatcher.h @@ -1,24 +1,17 @@ #pragma once -#include +#include "Rule.h" +#include "RE.h" +#include "CCL.h" + #include #include #include #include #include -#include "IPAddr.h" -#include "BroString.h" -#include "List.h" -#include "RE.h" -#include "Net.h" -#include "Sessions.h" -#include "IntSet.h" -#include "util.h" -#include "Rule.h" -#include "RuleAction.h" -#include "RuleCondition.h" -#include "iosource/Packet.h" +#include // for u_char +#include //#define MATCHER_PRINT_STATS @@ -34,6 +27,18 @@ extern FILE* rules_in; extern int rules_line_number; extern const char* current_rule_file; +using std::vector; +using std::map; +using std::set; +using std::string; + +class Val; +class BroFile; +class IntSet; +class IP_Hdr; +class IPPrefix; +class RE_Match_State; +class Specific_RE_Matcher; class RuleMatcher; extern RuleMatcher* rule_matcher; diff --git a/src/Scope.cc b/src/Scope.cc index d4a87bc658..509f686594 100644 --- a/src/Scope.cc +++ b/src/Scope.cc @@ -2,10 +2,12 @@ #include "zeek-config.h" +#include "Scope.h" +#include "Desc.h" #include "ID.h" #include "Val.h" -#include "Scope.h" #include "Reporter.h" +#include "module_util.h" typedef PList scope_list; diff --git a/src/Scope.h b/src/Scope.h index 9b046fafa1..7485347669 100644 --- a/src/Scope.h +++ b/src/Scope.h @@ -5,11 +5,9 @@ #include #include -#include "Dict.h" #include "Obj.h" #include "BroList.h" #include "TraverseTypes.h" -#include "module_util.h" class ID; class BroType; diff --git a/src/SerializationFormat.cc b/src/SerializationFormat.cc index 0bf731fcbc..1aa8346f3f 100644 --- a/src/SerializationFormat.cc +++ b/src/SerializationFormat.cc @@ -1,9 +1,10 @@ +#include "SerializationFormat.h" + #include -#include "net_util.h" -#include "SerializationFormat.h" #include "DebugLogger.h" #include "Reporter.h" +#include "net_util.h" const float SerializationFormat::GROWTH_FACTOR = 2.5; diff --git a/src/SerializationFormat.h b/src/SerializationFormat.h index 772837ed91..5045c6af92 100644 --- a/src/SerializationFormat.h +++ b/src/SerializationFormat.h @@ -4,7 +4,7 @@ #include -#include "util.h" +#include class IPAddr; class IPPrefix; diff --git a/src/Sessions.cc b/src/Sessions.cc index 981a73350b..8f66ae8633 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -2,6 +2,7 @@ #include "zeek-config.h" +#include "Sessions.h" #include #include @@ -9,11 +10,11 @@ #include #include +#include "Desc.h" #include "Net.h" #include "Event.h" #include "Timer.h" #include "NetVar.h" -#include "Sessions.h" #include "Reporter.h" #include "analyzer/protocol/icmp/ICMP.h" diff --git a/src/Sessions.h b/src/Sessions.h index e98dd53a9c..1335e2dd88 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -2,26 +2,25 @@ #pragma once +#include "Frag.h" +#include "PacketFilter.h" +#include "NetVar.h" +#include "analyzer/protocol/tcp/Stats.h" + #include #include -#include "Dict.h" -#include "CompHash.h" -#include "IP.h" -#include "Frag.h" -#include "PacketFilter.h" -#include "Stats.h" -#include "NetVar.h" -#include "TunnelEncapsulation.h" -#include "analyzer/protocol/tcp/Stats.h" +#include // for u_char class EncapsulationStack; +class EncapsulatingConn; +class Packet; +class PacketProfiler; class Connection; class ConnCompressor; struct ConnID; class Discarder; -class PacketFilter; namespace analyzer { namespace stepping_stone { class SteppingStoneManager; } } namespace analyzer { namespace arp { class ARP_Analyzer; } } diff --git a/src/SmithWaterman.cc b/src/SmithWaterman.cc index 857e45bb9b..a71b15a27f 100644 --- a/src/SmithWaterman.cc +++ b/src/SmithWaterman.cc @@ -9,6 +9,7 @@ #include "Var.h" #include "util.h" #include "Reporter.h" +#include "Val.h" BroSubstring::BroSubstring(const BroSubstring& bst) : BroString((const BroString&) bst), _num(), _new(bst._new) diff --git a/src/Stats.cc b/src/Stats.cc index cbc58c197e..39e6148fbe 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -1,15 +1,19 @@ +#include "Stats.h" +#include "RuleMatcher.h" #include "Conn.h" #include "File.h" #include "Event.h" +#include "Net.h" #include "NetVar.h" +#include "Var.h" // for internal_type() #include "Sessions.h" -#include "Stats.h" #include "Scope.h" #include "cq.h" #include "DNS_Mgr.h" #include "Trigger.h" #include "threading/Manager.h" #include "broker/Manager.h" +#include "input.h" uint64_t killed_by_inactivity = 0; diff --git a/src/Stats.h b/src/Stats.h index a01997136f..0e8dea52e5 100644 --- a/src/Stats.h +++ b/src/Stats.h @@ -5,6 +5,12 @@ #include #include #include +#include + +class Func; +class TableVal; +class Location; +class BroFile; // Object called by SegmentProfiler when it is done and reports its // cumulative CPU/memory statistics. diff --git a/src/Stmt.cc b/src/Stmt.cc index 945bd80faf..6b98b0322e 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -11,6 +11,7 @@ #include "Stmt.h" #include "Scope.h" #include "Var.h" +#include "Desc.h" #include "Debug.h" #include "Traverse.h" #include "Trigger.h" diff --git a/src/Stmt.h b/src/Stmt.h index f8e14386f3..a0ac3a4543 100644 --- a/src/Stmt.h +++ b/src/Stmt.h @@ -5,8 +5,9 @@ // BRO statements. #include "BroList.h" +#include "Dict.h" +#include "ID.h" #include "Obj.h" -#include "Expr.h" #include "Reporter.h" #include "StmtEnums.h" @@ -14,7 +15,11 @@ #include "TraverseTypes.h" class StmtList; +class CompositeHash; +class EventExpr; +class ListExpr; class ForStmt; +class Frame; class Stmt : public BroObj { public: diff --git a/src/Tag.h b/src/Tag.h index b62c6ce3d1..a1c24702c0 100644 --- a/src/Tag.h +++ b/src/Tag.h @@ -3,10 +3,13 @@ #pragma once #include "zeek-config.h" -#include "util.h" -#include "Type.h" + +#include + +#include class EnumVal; +class EnumType; /** * Class to identify an analyzer type. diff --git a/src/Timer.h b/src/Timer.h index 75df60a47f..5148248d20 100644 --- a/src/Timer.h +++ b/src/Timer.h @@ -2,10 +2,11 @@ #pragma once -#include #include "PriorityQueue.h" #include "iosource/IOSource.h" +#include + // If you add a timer here, adjust TimerNames in Timer.cc. enum TimerType : uint8_t { TIMER_BACKDOOR, diff --git a/src/Traverse.cc b/src/Traverse.cc index d19c6d3801..b9aa28b42d 100644 --- a/src/Traverse.cc +++ b/src/Traverse.cc @@ -2,6 +2,7 @@ #include "Scope.h" #include "Traverse.h" +#include "Stmt.h" #include "input.h" TraversalCode traverse_all(TraversalCallback* cb) diff --git a/src/Traverse.h b/src/Traverse.h index 834611b0d9..2ef672ddd7 100644 --- a/src/Traverse.h +++ b/src/Traverse.h @@ -2,14 +2,14 @@ #pragma once -#include "Obj.h" -#include "Stmt.h" -#include "Expr.h" -#include "ID.h" -#include "Scope.h" - #include "TraverseTypes.h" +class Func; +class Scope; +class Stmt; +class Expr; +class ID; + class TraversalCallback { public: TraversalCallback() { current_scope = 0; } diff --git a/src/Trigger.cc b/src/Trigger.cc index 271edf24aa..357bd5836f 100644 --- a/src/Trigger.cc +++ b/src/Trigger.cc @@ -1,7 +1,18 @@ +#include "Trigger.h" + #include -#include "Trigger.h" +#include + #include "Traverse.h" +#include "Expr.h" +#include "Frame.h" +#include "ID.h" +#include "Val.h" +#include "Stmt.h" +#include "Reporter.h" +#include "Desc.h" +#include "DebugLogger.h" #include "iosource/Manager.h" using namespace trigger; diff --git a/src/Trigger.h b/src/Trigger.h index c3efd62138..9296449778 100644 --- a/src/Trigger.h +++ b/src/Trigger.h @@ -1,16 +1,27 @@ #pragma once +#include "Obj.h" +#include "Desc.h" +#include "Notifier.h" +#include "iosource/IOSource.h" + #include +#include #include -#include "Notifier.h" -#include "Traverse.h" -#include "iosource/IOSource.h" +class CallExpr; +class Expr; +class Stmt; +class Frame; +class Val; +class ID; namespace trigger { // Triggers are the heart of "when" statements: expressions that when // they become true execute a body of statements. +using std::map; + class TriggerTimer; class TriggerTraversalCallback; diff --git a/src/TunnelEncapsulation.h b/src/TunnelEncapsulation.h index ca762eb704..4f18b6118b 100644 --- a/src/TunnelEncapsulation.h +++ b/src/TunnelEncapsulation.h @@ -5,10 +5,12 @@ #include "zeek-config.h" #include "NetVar.h" #include "IPAddr.h" -#include "Val.h" +#include "Var.h" // for internal_type() #include "UID.h" + #include +using std::vector; class Connection; /** diff --git a/src/Type.cc b/src/Type.cc index 6a4139bc8b..501a27ad8d 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -4,11 +4,17 @@ #include "Type.h" #include "Attr.h" +#include "Desc.h" #include "Expr.h" #include "Scope.h" +#include "Val.h" +#include "Var.h" #include "Reporter.h" #include "zeekygen/Manager.h" +#include "zeekygen/IdentifierInfo.h" +#include "zeekygen/ScriptInfo.h" #include "zeekygen/utils.h" +#include "module_util.h" #include #include diff --git a/src/Type.h b/src/Type.h index 371e4385a5..02c4fbb592 100644 --- a/src/Type.h +++ b/src/Type.h @@ -2,17 +2,16 @@ #pragma once +#include "Obj.h" +#include "Attr.h" +#include "BroList.h" + #include #include #include #include #include -#include "Obj.h" -#include "Attr.h" -#include "BroList.h" -#include "Dict.h" - // BRO types. typedef enum { diff --git a/src/UID.cc b/src/UID.cc index 378a50b873..cff534edfb 100644 --- a/src/UID.cc +++ b/src/UID.cc @@ -1,9 +1,9 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include - #include "UID.h" +#include + using namespace Bro; using namespace std; diff --git a/src/UID.h b/src/UID.h index 9f1cd75c8d..2fa39badc2 100644 --- a/src/UID.h +++ b/src/UID.h @@ -2,10 +2,12 @@ #pragma once +#include "Reporter.h" +#include "util.h" // for bro_int_t + #include -#include "Reporter.h" -#include "util.h" +#include #define BRO_UID_LEN 2 diff --git a/src/Val.cc b/src/Val.cc index 2c690c590e..1d2d0588a2 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -1,6 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek-config.h" +#include "Val.h" #include #include @@ -12,10 +13,11 @@ #include #include -#include "Val.h" #include "Net.h" #include "File.h" #include "Func.h" +#include "Desc.h" +#include "IntrusivePtr.h" #include "RE.h" #include "Scope.h" #include "NetVar.h" diff --git a/src/Val.h b/src/Val.h index cda2dd43f8..c0cd79e126 100644 --- a/src/Val.h +++ b/src/Val.h @@ -2,12 +2,6 @@ #pragma once -#include -#include -#include -#include - -#include "net_util.h" #include "Type.h" #include "Dict.h" #include "CompHash.h" @@ -17,9 +11,18 @@ #include "ID.h" #include "Scope.h" #include "Notifier.h" -#include "IPAddr.h" -#include "DebugLogger.h" #include "RE.h" +#include "net_util.h" + +#include +#include +#include +#include + +#include // for u_char + +using std::vector; +using std::string; // We have four different port name spaces: TCP, UDP, ICMP, and UNKNOWN. // We distinguish between them based on the bits specified in the *_PORT_MASK @@ -50,6 +53,9 @@ class StringVal; class EnumVal; class OpaqueVal; +class IPAddr; +class IPPrefix; + class StateAccess; class VectorVal; diff --git a/src/Var.cc b/src/Var.cc index 212ee58a54..971a09cfa9 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -1,15 +1,19 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include "zeek-config.h" +#include "Var.h" + #include -#include "zeek-config.h" - -#include "Var.h" +#include "Val.h" +#include "Expr.h" #include "Func.h" #include "Stmt.h" #include "Scope.h" +#include "Reporter.h" #include "EventRegistry.h" #include "Traverse.h" +#include "module_util.h" static Val* init_val(Expr* init, const BroType* t, Val* aggr) { diff --git a/src/Var.h b/src/Var.h index 5ae6720b62..0282c4a489 100644 --- a/src/Var.h +++ b/src/Var.h @@ -2,15 +2,17 @@ #pragma once -#include // std::unique_ptr - #include "ID.h" -#include "Expr.h" #include "Type.h" -#include "Func.h" // function_ingredients -class Func; +class Expr; +class FuncType; +class Stmt; +class Scope; class EventHandlerPtr; +class StringVal; +class TableVal; +class ListVal; typedef enum { VAR_REGULAR, VAR_CONST, VAR_REDEF, VAR_OPTION, } decl_type; diff --git a/src/WeirdState.cc b/src/WeirdState.cc index 1f1407a1d2..48a74e32cf 100644 --- a/src/WeirdState.cc +++ b/src/WeirdState.cc @@ -1,5 +1,6 @@ #include "WeirdState.h" #include "Net.h" +#include "util.h" bool PermitWeird(WeirdStateMap& wsm, const char* name, uint64_t threshold, uint64_t rate, double duration) diff --git a/src/analyzer/Analyzer.h b/src/analyzer/Analyzer.h index a896563abe..2fcde63187 100644 --- a/src/analyzer/Analyzer.h +++ b/src/analyzer/Analyzer.h @@ -2,15 +2,21 @@ #pragma once -#include -#include - #include "Tag.h" #include "../Obj.h" #include "../EventHandler.h" #include "../Timer.h" +#include +#include + +#include // for u_char + +using std::list; +using std::string; + +class BroFile; class Rule; class Connection; class IP_Hdr; diff --git a/src/analyzer/Tag.h b/src/analyzer/Tag.h index b36d10863b..46545e4bd1 100644 --- a/src/analyzer/Tag.h +++ b/src/analyzer/Tag.h @@ -3,13 +3,17 @@ #pragma once #include "zeek-config.h" -#include "util.h" #include "../Tag.h" -#include "plugin/TaggedComponent.h" -#include "plugin/ComponentManager.h" class EnumVal; +namespace plugin { +template +class TaggedComponent; +template +class ComponentManager; +} + namespace analyzer { class Manager; diff --git a/src/analyzer/protocol/arp/ARP.cc b/src/analyzer/protocol/arp/ARP.cc index fe56a446e0..360a24d5bd 100644 --- a/src/analyzer/protocol/arp/ARP.cc +++ b/src/analyzer/protocol/arp/ARP.cc @@ -3,6 +3,7 @@ #include "ARP.h" #include "Event.h" #include "Reporter.h" +#include "Desc.h" #include "events.bif.h" diff --git a/src/analyzer/protocol/ayiya/Plugin.cc b/src/analyzer/protocol/ayiya/Plugin.cc index 2b4b8ee7d9..23c04543e3 100644 --- a/src/analyzer/protocol/ayiya/Plugin.cc +++ b/src/analyzer/protocol/ayiya/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "AYIYA.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_AYIYA { diff --git a/src/analyzer/protocol/ayiya/ayiya-analyzer.pac b/src/analyzer/protocol/ayiya/ayiya-analyzer.pac index 1d8cbe90b6..67a74f4444 100644 --- a/src/analyzer/protocol/ayiya/ayiya-analyzer.pac +++ b/src/analyzer/protocol/ayiya/ayiya-analyzer.pac @@ -1,3 +1,6 @@ +%extern{ +#include "Sessions.h" +%} connection AYIYA_Conn(bro_analyzer: BroAnalyzer) { diff --git a/src/analyzer/protocol/ayiya/ayiya.pac b/src/analyzer/protocol/ayiya/ayiya.pac index ff0af4d47c..ad4b7582a8 100644 --- a/src/analyzer/protocol/ayiya/ayiya.pac +++ b/src/analyzer/protocol/ayiya/ayiya.pac @@ -2,6 +2,12 @@ %include binpac.pac %include bro.pac +%extern{ +#include "IP.h" +#include "Reporter.h" +#include "TunnelEncapsulation.h" +%} + analyzer AYIYA withcontext { connection: AYIYA_Conn; flow: AYIYA_Flow; diff --git a/src/analyzer/protocol/bittorrent/BitTorrentTracker.h b/src/analyzer/protocol/bittorrent/BitTorrentTracker.h index d86216cb18..0663901229 100644 --- a/src/analyzer/protocol/bittorrent/BitTorrentTracker.h +++ b/src/analyzer/protocol/bittorrent/BitTorrentTracker.h @@ -6,6 +6,8 @@ #define BTTRACKER_BUF 2048 +class StringVal; + namespace analyzer { namespace bittorrent { // If the following is defined, then the analyzer will store all of diff --git a/src/analyzer/protocol/bittorrent/Plugin.cc b/src/analyzer/protocol/bittorrent/Plugin.cc index 14f778ac9f..c9d94aa16f 100644 --- a/src/analyzer/protocol/bittorrent/Plugin.cc +++ b/src/analyzer/protocol/bittorrent/Plugin.cc @@ -1,10 +1,9 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "BitTorrent.h" #include "BitTorrentTracker.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_BitTorrent { diff --git a/src/analyzer/protocol/conn-size/ConnSize.cc b/src/analyzer/protocol/conn-size/ConnSize.cc index 4b0b542ac8..58e8ccb857 100644 --- a/src/analyzer/protocol/conn-size/ConnSize.cc +++ b/src/analyzer/protocol/conn-size/ConnSize.cc @@ -5,6 +5,8 @@ #include "ConnSize.h" #include "analyzer/protocol/tcp/TCP.h" +#include "IP.h" +#include "Reporter.h" #include "events.bif.h" diff --git a/src/analyzer/protocol/conn-size/Plugin.cc b/src/analyzer/protocol/conn-size/Plugin.cc index ce1b600da2..1ba5225278 100644 --- a/src/analyzer/protocol/conn-size/Plugin.cc +++ b/src/analyzer/protocol/conn-size/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "ConnSize.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_ConnSize { diff --git a/src/analyzer/protocol/conn-size/functions.bif b/src/analyzer/protocol/conn-size/functions.bif index 74d153363e..24b51c895e 100644 --- a/src/analyzer/protocol/conn-size/functions.bif +++ b/src/analyzer/protocol/conn-size/functions.bif @@ -1,5 +1,7 @@ %%{ #include "analyzer/protocol/conn-size/ConnSize.h" +#include "Reporter.h" +#include "Sessions.h" static analyzer::Analyzer* GetConnsizeAnalyzer(Val* cid) { diff --git a/src/analyzer/protocol/dhcp/Plugin.cc b/src/analyzer/protocol/dhcp/Plugin.cc index 62318604c4..f4895255ed 100644 --- a/src/analyzer/protocol/dhcp/Plugin.cc +++ b/src/analyzer/protocol/dhcp/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "DHCP.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_DHCP { diff --git a/src/analyzer/protocol/dnp3/DNP3.cc b/src/analyzer/protocol/dnp3/DNP3.cc index 1812f21aba..8d22e32e2f 100644 --- a/src/analyzer/protocol/dnp3/DNP3.cc +++ b/src/analyzer/protocol/dnp3/DNP3.cc @@ -97,6 +97,7 @@ // Binpac DNP3 Analyzer #include "DNP3.h" +#include "Reporter.h" #include "events.bif.h" using namespace analyzer::dnp3; diff --git a/src/analyzer/protocol/dnp3/Plugin.cc b/src/analyzer/protocol/dnp3/Plugin.cc index 8543360b6a..e75e19f4c5 100644 --- a/src/analyzer/protocol/dnp3/Plugin.cc +++ b/src/analyzer/protocol/dnp3/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "DNP3.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_DNP3 { diff --git a/src/analyzer/protocol/dns/DNS.cc b/src/analyzer/protocol/dns/DNS.cc index eab5b48461..a4868056f6 100644 --- a/src/analyzer/protocol/dns/DNS.cc +++ b/src/analyzer/protocol/dns/DNS.cc @@ -1,6 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek-config.h" +#include "DNS.h" #include #include @@ -9,9 +10,9 @@ #include #include "NetVar.h" -#include "DNS.h" #include "Sessions.h" #include "Event.h" +#include "Net.h" #include "events.bif.h" diff --git a/src/analyzer/protocol/dns/Plugin.cc b/src/analyzer/protocol/dns/Plugin.cc index 3ceef34ea1..d360a1771c 100644 --- a/src/analyzer/protocol/dns/Plugin.cc +++ b/src/analyzer/protocol/dns/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "DNS.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_DNS { diff --git a/src/analyzer/protocol/file/Plugin.cc b/src/analyzer/protocol/file/Plugin.cc index 36586fb6a9..974c5949a3 100644 --- a/src/analyzer/protocol/file/Plugin.cc +++ b/src/analyzer/protocol/file/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - +#include "File.h" #include "plugin/Plugin.h" - -#include "./File.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_File { diff --git a/src/analyzer/protocol/finger/Plugin.cc b/src/analyzer/protocol/finger/Plugin.cc index b6fafd3b4c..825860b49e 100644 --- a/src/analyzer/protocol/finger/Plugin.cc +++ b/src/analyzer/protocol/finger/Plugin.cc @@ -1,8 +1,8 @@ // See the file in the main distribution directory for copyright. -#include "plugin/Plugin.h" - #include "Finger.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_Finger { diff --git a/src/analyzer/protocol/ftp/FTP.cc b/src/analyzer/protocol/ftp/FTP.cc index da86ad7db4..26bb780caa 100644 --- a/src/analyzer/protocol/ftp/FTP.cc +++ b/src/analyzer/protocol/ftp/FTP.cc @@ -1,15 +1,16 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek-config.h" +#include "FTP.h" #include #include "NetVar.h" -#include "FTP.h" #include "Event.h" #include "Base64.h" #include "analyzer/Manager.h" #include "analyzer/protocol/login/NVT.h" +#include "RuleMatcher.h" #include "events.bif.h" diff --git a/src/analyzer/protocol/ftp/FTP.h b/src/analyzer/protocol/ftp/FTP.h index 6bf0edd46a..47d93e8f10 100644 --- a/src/analyzer/protocol/ftp/FTP.h +++ b/src/analyzer/protocol/ftp/FTP.h @@ -2,9 +2,10 @@ #pragma once -#include "analyzer/protocol/login/NVT.h" #include "analyzer/protocol/tcp/TCP.h" +namespace analyzer { namespace login { class NVT_Analyzer; }} + namespace analyzer { namespace ftp { class FTP_Analyzer : public tcp::TCP_ApplicationAnalyzer { diff --git a/src/analyzer/protocol/ftp/Plugin.cc b/src/analyzer/protocol/ftp/Plugin.cc index ae70d2f705..1cade7807d 100644 --- a/src/analyzer/protocol/ftp/Plugin.cc +++ b/src/analyzer/protocol/ftp/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "FTP.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_FTP { diff --git a/src/analyzer/protocol/ftp/functions.bif b/src/analyzer/protocol/ftp/functions.bif index facabeb679..7f2b6cf643 100644 --- a/src/analyzer/protocol/ftp/functions.bif +++ b/src/analyzer/protocol/ftp/functions.bif @@ -2,6 +2,7 @@ type ftp_port: record; %%{ +#include "Reporter.h" static Val* parse_port(const char* line) { diff --git a/src/analyzer/protocol/gnutella/Plugin.cc b/src/analyzer/protocol/gnutella/Plugin.cc index b6a560ec58..93031060ee 100644 --- a/src/analyzer/protocol/gnutella/Plugin.cc +++ b/src/analyzer/protocol/gnutella/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "Gnutella.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_Gnutella { diff --git a/src/analyzer/protocol/gtpv1/Plugin.cc b/src/analyzer/protocol/gtpv1/Plugin.cc index 4b7929a747..6ea6230e16 100644 --- a/src/analyzer/protocol/gtpv1/Plugin.cc +++ b/src/analyzer/protocol/gtpv1/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "GTPv1.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_GTPv1 { diff --git a/src/analyzer/protocol/gtpv1/gtpv1-analyzer.pac b/src/analyzer/protocol/gtpv1/gtpv1-analyzer.pac index 6cf9439363..b693f4e792 100644 --- a/src/analyzer/protocol/gtpv1/gtpv1-analyzer.pac +++ b/src/analyzer/protocol/gtpv1/gtpv1-analyzer.pac @@ -1,3 +1,6 @@ +%extern{ +#include "Sessions.h" +%} %code{ RecordVal* BuildGTPv1Hdr(const GTPv1_Header* pdu) diff --git a/src/analyzer/protocol/gtpv1/gtpv1.pac b/src/analyzer/protocol/gtpv1/gtpv1.pac index 0305951cc5..6223df22d6 100644 --- a/src/analyzer/protocol/gtpv1/gtpv1.pac +++ b/src/analyzer/protocol/gtpv1/gtpv1.pac @@ -2,6 +2,9 @@ %include bro.pac %extern{ +#include "IP.h" +#include "TunnelEncapsulation.h" +#include "Reporter.h" #include "events.bif.h" %} diff --git a/src/analyzer/protocol/http/Plugin.cc b/src/analyzer/protocol/http/Plugin.cc index f2b7402415..fcd8f29444 100644 --- a/src/analyzer/protocol/http/Plugin.cc +++ b/src/analyzer/protocol/http/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "HTTP.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_HTTP { diff --git a/src/analyzer/protocol/icmp/ICMP.cc b/src/analyzer/protocol/icmp/ICMP.cc index 433284ac7b..145c00dfee 100644 --- a/src/analyzer/protocol/icmp/ICMP.cc +++ b/src/analyzer/protocol/icmp/ICMP.cc @@ -1,14 +1,18 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include "ICMP.h" + #include #include "zeek-config.h" +#include "IP.h" #include "Net.h" #include "NetVar.h" #include "Event.h" -#include "ICMP.h" #include "Conn.h" +#include "Desc.h" +#include "Reporter.h" #include "events.bif.h" diff --git a/src/analyzer/protocol/icmp/ICMP.h b/src/analyzer/protocol/icmp/ICMP.h index 042052266f..7106f53dd9 100644 --- a/src/analyzer/protocol/icmp/ICMP.h +++ b/src/analyzer/protocol/icmp/ICMP.h @@ -4,6 +4,9 @@ #include "RuleMatcher.h" #include "analyzer/Analyzer.h" +#include "net_util.h" + +class VectorVal; namespace analyzer { namespace icmp { diff --git a/src/analyzer/protocol/icmp/Plugin.cc b/src/analyzer/protocol/icmp/Plugin.cc index 390eb751d1..2aef006315 100644 --- a/src/analyzer/protocol/icmp/Plugin.cc +++ b/src/analyzer/protocol/icmp/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "ICMP.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_ICMP { diff --git a/src/analyzer/protocol/ident/Plugin.cc b/src/analyzer/protocol/ident/Plugin.cc index 23a798a72f..5398ba0523 100644 --- a/src/analyzer/protocol/ident/Plugin.cc +++ b/src/analyzer/protocol/ident/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "Ident.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_Ident { diff --git a/src/analyzer/protocol/imap/Plugin.cc b/src/analyzer/protocol/imap/Plugin.cc index 3192ea8f28..bbdac9b444 100644 --- a/src/analyzer/protocol/imap/Plugin.cc +++ b/src/analyzer/protocol/imap/Plugin.cc @@ -1,6 +1,8 @@ // See the file in the main distribution directory for copyright. -#include "plugin/Plugin.h" + #include "IMAP.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_IMAP { diff --git a/src/analyzer/protocol/imap/imap.pac b/src/analyzer/protocol/imap/imap.pac index f5c7559294..4f16af8523 100644 --- a/src/analyzer/protocol/imap/imap.pac +++ b/src/analyzer/protocol/imap/imap.pac @@ -7,6 +7,7 @@ %include bro.pac %extern{ +#include "Reporter.h" #include "events.bif.h" namespace analyzer { namespace imap { class IMAP_Analyzer; } } diff --git a/src/analyzer/protocol/irc/Plugin.cc b/src/analyzer/protocol/irc/Plugin.cc index fc63baad12..b4d7266472 100644 --- a/src/analyzer/protocol/irc/Plugin.cc +++ b/src/analyzer/protocol/irc/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "IRC.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_IRC { diff --git a/src/analyzer/protocol/krb/KRB.cc b/src/analyzer/protocol/krb/KRB.cc index 65e764eca1..890947e5b7 100644 --- a/src/analyzer/protocol/krb/KRB.cc +++ b/src/analyzer/protocol/krb/KRB.cc @@ -1,8 +1,9 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include "KRB.h" + #include -#include "KRB.h" #include "types.bif.h" #include "events.bif.h" diff --git a/src/analyzer/protocol/krb/Plugin.cc b/src/analyzer/protocol/krb/Plugin.cc index 707498f729..567fd080c8 100644 --- a/src/analyzer/protocol/krb/Plugin.cc +++ b/src/analyzer/protocol/krb/Plugin.cc @@ -1,8 +1,9 @@ //See the file in the main distribution directory for copyright. -#include "plugin/Plugin.h" #include "KRB.h" #include "KRB_TCP.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_KRB { diff --git a/src/analyzer/protocol/krb/krb-padata.pac b/src/analyzer/protocol/krb/krb-padata.pac index feb1089815..66025afda4 100644 --- a/src/analyzer/protocol/krb/krb-padata.pac +++ b/src/analyzer/protocol/krb/krb-padata.pac @@ -3,6 +3,7 @@ %extern{ #include "file_analysis/Manager.h" +#include "Desc.h" %} %header{ diff --git a/src/analyzer/protocol/login/Login.cc b/src/analyzer/protocol/login/Login.cc index 277bb752ff..5bca5930c6 100644 --- a/src/analyzer/protocol/login/Login.cc +++ b/src/analyzer/protocol/login/Login.cc @@ -1,13 +1,14 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek-config.h" +#include "Login.h" #include #include #include "NetVar.h" -#include "Login.h" #include "RE.h" +#include "Reporter.h" #include "Event.h" #include "events.bif.h" diff --git a/src/analyzer/protocol/login/NVT.cc b/src/analyzer/protocol/login/NVT.cc index 3cc9bdb1e0..0105df59f0 100644 --- a/src/analyzer/protocol/login/NVT.cc +++ b/src/analyzer/protocol/login/NVT.cc @@ -1,12 +1,13 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek-config.h" +#include "NVT.h" #include -#include "NVT.h" #include "NetVar.h" #include "Event.h" +#include "Reporter.h" #include "analyzer/protocol/tcp/TCP.h" #include "events.bif.h" diff --git a/src/analyzer/protocol/login/Plugin.cc b/src/analyzer/protocol/login/Plugin.cc index 182c070592..86cac17a14 100644 --- a/src/analyzer/protocol/login/Plugin.cc +++ b/src/analyzer/protocol/login/Plugin.cc @@ -1,12 +1,11 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "Login.h" #include "Telnet.h" #include "RSH.h" #include "Rlogin.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_Login { diff --git a/src/analyzer/protocol/login/RSH.cc b/src/analyzer/protocol/login/RSH.cc index 7fc5f932e0..ef9fb3fa18 100644 --- a/src/analyzer/protocol/login/RSH.cc +++ b/src/analyzer/protocol/login/RSH.cc @@ -2,9 +2,10 @@ #include "zeek-config.h" +#include "RSH.h" #include "NetVar.h" #include "Event.h" -#include "RSH.h" +#include "Reporter.h" #include "events.bif.h" diff --git a/src/analyzer/protocol/login/Rlogin.cc b/src/analyzer/protocol/login/Rlogin.cc index 62b391849b..bed3c0317d 100644 --- a/src/analyzer/protocol/login/Rlogin.cc +++ b/src/analyzer/protocol/login/Rlogin.cc @@ -2,9 +2,10 @@ #include "zeek-config.h" +#include "Rlogin.h" #include "NetVar.h" #include "Event.h" -#include "Rlogin.h" +#include "Reporter.h" #include "events.bif.h" diff --git a/src/analyzer/protocol/login/functions.bif b/src/analyzer/protocol/login/functions.bif index 932020595c..2872a5aa02 100644 --- a/src/analyzer/protocol/login/functions.bif +++ b/src/analyzer/protocol/login/functions.bif @@ -1,6 +1,8 @@ %%{ #include "Login.h" +#include "Reporter.h" +#include "Sessions.h" %%} ## Returns the state of the given login (Telnet or Rlogin) connection. diff --git a/src/analyzer/protocol/mime/MIME.cc b/src/analyzer/protocol/mime/MIME.cc index b62dc11633..e13688d81b 100644 --- a/src/analyzer/protocol/mime/MIME.cc +++ b/src/analyzer/protocol/mime/MIME.cc @@ -1,8 +1,8 @@ #include "zeek-config.h" -#include "NetVar.h" #include "MIME.h" -#include "Event.h" +#include "NetVar.h" +#include "Base64.h" #include "Reporter.h" #include "digest.h" #include "file_analysis/Manager.h" diff --git a/src/analyzer/protocol/mime/MIME.h b/src/analyzer/protocol/mime/MIME.h index 0a5b05283b..33acab12be 100644 --- a/src/analyzer/protocol/mime/MIME.h +++ b/src/analyzer/protocol/mime/MIME.h @@ -7,10 +7,13 @@ #include using namespace std; -#include "Base64.h" #include "BroString.h" +#include "Reporter.h" #include "analyzer/Analyzer.h" +class StringVal; +class Base64Converter; + namespace analyzer { namespace mime { // MIME: Multipurpose Internet Mail Extensions diff --git a/src/analyzer/protocol/modbus/Plugin.cc b/src/analyzer/protocol/modbus/Plugin.cc index 68b78fcbe7..ee89d8efc1 100644 --- a/src/analyzer/protocol/modbus/Plugin.cc +++ b/src/analyzer/protocol/modbus/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "Modbus.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_Modbus { diff --git a/src/analyzer/protocol/mqtt/Plugin.cc b/src/analyzer/protocol/mqtt/Plugin.cc index 6ebcbb89ba..8640fde129 100644 --- a/src/analyzer/protocol/mqtt/Plugin.cc +++ b/src/analyzer/protocol/mqtt/Plugin.cc @@ -1,8 +1,8 @@ // See the file in the main distribution directory for copyright. -#include "plugin/Plugin.h" - #include "MQTT.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_MQTT { diff --git a/src/analyzer/protocol/mysql/Plugin.cc b/src/analyzer/protocol/mysql/Plugin.cc index 0f484e29ce..e4f7ec4549 100644 --- a/src/analyzer/protocol/mysql/Plugin.cc +++ b/src/analyzer/protocol/mysql/Plugin.cc @@ -1,8 +1,8 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include "plugin/Plugin.h" - #include "MySQL.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_MySQL { diff --git a/src/analyzer/protocol/ncp/Plugin.cc b/src/analyzer/protocol/ncp/Plugin.cc index 9ea75a4674..5e54f977bb 100644 --- a/src/analyzer/protocol/ncp/Plugin.cc +++ b/src/analyzer/protocol/ncp/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "NCP.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_NCP { diff --git a/src/analyzer/protocol/netbios/NetbiosSSN.cc b/src/analyzer/protocol/netbios/NetbiosSSN.cc index 5d5ef06080..4aedd0d0f0 100644 --- a/src/analyzer/protocol/netbios/NetbiosSSN.cc +++ b/src/analyzer/protocol/netbios/NetbiosSSN.cc @@ -1,13 +1,14 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek-config.h" +#include "NetbiosSSN.h" #include #include "NetVar.h" -#include "NetbiosSSN.h" #include "Sessions.h" #include "Event.h" +#include "Net.h" #include "events.bif.h" diff --git a/src/analyzer/protocol/netbios/Plugin.cc b/src/analyzer/protocol/netbios/Plugin.cc index 7f49cdfb09..3024c48acd 100644 --- a/src/analyzer/protocol/netbios/Plugin.cc +++ b/src/analyzer/protocol/netbios/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "NetbiosSSN.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_NetBIOS { diff --git a/src/analyzer/protocol/netbios/functions.bif b/src/analyzer/protocol/netbios/functions.bif index c86156931f..37a13a7678 100644 --- a/src/analyzer/protocol/netbios/functions.bif +++ b/src/analyzer/protocol/netbios/functions.bif @@ -1,3 +1,6 @@ +%%{ +#include "Reporter.h" +%%} ## Decode a NetBIOS name. See http://support.microsoft.com/kb/194203. ## diff --git a/src/analyzer/protocol/ntp/Plugin.cc b/src/analyzer/protocol/ntp/Plugin.cc index edb2b8c3d7..30210731b2 100644 --- a/src/analyzer/protocol/ntp/Plugin.cc +++ b/src/analyzer/protocol/ntp/Plugin.cc @@ -1,8 +1,8 @@ // See the file in the main distribution directory for copyright. -#include "plugin/Plugin.h" - #include "NTP.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_NTP { diff --git a/src/analyzer/protocol/pia/PIA.cc b/src/analyzer/protocol/pia/PIA.cc index 0d8f382dd9..0c3a1cfbcd 100644 --- a/src/analyzer/protocol/pia/PIA.cc +++ b/src/analyzer/protocol/pia/PIA.cc @@ -1,6 +1,10 @@ #include "PIA.h" #include "RuleMatcher.h" #include "Event.h" +#include "NetVar.h" +#include "IP.h" +#include "DebugLogger.h" +#include "Reporter.h" #include "analyzer/protocol/tcp/TCP_Flags.h" #include "analyzer/protocol/tcp/TCP_Reassembler.h" diff --git a/src/analyzer/protocol/pia/PIA.h b/src/analyzer/protocol/pia/PIA.h index c0df3c32f2..467d353464 100644 --- a/src/analyzer/protocol/pia/PIA.h +++ b/src/analyzer/protocol/pia/PIA.h @@ -4,6 +4,7 @@ #include "analyzer/Analyzer.h" #include "analyzer/protocol/tcp/TCP.h" +#include "RuleMatcher.h" class RuleEndpointState; diff --git a/src/analyzer/protocol/pia/Plugin.cc b/src/analyzer/protocol/pia/Plugin.cc index c46e710f9d..617623d403 100644 --- a/src/analyzer/protocol/pia/Plugin.cc +++ b/src/analyzer/protocol/pia/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "PIA.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_PIA { diff --git a/src/analyzer/protocol/pop3/POP3.cc b/src/analyzer/protocol/pop3/POP3.cc index 50ab3142dc..d0c5d0b3bd 100644 --- a/src/analyzer/protocol/pop3/POP3.cc +++ b/src/analyzer/protocol/pop3/POP3.cc @@ -2,19 +2,15 @@ // Robin Sommer. #include "zeek-config.h" +#include "POP3.h" -#include -#include #include #include #include -#include "NetVar.h" -#include "POP3.h" -#include "Event.h" +#include "Base64.h" #include "Reporter.h" #include "analyzer/Manager.h" -#include "analyzer/protocol/login/NVT.h" #include "events.bif.h" diff --git a/src/analyzer/protocol/pop3/Plugin.cc b/src/analyzer/protocol/pop3/Plugin.cc index 0fed697e83..25311c928d 100644 --- a/src/analyzer/protocol/pop3/Plugin.cc +++ b/src/analyzer/protocol/pop3/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "POP3.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_POP3 { diff --git a/src/analyzer/protocol/radius/Plugin.cc b/src/analyzer/protocol/radius/Plugin.cc index 8b6efe15b8..7ec57fe252 100644 --- a/src/analyzer/protocol/radius/Plugin.cc +++ b/src/analyzer/protocol/radius/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "RADIUS.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_RADIUS { diff --git a/src/analyzer/protocol/rdp/Plugin.cc b/src/analyzer/protocol/rdp/Plugin.cc index 169c7501d6..324fbdccc8 100644 --- a/src/analyzer/protocol/rdp/Plugin.cc +++ b/src/analyzer/protocol/rdp/Plugin.cc @@ -1,6 +1,6 @@ -#include "plugin/Plugin.h" - #include "RDP.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_RDP { diff --git a/src/analyzer/protocol/rdp/rdp-analyzer.pac b/src/analyzer/protocol/rdp/rdp-analyzer.pac index dd76d07a87..73f68f29d2 100644 --- a/src/analyzer/protocol/rdp/rdp-analyzer.pac +++ b/src/analyzer/protocol/rdp/rdp-analyzer.pac @@ -1,4 +1,5 @@ %extern{ +#include "Desc.h" #include "file_analysis/Manager.h" #include "types.bif.h" %} diff --git a/src/analyzer/protocol/rfb/Plugin.cc b/src/analyzer/protocol/rfb/Plugin.cc index 8cf53bb007..a195136aec 100644 --- a/src/analyzer/protocol/rfb/Plugin.cc +++ b/src/analyzer/protocol/rfb/Plugin.cc @@ -1,6 +1,6 @@ -#include "plugin/Plugin.h" - #include "RFB.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_RFB { diff --git a/src/analyzer/protocol/rpc/MOUNT.cc b/src/analyzer/protocol/rpc/MOUNT.cc index 61117f2a51..b66bf34206 100644 --- a/src/analyzer/protocol/rpc/MOUNT.cc +++ b/src/analyzer/protocol/rpc/MOUNT.cc @@ -1,13 +1,13 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include "zeek-config.h" +#include "MOUNT.h" + #include #include -#include "zeek-config.h" - #include "NetVar.h" #include "XDR.h" -#include "MOUNT.h" #include "Event.h" #include "events.bif.h" diff --git a/src/analyzer/protocol/rpc/MOUNT.h b/src/analyzer/protocol/rpc/MOUNT.h index bb05bcd7cc..182922dfd4 100644 --- a/src/analyzer/protocol/rpc/MOUNT.h +++ b/src/analyzer/protocol/rpc/MOUNT.h @@ -3,8 +3,6 @@ #pragma once #include "RPC.h" -#include "XDR.h" -#include "Event.h" namespace analyzer { namespace rpc { diff --git a/src/analyzer/protocol/rpc/NFS.cc b/src/analyzer/protocol/rpc/NFS.cc index c2764a1229..bba3375ec6 100644 --- a/src/analyzer/protocol/rpc/NFS.cc +++ b/src/analyzer/protocol/rpc/NFS.cc @@ -1,13 +1,13 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include -#include - #include "zeek-config.h" +#include "NFS.h" + +#include +#include #include "NetVar.h" #include "XDR.h" -#include "NFS.h" #include "Event.h" #include "events.bif.h" diff --git a/src/analyzer/protocol/rpc/NFS.h b/src/analyzer/protocol/rpc/NFS.h index fecff7532b..8c4e259bd0 100644 --- a/src/analyzer/protocol/rpc/NFS.h +++ b/src/analyzer/protocol/rpc/NFS.h @@ -3,8 +3,7 @@ #pragma once #include "RPC.h" -#include "XDR.h" -#include "Event.h" +#include "NetVar.h" namespace analyzer { namespace rpc { diff --git a/src/analyzer/protocol/rpc/Plugin.cc b/src/analyzer/protocol/rpc/Plugin.cc index 2fff0ff6cf..1099553f0b 100644 --- a/src/analyzer/protocol/rpc/Plugin.cc +++ b/src/analyzer/protocol/rpc/Plugin.cc @@ -1,12 +1,11 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "RPC.h" #include "NFS.h" #include "MOUNT.h" #include "Portmap.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_RPC { diff --git a/src/analyzer/protocol/rpc/Portmap.cc b/src/analyzer/protocol/rpc/Portmap.cc index 7d80dd5f9b..d1dafc49c5 100644 --- a/src/analyzer/protocol/rpc/Portmap.cc +++ b/src/analyzer/protocol/rpc/Portmap.cc @@ -1,14 +1,14 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include "zeek-config.h" - +#include "Portmap.h" #include "NetVar.h" #include "XDR.h" -#include "Portmap.h" #include "Event.h" #include "events.bif.h" +#include "zeek-config.h" + using namespace analyzer::rpc; #define PMAPPROC_NULL 0 diff --git a/src/analyzer/protocol/rpc/RPC.cc b/src/analyzer/protocol/rpc/RPC.cc index 2820b5e033..9a87117435 100644 --- a/src/analyzer/protocol/rpc/RPC.cc +++ b/src/analyzer/protocol/rpc/RPC.cc @@ -1,18 +1,19 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include - -#include - #include "zeek-config.h" +#include "RPC.h" + +#include #include "NetVar.h" #include "XDR.h" -#include "RPC.h" +#include "Reporter.h" #include "Sessions.h" #include "events.bif.h" +#include + using namespace analyzer::rpc; namespace { // local namespace diff --git a/src/analyzer/protocol/rpc/RPC.h b/src/analyzer/protocol/rpc/RPC.h index 977272647b..b7a03ebac5 100644 --- a/src/analyzer/protocol/rpc/RPC.h +++ b/src/analyzer/protocol/rpc/RPC.h @@ -3,7 +3,7 @@ #pragma once #include "analyzer/protocol/tcp/TCP.h" -#include "analyzer/protocol/udp/UDP.h" +#include "NetVar.h" namespace analyzer { namespace rpc { diff --git a/src/analyzer/protocol/rpc/XDR.cc b/src/analyzer/protocol/rpc/XDR.cc index 18776eee5f..95992d142c 100644 --- a/src/analyzer/protocol/rpc/XDR.cc +++ b/src/analyzer/protocol/rpc/XDR.cc @@ -1,12 +1,11 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include "zeek-config.h" +#include "XDR.h" + #include #include -#include "zeek-config.h" - -#include "XDR.h" - #include "events.bif.h" using namespace analyzer::rpc; diff --git a/src/analyzer/protocol/sip/Plugin.cc b/src/analyzer/protocol/sip/Plugin.cc index 23ddebc12c..0f427c3a3a 100644 --- a/src/analyzer/protocol/sip/Plugin.cc +++ b/src/analyzer/protocol/sip/Plugin.cc @@ -1,10 +1,9 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "SIP.h" #include "SIP_TCP.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_SIP { diff --git a/src/analyzer/protocol/smtp/Plugin.cc b/src/analyzer/protocol/smtp/Plugin.cc index 784da4d860..4a1a70c71e 100644 --- a/src/analyzer/protocol/smtp/Plugin.cc +++ b/src/analyzer/protocol/smtp/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "SMTP.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_SMTP { diff --git a/src/analyzer/protocol/snmp/Plugin.cc b/src/analyzer/protocol/snmp/Plugin.cc index d5c6e98309..f5ce06cde0 100644 --- a/src/analyzer/protocol/snmp/Plugin.cc +++ b/src/analyzer/protocol/snmp/Plugin.cc @@ -1,8 +1,8 @@ // See the file in the main distribution directory for copyright. -#include "plugin/Plugin.h" - #include "SNMP.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_SNMP { diff --git a/src/analyzer/protocol/snmp/SNMP.cc b/src/analyzer/protocol/snmp/SNMP.cc index 2817bfec52..6e0988ae2b 100644 --- a/src/analyzer/protocol/snmp/SNMP.cc +++ b/src/analyzer/protocol/snmp/SNMP.cc @@ -2,6 +2,7 @@ #include "SNMP.h" #include "Func.h" +#include "Reporter.h" #include "types.bif.h" #include "events.bif.h" diff --git a/src/analyzer/protocol/snmp/snmp.pac b/src/analyzer/protocol/snmp/snmp.pac index 29b9d32e73..33bdecf24a 100644 --- a/src/analyzer/protocol/snmp/snmp.pac +++ b/src/analyzer/protocol/snmp/snmp.pac @@ -2,6 +2,7 @@ %include bro.pac %extern{ +#include "Reporter.h" #include "types.bif.h" #include "events.bif.h" %} diff --git a/src/analyzer/protocol/socks/Plugin.cc b/src/analyzer/protocol/socks/Plugin.cc index 8efbeeb23e..6e16935a26 100644 --- a/src/analyzer/protocol/socks/Plugin.cc +++ b/src/analyzer/protocol/socks/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "SOCKS.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_SOCKS { diff --git a/src/analyzer/protocol/socks/socks.pac b/src/analyzer/protocol/socks/socks.pac index 9aed2820af..a6c4ad3605 100644 --- a/src/analyzer/protocol/socks/socks.pac +++ b/src/analyzer/protocol/socks/socks.pac @@ -3,6 +3,7 @@ %extern{ #include "SOCKS.h" +#include "Reporter.h" #include "events.bif.h" %} diff --git a/src/analyzer/protocol/ssh/Plugin.cc b/src/analyzer/protocol/ssh/Plugin.cc index 7b6ac67c88..641bad8ab0 100644 --- a/src/analyzer/protocol/ssh/Plugin.cc +++ b/src/analyzer/protocol/ssh/Plugin.cc @@ -1,7 +1,8 @@ // See the file in the main distribution directory for copyright. -#include "plugin/Plugin.h" #include "SSH.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_SSH { diff --git a/src/analyzer/protocol/ssl/Plugin.cc b/src/analyzer/protocol/ssl/Plugin.cc index 60d6b0d4a3..8e67ddbee6 100644 --- a/src/analyzer/protocol/ssl/Plugin.cc +++ b/src/analyzer/protocol/ssl/Plugin.cc @@ -1,10 +1,9 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "SSL.h" #include "DTLS.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_SSL { diff --git a/src/analyzer/protocol/ssl/functions.bif b/src/analyzer/protocol/ssl/functions.bif index 17720bcbb1..6ca1a6f9a7 100644 --- a/src/analyzer/protocol/ssl/functions.bif +++ b/src/analyzer/protocol/ssl/functions.bif @@ -1,6 +1,7 @@ %%{ #include "analyzer/protocol/ssl/SSL.h" +#include "Reporter.h" #include %%} diff --git a/src/analyzer/protocol/ssl/ssl.pac b/src/analyzer/protocol/ssl/ssl.pac index f7e7c17e7f..e7bf1bf23e 100644 --- a/src/analyzer/protocol/ssl/ssl.pac +++ b/src/analyzer/protocol/ssl/ssl.pac @@ -9,6 +9,7 @@ %include bro.pac %extern{ +#include "Desc.h" #include "events.bif.h" namespace analyzer { namespace ssl { class SSL_Analyzer; } } diff --git a/src/analyzer/protocol/ssl/tls-handshake.pac b/src/analyzer/protocol/ssl/tls-handshake.pac index 3bc03eeddb..c545d1bc71 100644 --- a/src/analyzer/protocol/ssl/tls-handshake.pac +++ b/src/analyzer/protocol/ssl/tls-handshake.pac @@ -4,6 +4,7 @@ %include bro.pac %extern{ +#include "Desc.h" #include "types.bif.h" #include "events.bif.h" %} diff --git a/src/analyzer/protocol/stepping-stone/Plugin.cc b/src/analyzer/protocol/stepping-stone/Plugin.cc index 5d76fa7d74..3152adc4c9 100644 --- a/src/analyzer/protocol/stepping-stone/Plugin.cc +++ b/src/analyzer/protocol/stepping-stone/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "SteppingStone.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_SteppingStone { diff --git a/src/analyzer/protocol/stepping-stone/SteppingStone.cc b/src/analyzer/protocol/stepping-stone/SteppingStone.cc index f4055028f0..1d37e83815 100644 --- a/src/analyzer/protocol/stepping-stone/SteppingStone.cc +++ b/src/analyzer/protocol/stepping-stone/SteppingStone.cc @@ -1,6 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek-config.h" +#include "SteppingStone.h" #include @@ -8,9 +9,8 @@ #include "Net.h" #include "NetVar.h" #include "analyzer/protocol/tcp/TCP.h" -#include "SteppingStone.h" +#include "Sessions.h" #include "util.h" - #include "events.bif.h" using namespace analyzer::stepping_stone; diff --git a/src/analyzer/protocol/syslog/Plugin.cc b/src/analyzer/protocol/syslog/Plugin.cc index e4d5f38fa1..7e2501575b 100644 --- a/src/analyzer/protocol/syslog/Plugin.cc +++ b/src/analyzer/protocol/syslog/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "Syslog.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_Syslog { diff --git a/src/analyzer/protocol/tcp/ContentLine.cc b/src/analyzer/protocol/tcp/ContentLine.cc index d98c5b4d2f..0c68e6ae2c 100644 --- a/src/analyzer/protocol/tcp/ContentLine.cc +++ b/src/analyzer/protocol/tcp/ContentLine.cc @@ -1,7 +1,6 @@ -#include - #include "ContentLine.h" -#include "analyzer/protocol/tcp/TCP.h" +#include "TCP.h" +#include "Reporter.h" #include "events.bif.h" diff --git a/src/analyzer/protocol/tcp/Plugin.cc b/src/analyzer/protocol/tcp/Plugin.cc index 3a99b2036a..078d6bb3d9 100644 --- a/src/analyzer/protocol/tcp/Plugin.cc +++ b/src/analyzer/protocol/tcp/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "TCP.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_TCP { diff --git a/src/analyzer/protocol/tcp/TCP.cc b/src/analyzer/protocol/tcp/TCP.cc index 543f767a2b..a19f425df3 100644 --- a/src/analyzer/protocol/tcp/TCP.cc +++ b/src/analyzer/protocol/tcp/TCP.cc @@ -1,14 +1,20 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include +#include "analyzer/protocol/tcp/TCP.h" +#include + +#include "analyzer/protocol/tcp/TCP_Reassembler.h" +#include "analyzer/protocol/pia/PIA.h" + +#include "IP.h" +#include "Net.h" #include "NetVar.h" #include "File.h" #include "Event.h" - -#include "analyzer/protocol/pia/PIA.h" -#include "analyzer/protocol/tcp/TCP.h" -#include "analyzer/protocol/tcp/TCP_Reassembler.h" +#include "Reporter.h" +#include "Sessions.h" +#include "DebugLogger.h" #include "events.bif.h" #include "types.bif.h" diff --git a/src/analyzer/protocol/tcp/TCP.h b/src/analyzer/protocol/tcp/TCP.h index 1204f56916..02c783b730 100644 --- a/src/analyzer/protocol/tcp/TCP.h +++ b/src/analyzer/protocol/tcp/TCP.h @@ -3,7 +3,6 @@ #pragma once #include "analyzer/Analyzer.h" -#include "PacketDumper.h" #include "IPAddr.h" #include "TCP_Endpoint.h" #include "TCP_Flags.h" diff --git a/src/analyzer/protocol/tcp/TCP_Endpoint.cc b/src/analyzer/protocol/tcp/TCP_Endpoint.cc index 47ed0a569f..fb3be41996 100644 --- a/src/analyzer/protocol/tcp/TCP_Endpoint.cc +++ b/src/analyzer/protocol/tcp/TCP_Endpoint.cc @@ -4,6 +4,7 @@ #include "NetVar.h" #include "analyzer/protocol/tcp/TCP.h" #include "TCP_Reassembler.h" +#include "Reporter.h" #include "Sessions.h" #include "Event.h" #include "File.h" diff --git a/src/analyzer/protocol/tcp/TCP_Endpoint.h b/src/analyzer/protocol/tcp/TCP_Endpoint.h index 160ad335fa..60ec19a5da 100644 --- a/src/analyzer/protocol/tcp/TCP_Endpoint.h +++ b/src/analyzer/protocol/tcp/TCP_Endpoint.h @@ -4,6 +4,7 @@ #include "IPAddr.h" +class BroFile; class Connection; class IP_Hdr; diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.cc b/src/analyzer/protocol/tcp/TCP_Reassembler.cc index 5df1cc468a..28f4139225 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.cc +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.cc @@ -1,13 +1,15 @@ -#include - +#include "TCP_Reassembler.h" +#include "TCP_Endpoint.h" #include "File.h" #include "analyzer/Analyzer.h" -#include "TCP_Reassembler.h" #include "analyzer/protocol/tcp/TCP.h" -#include "TCP_Endpoint.h" +#include "Reporter.h" +#include "RuleMatcher.h" #include "events.bif.h" +#include + using namespace analyzer::tcp; // Note, sequence numbers are relative. I.e., they start with 1. diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.h b/src/analyzer/protocol/tcp/TCP_Reassembler.h index dd499e79cc..3137160f36 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.h +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.h @@ -7,7 +7,11 @@ class BroFile; class Connection; -namespace analyzer { namespace tcp { +namespace analyzer { + +class Analyzer; + +namespace tcp { class TCP_Analyzer; diff --git a/src/analyzer/protocol/tcp/functions.bif b/src/analyzer/protocol/tcp/functions.bif index af8a894137..5fb0216c89 100644 --- a/src/analyzer/protocol/tcp/functions.bif +++ b/src/analyzer/protocol/tcp/functions.bif @@ -1,6 +1,8 @@ %%{ #include "File.h" +#include "Sessions.h" +#include "Reporter.h" #include "analyzer/protocol/tcp/TCP.h" %%} diff --git a/src/analyzer/protocol/teredo/Plugin.cc b/src/analyzer/protocol/teredo/Plugin.cc index eeebea870d..0098066304 100644 --- a/src/analyzer/protocol/teredo/Plugin.cc +++ b/src/analyzer/protocol/teredo/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "Teredo.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_Teredo { diff --git a/src/analyzer/protocol/teredo/Teredo.cc b/src/analyzer/protocol/teredo/Teredo.cc index adbbb9c964..1214f30e3b 100644 --- a/src/analyzer/protocol/teredo/Teredo.cc +++ b/src/analyzer/protocol/teredo/Teredo.cc @@ -4,6 +4,7 @@ #include "Conn.h" #include "IP.h" #include "Reporter.h" +#include "Sessions.h" #include "events.bif.h" diff --git a/src/analyzer/protocol/udp/Plugin.cc b/src/analyzer/protocol/udp/Plugin.cc index 9a42be6fa8..d94f898b31 100644 --- a/src/analyzer/protocol/udp/Plugin.cc +++ b/src/analyzer/protocol/udp/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "analyzer/protocol/udp/UDP.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_UDP { diff --git a/src/analyzer/protocol/vxlan/Plugin.cc b/src/analyzer/protocol/vxlan/Plugin.cc index 73c2cfd53b..2dee67952a 100644 --- a/src/analyzer/protocol/vxlan/Plugin.cc +++ b/src/analyzer/protocol/vxlan/Plugin.cc @@ -1,8 +1,8 @@ // See the file in the main distribution directory for copyright. -#include "plugin/Plugin.h" - #include "VXLAN.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_VXLAN { diff --git a/src/analyzer/protocol/vxlan/VXLAN.cc b/src/analyzer/protocol/vxlan/VXLAN.cc index f936ca4655..b7ed5322f7 100644 --- a/src/analyzer/protocol/vxlan/VXLAN.cc +++ b/src/analyzer/protocol/vxlan/VXLAN.cc @@ -6,10 +6,16 @@ #include "TunnelEncapsulation.h" #include "Conn.h" #include "IP.h" +#include "Net.h" +#include "Sessions.h" #include "Reporter.h" #include "events.bif.h" +extern "C" { +#include +} + using namespace analyzer::vxlan; void VXLAN_Analyzer::Done() diff --git a/src/analyzer/protocol/vxlan/VXLAN.h b/src/analyzer/protocol/vxlan/VXLAN.h index 62873959da..acc9accead 100644 --- a/src/analyzer/protocol/vxlan/VXLAN.h +++ b/src/analyzer/protocol/vxlan/VXLAN.h @@ -3,8 +3,6 @@ #pragma once #include "analyzer/Analyzer.h" -#include "NetVar.h" -#include "Reporter.h" namespace analyzer { namespace vxlan { diff --git a/src/analyzer/protocol/xmpp/Plugin.cc b/src/analyzer/protocol/xmpp/Plugin.cc index 92165e3d99..cd8eb16391 100644 --- a/src/analyzer/protocol/xmpp/Plugin.cc +++ b/src/analyzer/protocol/xmpp/Plugin.cc @@ -1,7 +1,8 @@ // See the file in the main distribution directory for copyright. -#include "plugin/Plugin.h" #include "XMPP.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_XMPP { diff --git a/src/analyzer/protocol/xmpp/xmpp.pac b/src/analyzer/protocol/xmpp/xmpp.pac index e6b5f4bba0..79e5159914 100644 --- a/src/analyzer/protocol/xmpp/xmpp.pac +++ b/src/analyzer/protocol/xmpp/xmpp.pac @@ -8,6 +8,7 @@ %extern{ +#include "Reporter.h" #include "events.bif.h" namespace analyzer { namespace xmpp { class XMPP_Analyzer; } } diff --git a/src/analyzer/protocol/zip/Plugin.cc b/src/analyzer/protocol/zip/Plugin.cc index f81576e1bb..2f00193ddd 100644 --- a/src/analyzer/protocol/zip/Plugin.cc +++ b/src/analyzer/protocol/zip/Plugin.cc @@ -1,9 +1,8 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "ZIP.h" +#include "plugin/Plugin.h" +#include "analyzer/Component.h" namespace plugin { namespace Zeek_ZIP { diff --git a/src/binpac_bro.h b/src/binpac_bro.h index be525ee918..1db3f0d3b8 100644 --- a/src/binpac_bro.h +++ b/src/binpac_bro.h @@ -9,10 +9,8 @@ namespace analyzer { class Analyzer; } #include "util.h" #include "Val.h" #include "event.bif.func_h" -#include "TunnelEncapsulation.h" #include "analyzer/Analyzer.h" #include "file_analysis/Analyzer.h" -#include "Conn.h" #include "binpac.h" diff --git a/src/bro-bif.h b/src/bro-bif.h index 0853647af8..a20996152b 100644 --- a/src/bro-bif.h +++ b/src/bro-bif.h @@ -6,3 +6,5 @@ #include "Conn.h" #include "NetVar.h" #include "Event.h" +#include "Reporter.h" +#include "Var.h" // for internal_type() diff --git a/src/broker/Data.cc b/src/broker/Data.cc index 0929961dc8..dc5f829082 100644 --- a/src/broker/Data.cc +++ b/src/broker/Data.cc @@ -1,5 +1,7 @@ #include "Data.h" #include "File.h" +#include "IntrusivePtr.h" +#include "module_util.h" #include "3rdparty/doctest.h" #include "broker/data.bif.h" diff --git a/src/broker/Data.h b/src/broker/Data.h index fa7ca89400..e31d27e1c7 100644 --- a/src/broker/Data.h +++ b/src/broker/Data.h @@ -1,13 +1,14 @@ #pragma once -#include -#include - #include "OpaqueVal.h" #include "Reporter.h" #include "Frame.h" #include "Expr.h" -#include "IntrusivePtr.h" +#include "Desc.h" +#include "Var.h" // for internal_type() + +template +class IntrusivePtr; namespace bro_broker { diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 1e54ce33a0..129f6ad525 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -1,3 +1,4 @@ +#include "Manager.h" #include #include @@ -5,12 +6,13 @@ #include #include -#include "Manager.h" #include "Data.h" #include "Store.h" #include "util.h" #include "Var.h" +#include "Desc.h" #include "Reporter.h" +#include "IntrusivePtr.h" #include "broker/comm.bif.h" #include "broker/data.bif.h" #include "broker/messaging.bif.h" @@ -19,6 +21,7 @@ #include "DebugLogger.h" #include "iosource/Manager.h" #include "SerializationFormat.h" +#include "Net.h" using namespace std; diff --git a/src/broker/Manager.h b/src/broker/Manager.h index 43698a22d3..5f98177a09 100644 --- a/src/broker/Manager.h +++ b/src/broker/Manager.h @@ -12,20 +12,21 @@ #include #include #include + #include #include -#include -#include #include -#include -#include "broker/Store.h" -#include "Reporter.h" + +#include "NetVar.h" #include "iosource/IOSource.h" -#include "Val.h" #include "logging/WriterBackend.h" +class Frame; + namespace bro_broker { +class StoreHandleVal; +class StoreQueryCallback; class BrokerState; /** diff --git a/src/broker/Store.cc b/src/broker/Store.cc index 2f61b14d37..20020375eb 100644 --- a/src/broker/Store.cc +++ b/src/broker/Store.cc @@ -1,4 +1,5 @@ #include "Store.h" +#include "Desc.h" #include "broker/Manager.h" namespace bro_broker { diff --git a/src/broker/Store.h b/src/broker/Store.h index 0ba3d2ad5d..fd8f0911d5 100644 --- a/src/broker/Store.h +++ b/src/broker/Store.h @@ -2,8 +2,8 @@ #include "broker/store.bif.h" #include "broker/data.bif.h" -#include "Reporter.h" #include "Type.h" +#include "Var.h" // for internal_type() #include "OpaqueVal.h" #include "Trigger.h" diff --git a/src/file_analysis/Component.h b/src/file_analysis/Component.h index a4aee3d5eb..e63a4db248 100644 --- a/src/file_analysis/Component.h +++ b/src/file_analysis/Component.h @@ -6,10 +6,9 @@ #include "plugin/Component.h" #include "plugin/TaggedComponent.h" -#include "Val.h" - #include "../zeek-config.h" -#include "../util.h" + +class RecordVal; namespace file_analysis { diff --git a/src/file_analysis/File.cc b/src/file_analysis/File.cc index 1367d69196..131bb7cbef 100644 --- a/src/file_analysis/File.cc +++ b/src/file_analysis/File.cc @@ -1,9 +1,10 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include -#include - #include "File.h" + +#include + +#include "FileReassembler.h" #include "FileTimer.h" #include "Analyzer.h" #include "Manager.h" diff --git a/src/file_analysis/File.h b/src/file_analysis/File.h index 94a15ae039..058de9f153 100644 --- a/src/file_analysis/File.h +++ b/src/file_analysis/File.h @@ -4,19 +4,20 @@ #include #include -#include -#include "FileReassembler.h" -#include "Conn.h" -#include "Val.h" -#include "Tag.h" #include "AnalyzerSet.h" #include "BroString.h" #include "WeirdState.h" +using std::string; + +class Connection; +class RecordVal; + namespace file_analysis { class FileReassembler; +class Tag; /** * Wrapper class around \c fa_file record values from script layer. diff --git a/src/file_analysis/FileReassembler.h b/src/file_analysis/FileReassembler.h index 287ebd8d22..caaa1d443a 100644 --- a/src/file_analysis/FileReassembler.h +++ b/src/file_analysis/FileReassembler.h @@ -1,7 +1,6 @@ #pragma once #include "Reassem.h" -#include "File.h" class BroFile; class Connection; diff --git a/src/file_analysis/FileTimer.cc b/src/file_analysis/FileTimer.cc index 40237ebb54..015915c36b 100644 --- a/src/file_analysis/FileTimer.cc +++ b/src/file_analysis/FileTimer.cc @@ -1,7 +1,8 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include "Manager.h" +#include "FileTimer.h" #include "File.h" +#include "Manager.h" using namespace file_analysis; diff --git a/src/file_analysis/FileTimer.h b/src/file_analysis/FileTimer.h index 1ccadee1f3..32ddd4fb9c 100644 --- a/src/file_analysis/FileTimer.h +++ b/src/file_analysis/FileTimer.h @@ -2,9 +2,12 @@ #pragma once -#include #include "Timer.h" +#include + +using std::string; + namespace file_analysis { /** diff --git a/src/file_analysis/Manager.cc b/src/file_analysis/Manager.cc index 29cb492acc..9c7f87d8b5 100644 --- a/src/file_analysis/Manager.cc +++ b/src/file_analysis/Manager.cc @@ -1,9 +1,5 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include -#include -#include - #include "Manager.h" #include "File.h" #include "Analyzer.h" @@ -15,6 +11,8 @@ #include "plugin/Manager.h" #include "analyzer/Manager.h" +#include + using namespace file_analysis; TableVal* Manager::disabled = 0; diff --git a/src/file_analysis/Manager.h b/src/file_analysis/Manager.h index 870d8e9d4e..37c48402b5 100644 --- a/src/file_analysis/Manager.h +++ b/src/file_analysis/Manager.h @@ -6,26 +6,30 @@ #include #include -#include "Dict.h" +#include "Component.h" #include "Net.h" -#include "Conn.h" -#include "Val.h" -#include "Analyzer.h" -#include "Timer.h" -#include "EventHandler.h" #include "RuleMatcher.h" -#include "File.h" -#include "FileTimer.h" -#include "Component.h" -#include "Tag.h" #include "plugin/ComponentManager.h" -#include "analyzer/Tag.h" #include "file_analysis/file_analysis.bif.h" +using std::map; +using std::set; + +class TableVal; +class VectorVal; + +namespace analyzer { +class Analyzer; +class Tag; +} + namespace file_analysis { +class File; +class Tag; + /** * Main entry point for interacting with file analysis. */ diff --git a/src/file_analysis/Tag.h b/src/file_analysis/Tag.h index 92100a2e69..6f5dae6b03 100644 --- a/src/file_analysis/Tag.h +++ b/src/file_analysis/Tag.h @@ -3,13 +3,17 @@ #pragma once #include "zeek-config.h" -#include "util.h" #include "../Tag.h" -#include "plugin/TaggedComponent.h" -#include "plugin/ComponentManager.h" class EnumVal; +namespace plugin { +template +class TaggedComponent; +template +class ComponentManager; +} + namespace file_analysis { class Component; diff --git a/src/file_analysis/analyzer/data_event/Plugin.cc b/src/file_analysis/analyzer/data_event/Plugin.cc index b41d2356a7..14eefac2e3 100644 --- a/src/file_analysis/analyzer/data_event/Plugin.cc +++ b/src/file_analysis/analyzer/data_event/Plugin.cc @@ -1,8 +1,8 @@ // See the file in the main distribution directory for copyright. -#include "plugin/Plugin.h" - #include "DataEvent.h" +#include "plugin/Plugin.h" +#include "file_analysis/Component.h" namespace plugin { namespace Zeek_FileDataEvent { diff --git a/src/file_analysis/analyzer/entropy/Plugin.cc b/src/file_analysis/analyzer/entropy/Plugin.cc index a4ae3416cd..6ac3dffd56 100644 --- a/src/file_analysis/analyzer/entropy/Plugin.cc +++ b/src/file_analysis/analyzer/entropy/Plugin.cc @@ -1,8 +1,8 @@ // See the file in the main distribution directory for copyright. -#include "plugin/Plugin.h" - #include "Entropy.h" +#include "plugin/Plugin.h" +#include "file_analysis/Component.h" namespace plugin { namespace Zeek_FileEntropy { diff --git a/src/file_analysis/analyzer/extract/Plugin.cc b/src/file_analysis/analyzer/extract/Plugin.cc index be8c44eaac..dc7107d858 100644 --- a/src/file_analysis/analyzer/extract/Plugin.cc +++ b/src/file_analysis/analyzer/extract/Plugin.cc @@ -1,8 +1,8 @@ // See the file in the main distribution directory for copyright. -#include "plugin/Plugin.h" - #include "Extract.h" +#include "plugin/Plugin.h" +#include "file_analysis/Component.h" namespace plugin { namespace Zeek_FileExtract { diff --git a/src/file_analysis/analyzer/hash/Plugin.cc b/src/file_analysis/analyzer/hash/Plugin.cc index 774e51511e..9e11ee3832 100644 --- a/src/file_analysis/analyzer/hash/Plugin.cc +++ b/src/file_analysis/analyzer/hash/Plugin.cc @@ -1,8 +1,8 @@ // See the file in the main distribution directory for copyright. -#include "plugin/Plugin.h" - #include "Hash.h" +#include "plugin/Plugin.h" +#include "file_analysis/Component.h" namespace plugin { namespace Zeek_FileHash { diff --git a/src/file_analysis/analyzer/pe/Plugin.cc b/src/file_analysis/analyzer/pe/Plugin.cc index 08a255785e..0b4ae65a12 100644 --- a/src/file_analysis/analyzer/pe/Plugin.cc +++ b/src/file_analysis/analyzer/pe/Plugin.cc @@ -1,8 +1,8 @@ // See the file in the main distribution directory for copyright. -#include "plugin/Plugin.h" - #include "PE.h" +#include "plugin/Plugin.h" +#include "file_analysis/Component.h" namespace plugin { namespace Zeek_PE { diff --git a/src/file_analysis/analyzer/unified2/Plugin.cc b/src/file_analysis/analyzer/unified2/Plugin.cc index 2fef6e5dfa..dba2fb0b46 100644 --- a/src/file_analysis/analyzer/unified2/Plugin.cc +++ b/src/file_analysis/analyzer/unified2/Plugin.cc @@ -2,9 +2,9 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include "plugin/Plugin.h" - #include "Unified2.h" +#include "plugin/Plugin.h" +#include "file_analysis/Component.h" namespace plugin { namespace Zeek_Unified2 { diff --git a/src/file_analysis/analyzer/x509/OCSP.cc b/src/file_analysis/analyzer/x509/OCSP.cc index 8f5ed3419c..dd7d378f0d 100644 --- a/src/file_analysis/analyzer/x509/OCSP.cc +++ b/src/file_analysis/analyzer/x509/OCSP.cc @@ -5,6 +5,7 @@ #include "OCSP.h" #include "X509.h" #include "Event.h" +#include "Reporter.h" #include "types.bif.h" #include "ocsp_events.bif.h" diff --git a/src/file_analysis/analyzer/x509/Plugin.cc b/src/file_analysis/analyzer/x509/Plugin.cc index 2f495e6337..221816fca5 100644 --- a/src/file_analysis/analyzer/x509/Plugin.cc +++ b/src/file_analysis/analyzer/x509/Plugin.cc @@ -1,10 +1,9 @@ // See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - #include "X509.h" #include "OCSP.h" +#include "plugin/Plugin.h" +#include "file_analysis/Component.h" namespace plugin { namespace Zeek_X509 { diff --git a/src/file_analysis/analyzer/x509/X509Common.cc b/src/file_analysis/analyzer/x509/X509Common.cc index 7fb3100e97..a14e73085a 100644 --- a/src/file_analysis/analyzer/x509/X509Common.cc +++ b/src/file_analysis/analyzer/x509/X509Common.cc @@ -2,6 +2,7 @@ #include "X509Common.h" #include "x509-extension_pac.h" +#include "Reporter.h" #include "events.bif.h" #include "ocsp_events.bif.h" diff --git a/src/file_analysis/analyzer/x509/X509Common.h b/src/file_analysis/analyzer/x509/X509Common.h index fe56e0b9a8..019433a47a 100644 --- a/src/file_analysis/analyzer/x509/X509Common.h +++ b/src/file_analysis/analyzer/x509/X509Common.h @@ -11,6 +11,8 @@ #include #include +class Reporter; + namespace file_analysis { class X509Common : public file_analysis::Analyzer { diff --git a/src/file_analysis/file_analysis.bif b/src/file_analysis/file_analysis.bif index f3086041b0..d85d8cdb5a 100644 --- a/src/file_analysis/file_analysis.bif +++ b/src/file_analysis/file_analysis.bif @@ -4,6 +4,8 @@ module Files; %%{ #include "file_analysis/Manager.h" +#include "file_analysis/File.h" +#include "Reporter.h" %%} type AnalyzerArgs: record; diff --git a/src/input/Manager.cc b/src/input/Manager.cc index cbfa4086d9..53981e589f 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1,12 +1,17 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include - #include "Manager.h" + +#include +#include + #include "ReaderFrontend.h" #include "ReaderBackend.h" +#include "Desc.h" +#include "module_util.h" #include "input.bif.h" +#include "Expr.h" #include "Event.h" #include "EventHandler.h" #include "NetVar.h" diff --git a/src/input/Manager.h b/src/input/Manager.h index 2cdfb9e192..69e3c2964d 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -4,14 +4,16 @@ #pragma once -#include "BroString.h" -#include "EventHandler.h" -#include "Val.h" - #include "Component.h" +#include "EventHandler.h" +#include "plugin/ComponentManager.h" +#include "threading/SerialTypes.h" +#include "Tag.h" #include +class RecordVal; + namespace input { class ReaderFrontend; diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc index 3852a1002a..09e9dba456 100644 --- a/src/input/ReaderFrontend.cc +++ b/src/input/ReaderFrontend.cc @@ -4,8 +4,6 @@ #include "ReaderFrontend.h" #include "ReaderBackend.h" -#include "threading/MsgThread.h" - namespace input { class InitMessage : public threading::InputMessage diff --git a/src/input/ReaderFrontend.h b/src/input/ReaderFrontend.h index c9f597eb74..9a30995ada 100644 --- a/src/input/ReaderFrontend.h +++ b/src/input/ReaderFrontend.h @@ -3,10 +3,9 @@ #pragma once #include "ReaderBackend.h" -#include "threading/MsgThread.h" #include "threading/SerialTypes.h" -#include "Val.h" +class EnumVal; namespace input { diff --git a/src/input/Tag.h b/src/input/Tag.h index 42f24a0345..86f85ac36b 100644 --- a/src/input/Tag.h +++ b/src/input/Tag.h @@ -3,13 +3,17 @@ #pragma once #include "zeek-config.h" -#include "util.h" #include "../Tag.h" -#include "plugin/TaggedComponent.h" -#include "plugin/ComponentManager.h" class EnumVal; +namespace plugin { +template +class TaggedComponent; +template +class ComponentManager; +} + namespace input { class Manager; diff --git a/src/iosource/BPF_Program.cc b/src/iosource/BPF_Program.cc index a99acd07e4..8a97a6fc25 100644 --- a/src/iosource/BPF_Program.cc +++ b/src/iosource/BPF_Program.cc @@ -2,9 +2,10 @@ #include "zeek-config.h" -#include "util.h" #include "BPF_Program.h" +#include + #ifdef DONT_HAVE_LIBPCAP_PCAP_FREECODE extern "C" { #include "pcap-int.h" diff --git a/src/iosource/BPF_Program.h b/src/iosource/BPF_Program.h index 4823b7c9b4..de06e9d13a 100644 --- a/src/iosource/BPF_Program.h +++ b/src/iosource/BPF_Program.h @@ -6,7 +6,7 @@ extern "C" { #include } -#include "util.h" +#include // BPF_Programs are an abstraction around struct bpf_program, // to create a clean facility for creating, compiling, and diff --git a/src/iosource/Component.h b/src/iosource/Component.h index cf072feb36..9d8367ac0a 100644 --- a/src/iosource/Component.h +++ b/src/iosource/Component.h @@ -2,11 +2,11 @@ #pragma once +#include "plugin/Component.h" + #include #include -#include "plugin/Component.h" - namespace iosource { class IOSource; diff --git a/src/iosource/IOSource.h b/src/iosource/IOSource.h index cc8cf6a97f..ea218a675b 100644 --- a/src/iosource/IOSource.h +++ b/src/iosource/IOSource.h @@ -2,12 +2,6 @@ #pragma once -extern "C" { -#include -} - -#include - namespace iosource { /** diff --git a/src/iosource/Manager.cc b/src/iosource/Manager.cc index b93bb03c60..ded3348040 100644 --- a/src/iosource/Manager.cc +++ b/src/iosource/Manager.cc @@ -1,12 +1,14 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include + #include +#include #include #include #include #include "Manager.h" +#include "Component.h" #include "IOSource.h" #include "Net.h" #include "PktSrc.h" diff --git a/src/iosource/Manager.h b/src/iosource/Manager.h index 1cd8eb965d..2b0f4bec9a 100644 --- a/src/iosource/Manager.h +++ b/src/iosource/Manager.h @@ -11,6 +11,9 @@ #include "IOSource.h" #include "Flare.h" +struct timespec; +struct kevent; + namespace iosource { class PktSrc; diff --git a/src/iosource/Packet.cc b/src/iosource/Packet.cc index 648aef3b6e..f498fe0402 100644 --- a/src/iosource/Packet.cc +++ b/src/iosource/Packet.cc @@ -1,9 +1,11 @@ #include "Packet.h" #include "Sessions.h" +#include "Desc.h" #include "iosource/Manager.h" extern "C" { +#include #ifdef HAVE_NET_ETHERNET_H #include #elif defined(HAVE_SYS_ETHERNET_H) diff --git a/src/iosource/Packet.h b/src/iosource/Packet.h index 0a0995f944..2017f11677 100644 --- a/src/iosource/Packet.h +++ b/src/iosource/Packet.h @@ -1,8 +1,11 @@ #pragma once -#include "Desc.h" #include "IP.h" -#include "NetVar.h" + +#include + +#include +#include // for u_char #if defined(__OpenBSD__) #include @@ -11,6 +14,9 @@ typedef struct bpf_timeval pkt_timeval; typedef struct timeval pkt_timeval; #endif +class Val; +class ODesc; + /** * The Layer 3 type of a packet, as determined by the parsing code in Packet. */ diff --git a/src/iosource/PktDumper.cc b/src/iosource/PktDumper.cc index 863c46ec81..9a49ccb6fb 100644 --- a/src/iosource/PktDumper.cc +++ b/src/iosource/PktDumper.cc @@ -1,12 +1,10 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include -#include - #include "zeek-config.h" #include "PktDumper.h" +#include "DebugLogger.h" using namespace iosource; diff --git a/src/iosource/PktDumper.h b/src/iosource/PktDumper.h index 2b6251ba0a..406e90ffc2 100644 --- a/src/iosource/PktDumper.h +++ b/src/iosource/PktDumper.h @@ -2,8 +2,9 @@ #pragma once -#include "Packet.h" -#include "IOSource.h" +#include + +class Packet; namespace iosource { diff --git a/src/iosource/PktSrc.cc b/src/iosource/PktSrc.cc index 16eaf5aec7..19bf1f3f52 100644 --- a/src/iosource/PktSrc.cc +++ b/src/iosource/PktSrc.cc @@ -1,17 +1,17 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include +#include "zeek-config.h" +#include "PktSrc.h" + #include -#include "zeek-config.h" - #include "util.h" -#include "PktSrc.h" #include "Hash.h" #include "Net.h" #include "Sessions.h" #include "broker/Manager.h" #include "iosource/Manager.h" +#include "BPF_Program.h" #include "pcap/pcap.bif.h" diff --git a/src/iosource/PktSrc.h b/src/iosource/PktSrc.h index 8207e030e9..bbb7c4a830 100644 --- a/src/iosource/PktSrc.h +++ b/src/iosource/PktSrc.h @@ -5,10 +5,13 @@ #include #include "IOSource.h" -#include "BPF_Program.h" -#include "Dict.h" #include "Packet.h" +#include // for u_char + +struct pcap_pkthdr; +class BPF_Program; + namespace iosource { /** diff --git a/src/iosource/pcap/Plugin.cc b/src/iosource/pcap/Plugin.cc index 75f8f54a2c..4aec5bd4cd 100644 --- a/src/iosource/pcap/Plugin.cc +++ b/src/iosource/pcap/Plugin.cc @@ -1,9 +1,9 @@ // See the file in the main distribution directory for copyright. -#include "plugin/Plugin.h" - #include "Source.h" #include "Dumper.h" +#include "plugin/Plugin.h" +#include "iosource/Component.h" namespace plugin { namespace Zeek_Pcap { diff --git a/src/iosource/pcap/Source.cc b/src/iosource/pcap/Source.cc index 47df453ff0..eb13b4d024 100644 --- a/src/iosource/pcap/Source.cc +++ b/src/iosource/pcap/Source.cc @@ -1,11 +1,10 @@ // See the file in the main distribution directory for copyright. -#include - #include "zeek-config.h" #include "Source.h" #include "iosource/Packet.h" +#include "iosource/BPF_Program.h" #include "pcap.bif.h" diff --git a/src/iosource/pcap/Source.h b/src/iosource/pcap/Source.h index df57d6612d..fd2fa0ec53 100644 --- a/src/iosource/pcap/Source.h +++ b/src/iosource/pcap/Source.h @@ -4,6 +4,12 @@ #include "../PktSrc.h" +extern "C" { +#include +} + +#include // for u_char + namespace iosource { namespace pcap { diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index a5ed5e3c35..0b92c911b9 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -1,6 +1,8 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include +#include "Manager.h" + +#include #include "Event.h" #include "EventHandler.h" @@ -8,18 +10,21 @@ #include "Net.h" #include "Type.h" #include "File.h" +#include "input.h" #include "broker/Manager.h" #include "threading/Manager.h" #include "threading/SerialTypes.h" -#include "Manager.h" +#include "Desc.h" #include "WriterFrontend.h" #include "WriterBackend.h" #include "logging.bif.h" #include "plugin/Plugin.h" #include "plugin/Manager.h" +#include + using namespace logging; struct Manager::Filter { diff --git a/src/logging/Manager.h b/src/logging/Manager.h index 06c551e8ed..2d6a9728c3 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -4,8 +4,6 @@ #pragma once -#include - #include "../Val.h" #include "../Tag.h" #include "../EventHandler.h" @@ -14,6 +12,7 @@ #include "Component.h" #include "WriterBackend.h" +namespace broker { struct endpoint_info; } class SerializationFormat; class RotationTimer; diff --git a/src/logging/Tag.h b/src/logging/Tag.h index d37561dd91..40a7b40b5f 100644 --- a/src/logging/Tag.h +++ b/src/logging/Tag.h @@ -3,13 +3,17 @@ #pragma once #include "zeek-config.h" -#include "util.h" #include "../Tag.h" -#include "plugin/TaggedComponent.h" -#include "plugin/ComponentManager.h" class EnumVal; +namespace plugin { +template +class TaggedComponent; +template +class ComponentManager; +} + namespace logging { class Manager; diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h index b202621ec6..9a78ecf7db 100644 --- a/src/logging/WriterFrontend.h +++ b/src/logging/WriterFrontend.h @@ -4,8 +4,6 @@ #include "WriterBackend.h" -#include "threading/MsgThread.h" - namespace logging { class Manager; diff --git a/src/logging/writers/ascii/Ascii.h b/src/logging/writers/ascii/Ascii.h index 721ba95e20..875216b880 100644 --- a/src/logging/writers/ascii/Ascii.h +++ b/src/logging/writers/ascii/Ascii.h @@ -7,6 +7,7 @@ #include "logging/WriterBackend.h" #include "threading/formatters/Ascii.h" #include "threading/formatters/JSON.h" +#include "Desc.h" #include "zlib.h" namespace logging { namespace writer { diff --git a/src/logging/writers/none/None.cc b/src/logging/writers/none/None.cc index 0bd507e1f8..5691430476 100644 --- a/src/logging/writers/none/None.cc +++ b/src/logging/writers/none/None.cc @@ -4,6 +4,8 @@ #include "None.h" #include "none.bif.h" +#include + using namespace logging; using namespace writer; diff --git a/src/logging/writers/sqlite/SQLite.h b/src/logging/writers/sqlite/SQLite.h index b45357015d..87003f35cb 100644 --- a/src/logging/writers/sqlite/SQLite.h +++ b/src/logging/writers/sqlite/SQLite.h @@ -9,6 +9,7 @@ #include "logging/WriterBackend.h" #include "threading/formatters/Ascii.h" #include "3rdparty/sqlite3.h" +#include "Desc.h" namespace logging { namespace writer { diff --git a/src/main.cc b/src/main.cc index c4fae32fb4..344218331a 100644 --- a/src/main.cc +++ b/src/main.cc @@ -33,6 +33,7 @@ extern "C" { #include "Var.h" #include "Timer.h" #include "Stmt.h" +#include "Desc.h" #include "Debug.h" #include "DFA.h" #include "RuleMatcher.h" diff --git a/src/parse.y b/src/parse.y index 4487998c5d..bb9e3ba82d 100644 --- a/src/parse.y +++ b/src/parse.y @@ -78,8 +78,12 @@ #include #include "input.h" +#include "BroList.h" +#include "Desc.h" #include "Expr.h" +#include "Func.h" #include "Stmt.h" +#include "Val.h" #include "Var.h" /* #include "analyzer/protocol/dns/DNS.h" */ #include "RE.h" @@ -87,6 +91,7 @@ #include "Reporter.h" #include "Brofiler.h" #include "zeekygen/Manager.h" +#include "module_util.h" #include #include diff --git a/src/plugin/ComponentManager.h b/src/plugin/ComponentManager.h index 88d04bb3f9..8f146c573e 100644 --- a/src/plugin/ComponentManager.h +++ b/src/plugin/ComponentManager.h @@ -5,11 +5,11 @@ #include #include "Type.h" -#include "ID.h" -#include "Var.h" +#include "Var.h" // for add_type() #include "Val.h" #include "Reporter.h" #include "zeekygen/Manager.h" +#include "DebugLogger.h" namespace plugin { diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index 51b85bdbb5..12c13ac3c3 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -14,6 +14,7 @@ #include "../Func.h" #include "../Event.h" #include "../util.h" +#include "../input.h" using namespace plugin; diff --git a/src/plugin/Plugin.cc b/src/plugin/Plugin.cc index 92bfcae8dc..29f177d329 100644 --- a/src/plugin/Plugin.cc +++ b/src/plugin/Plugin.cc @@ -2,15 +2,19 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include "Plugin.h" + #include -#include "Plugin.h" #include "Manager.h" #include "Component.h" +#include "Val.h" #include "../Desc.h" #include "../Event.h" +#include "../Func.h" #include "../Conn.h" +#include "../input.h" #include "threading/SerialTypes.h" using namespace plugin; diff --git a/src/plugin/Plugin.h b/src/plugin/Plugin.h index b1fc642667..7e8a35427c 100644 --- a/src/plugin/Plugin.h +++ b/src/plugin/Plugin.h @@ -7,9 +7,6 @@ #include #include "zeek-config.h" -#include "analyzer/Component.h" -#include "file_analysis/Component.h" -#include "iosource/Component.h" #include "logging/WriterBackend.h" // Increase this when making incompatible changes to the plugin API. Note @@ -19,6 +16,7 @@ #define BRO_PLUGIN_BRO_VERSION BRO_VERSION_FUNCTION class ODesc; +class Frame; class Func; class Event; diff --git a/src/probabilistic/BitVector.cc b/src/probabilistic/BitVector.cc index abf13302cb..5b3139388a 100644 --- a/src/probabilistic/BitVector.cc +++ b/src/probabilistic/BitVector.cc @@ -1,10 +1,13 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include "BitVector.h" + +#include + #include #include #include -#include "BitVector.h" #include "digest.h" using namespace probabilistic; diff --git a/src/probabilistic/BitVector.h b/src/probabilistic/BitVector.h index 8e9333fd6e..cbbcedd80b 100644 --- a/src/probabilistic/BitVector.h +++ b/src/probabilistic/BitVector.h @@ -2,11 +2,13 @@ #pragma once +#include + #include +#include #include -#include -#include +namespace broker { class data; } namespace probabilistic { diff --git a/src/probabilistic/BloomFilter.cc b/src/probabilistic/BloomFilter.cc index e5062f64c9..71996dc6c8 100644 --- a/src/probabilistic/BloomFilter.cc +++ b/src/probabilistic/BloomFilter.cc @@ -1,13 +1,13 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include +#include "BloomFilter.h" + #include #include +#include #include -#include "BloomFilter.h" - #include "CounterVector.h" #include "../util.h" diff --git a/src/probabilistic/BloomFilter.h b/src/probabilistic/BloomFilter.h index 950235e5f5..cc3a4a8e72 100644 --- a/src/probabilistic/BloomFilter.h +++ b/src/probabilistic/BloomFilter.h @@ -2,15 +2,17 @@ #pragma once +#include #include #include -#include #include #include "BitVector.h" #include "Hasher.h" +namespace broker { class data; } + namespace probabilistic { class CounterVector; diff --git a/src/probabilistic/CardinalityCounter.cc b/src/probabilistic/CardinalityCounter.cc index e5ce31e855..bdc9797125 100644 --- a/src/probabilistic/CardinalityCounter.cc +++ b/src/probabilistic/CardinalityCounter.cc @@ -1,10 +1,13 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include "CardinalityCounter.h" + #include #include -#include +#include + +#include -#include "CardinalityCounter.h" #include "Reporter.h" using namespace probabilistic; diff --git a/src/probabilistic/CardinalityCounter.h b/src/probabilistic/CardinalityCounter.h index ec9397fd84..26f61591b1 100644 --- a/src/probabilistic/CardinalityCounter.h +++ b/src/probabilistic/CardinalityCounter.h @@ -2,12 +2,14 @@ #pragma once -#include +#include + #include #include -#include -#include +#include + +namespace broker { class data; } namespace probabilistic { diff --git a/src/probabilistic/CounterVector.cc b/src/probabilistic/CounterVector.cc index 48f71c401b..d4b073bca1 100644 --- a/src/probabilistic/CounterVector.cc +++ b/src/probabilistic/CounterVector.cc @@ -4,11 +4,13 @@ #include #include + +#include +#include + #include "BitVector.h" #include "util.h" -#include - using namespace probabilistic; CounterVector::CounterVector(size_t arg_width, size_t cells) diff --git a/src/probabilistic/CounterVector.h b/src/probabilistic/CounterVector.h index 3476b9f4e6..0cdd05be95 100644 --- a/src/probabilistic/CounterVector.h +++ b/src/probabilistic/CounterVector.h @@ -4,10 +4,12 @@ #include #include +#include -#include #include +namespace broker { class data; } + namespace probabilistic { class BitVector; diff --git a/src/probabilistic/Hasher.cc b/src/probabilistic/Hasher.cc index c01d9efe98..bc46a1ebb6 100644 --- a/src/probabilistic/Hasher.cc +++ b/src/probabilistic/Hasher.cc @@ -1,13 +1,17 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include "Hasher.h" + #include + #include -#include "Hasher.h" #include "NetVar.h" #include "digest.h" #include "siphash24.h" +#include + using namespace probabilistic; Hasher::seed_t Hasher::MakeSeed(const void* data, size_t size) diff --git a/src/probabilistic/Hasher.h b/src/probabilistic/Hasher.h index 6298b881f1..6892b40880 100644 --- a/src/probabilistic/Hasher.h +++ b/src/probabilistic/Hasher.h @@ -2,12 +2,13 @@ #pragma once -#include +#include "Hash.h" + #include #include -#include "Hash.h" +namespace broker { class data; } namespace probabilistic { diff --git a/src/probabilistic/Topk.cc b/src/probabilistic/Topk.cc index 131f490fae..97ef572ddd 100644 --- a/src/probabilistic/Topk.cc +++ b/src/probabilistic/Topk.cc @@ -1,10 +1,12 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include "probabilistic/Topk.h" + #include #include "broker/Data.h" -#include "probabilistic/Topk.h" #include "CompHash.h" +#include "IntrusivePtr.h" #include "Reporter.h" #include "NetVar.h" diff --git a/src/re-parse.y b/src/re-parse.y index 164705c040..a7d26420d6 100644 --- a/src/re-parse.y +++ b/src/re-parse.y @@ -3,6 +3,7 @@ %{ #include +#include "RE.h" #include "CCL.h" #include "NFA.h" #include "EquivClass.h" diff --git a/src/re-scan.l b/src/re-scan.l index 99dde0ca6c..f7f29027c5 100644 --- a/src/re-scan.l +++ b/src/re-scan.l @@ -5,6 +5,7 @@ */ %{ +#include "RE.h" #include "CCL.h" #include "NFA.h" #include "util.h" diff --git a/src/rule-parse.y b/src/rule-parse.y index df12bd1d9b..cd44e4d205 100644 --- a/src/rule-parse.y +++ b/src/rule-parse.y @@ -3,6 +3,8 @@ #include #include #include "zeek-config.h" +#include "RuleAction.h" +#include "RuleCondition.h" #include "RuleMatcher.h" #include "Reporter.h" #include "IPAddr.h" diff --git a/src/rule-scan.l b/src/rule-scan.l index e9d2b4fece..f34935d361 100644 --- a/src/rule-scan.l +++ b/src/rule-scan.l @@ -6,6 +6,7 @@ #include #include #include "RuleMatcher.h" +#include "RuleCondition.h" #include "IPAddr.h" #include "util.h" #include "rule-parse.h" diff --git a/src/scan.l b/src/scan.l index 9e0a95f56d..5e32a41ab9 100644 --- a/src/scan.l +++ b/src/scan.l @@ -15,10 +15,12 @@ #include "input.h" #include "util.h" #include "Scope.h" +#include "BroString.h" #include "DNS_Mgr.h" #include "Expr.h" #include "Func.h" #include "Stmt.h" +#include "Val.h" #include "Var.h" #include "Debug.h" #include "PolicyFile.h" @@ -27,6 +29,7 @@ #include "RE.h" #include "Net.h" #include "Traverse.h" +#include "module_util.h" #include "analyzer/Analyzer.h" #include "zeekygen/Manager.h" diff --git a/src/supervisor/Supervisor.cc b/src/supervisor/Supervisor.cc index 124c9b7771..fea5e2a3ee 100644 --- a/src/supervisor/Supervisor.cc +++ b/src/supervisor/Supervisor.cc @@ -1,5 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include "Supervisor.h" + #include #include #include @@ -11,14 +13,15 @@ #include #include "iosource/Manager.h" -#include "Supervisor.h" #include "Reporter.h" #include "DebugLogger.h" +#include "ID.h" #include "Val.h" #include "Net.h" #include "NetVar.h" #include "zeek-config.h" #include "util.h" +#include "input.h" #include "zeek-affinity.h" #define RAPIDJSON_HAS_STDSTRING 1 diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h index d9c1b809b9..fba5acaeeb 100644 --- a/src/threading/BasicThread.h +++ b/src/threading/BasicThread.h @@ -1,9 +1,10 @@ #pragma once +#include #include -#include "util.h" +#include using namespace std; diff --git a/src/threading/Formatter.cc b/src/threading/Formatter.cc index 5fa8ab6fa9..462067303a 100644 --- a/src/threading/Formatter.cc +++ b/src/threading/Formatter.cc @@ -1,11 +1,11 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek-config.h" +#include "Formatter.h" -#include #include -#include "Formatter.h" +#include "MsgThread.h" #include "bro_inet_ntop.h" using namespace threading; diff --git a/src/threading/Formatter.h b/src/threading/Formatter.h index a3823d3a57..64a8502bc5 100644 --- a/src/threading/Formatter.h +++ b/src/threading/Formatter.h @@ -2,10 +2,18 @@ #pragma once -#include "../Desc.h" -#include "MsgThread.h" +#include "Type.h" +#include "SerialTypes.h" -namespace threading { namespace formatter { +#include + +using std::string; + +namespace threading { + +class MsgThread; + +namespace formatter { /** * A thread-safe class for converting values into some textual format. This diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index a8d721b76c..d10e38c89b 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -1,7 +1,8 @@ +#include "Manager.h" + #include #include -#include "Manager.h" #include "NetVar.h" #include "iosource/Manager.h" diff --git a/src/threading/Manager.h b/src/threading/Manager.h index 132d2238c4..c9acbbef8a 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -1,12 +1,12 @@ #pragma once -#include - -#include "BasicThread.h" #include "MsgThread.h" #include "Timer.h" +#include +#include + namespace threading { class HeartbeatTimer : public Timer { diff --git a/src/threading/formatters/Ascii.cc b/src/threading/formatters/Ascii.cc index b828616972..a8dd8efa69 100644 --- a/src/threading/formatters/Ascii.cc +++ b/src/threading/formatters/Ascii.cc @@ -2,11 +2,13 @@ #include "zeek-config.h" +#include "Ascii.h" +#include "Desc.h" +#include "threading/MsgThread.h" + #include #include -#include "./Ascii.h" - using namespace threading::formatter; // If the value we'd write out would match exactly the a reserved string, we diff --git a/src/threading/formatters/JSON.cc b/src/threading/formatters/JSON.cc index 065de3f086..ad03aaf8a0 100644 --- a/src/threading/formatters/JSON.cc +++ b/src/threading/formatters/JSON.cc @@ -2,6 +2,11 @@ #include "zeek-config.h" +#include "JSON.h" +#include "3rdparty/rapidjson/include/rapidjson/internal/ieee754.h" +#include "Desc.h" +#include "threading/MsgThread.h" + #ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS #endif @@ -11,9 +16,6 @@ #include #include -#include "JSON.h" -#include "3rdparty/rapidjson/include/rapidjson/internal/ieee754.h" - using namespace threading::formatter; bool JSON::NullDoubleWriter::Double(double d) diff --git a/src/util.cc b/src/util.cc index de3bacbe9d..1a9b01d5bb 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1,6 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek-config.h" +#include "util.h" #include "util-config.h" #ifdef TIME_WITH_SYS_TIME @@ -42,9 +43,9 @@ # include #endif +#include "Desc.h" #include "digest.h" #include "input.h" -#include "util.h" #include "Obj.h" #include "Val.h" #include "NetVar.h" diff --git a/src/zeek.bif b/src/zeek.bif index 7129add1c6..91055a1257 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -26,6 +26,7 @@ #include "iosource/PktSrc.h" #include "iosource/PktDumper.h" #include "IntrusivePtr.h" +#include "input.h" using namespace std; diff --git a/src/zeekygen/Configuration.cc b/src/zeekygen/Configuration.cc index dbbbebf578..3e7708dd6a 100644 --- a/src/zeekygen/Configuration.cc +++ b/src/zeekygen/Configuration.cc @@ -3,7 +3,6 @@ #include "Configuration.h" #include "utils.h" -#include "util.h" #include "Reporter.h" #include diff --git a/src/zeekygen/Configuration.h b/src/zeekygen/Configuration.h index 8829e5046d..669d341e6d 100644 --- a/src/zeekygen/Configuration.h +++ b/src/zeekygen/Configuration.h @@ -2,14 +2,17 @@ #pragma once -#include "Info.h" #include "Target.h" #include #include +#include // for time_t + namespace zeekygen { +class Info; + /** * Manages the generation of reStructuredText documents corresponding to * particular targets that are specified in a config file. The config file diff --git a/src/zeekygen/IdentifierInfo.cc b/src/zeekygen/IdentifierInfo.cc index 5db21ed956..cc8cd13c61 100644 --- a/src/zeekygen/IdentifierInfo.cc +++ b/src/zeekygen/IdentifierInfo.cc @@ -1,6 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "IdentifierInfo.h" +#include "ScriptInfo.h" #include "utils.h" #include "Desc.h" diff --git a/src/zeekygen/IdentifierInfo.h b/src/zeekygen/IdentifierInfo.h index 1860c0f25a..69e7e10d1c 100644 --- a/src/zeekygen/IdentifierInfo.h +++ b/src/zeekygen/IdentifierInfo.h @@ -3,8 +3,6 @@ #pragma once #include "Info.h" -#include "ScriptInfo.h" - #include "ID.h" #include "Type.h" @@ -13,6 +11,10 @@ #include #include +#include // for time_t + +class TypeDecl; + namespace zeekygen { class ScriptInfo; diff --git a/src/zeekygen/Manager.cc b/src/zeekygen/Manager.cc index 57cc19d531..555f178133 100644 --- a/src/zeekygen/Manager.cc +++ b/src/zeekygen/Manager.cc @@ -3,6 +3,10 @@ #include "Manager.h" #include "plugin/Manager.h" #include "util.h" +#include "Info.h" +#include "PackageInfo.h" +#include "ScriptInfo.h" +#include "IdentifierInfo.h" #include #include diff --git a/src/zeekygen/Manager.h b/src/zeekygen/Manager.h index 1f988c880b..988d641370 100644 --- a/src/zeekygen/Manager.h +++ b/src/zeekygen/Manager.h @@ -3,15 +3,9 @@ #pragma once #include "Configuration.h" -#include "Info.h" -#include "PackageInfo.h" -#include "ScriptInfo.h" -#include "IdentifierInfo.h" #include "Reporter.h" #include "ID.h" -#include "Type.h" -#include "Val.h" #include #include @@ -20,8 +14,13 @@ #include #include +class TypeDecl; + namespace zeekygen { +class PackageInfo; +class ScriptInfo; + /** * Map of info objects. Just a wrapper around std::map to improve code * readability (less typedefs for specific map types and not having to use diff --git a/src/zeekygen/PackageInfo.h b/src/zeekygen/PackageInfo.h index 00be437bac..b9d2591a2e 100644 --- a/src/zeekygen/PackageInfo.h +++ b/src/zeekygen/PackageInfo.h @@ -7,6 +7,8 @@ #include #include +#include // for time_t + namespace zeekygen { /** diff --git a/src/zeekygen/ScriptInfo.cc b/src/zeekygen/ScriptInfo.cc index d55b42b7bc..6e8afda0ef 100644 --- a/src/zeekygen/ScriptInfo.cc +++ b/src/zeekygen/ScriptInfo.cc @@ -5,6 +5,8 @@ #include "ReStructuredTextTable.h" #include "utils.h" #include "Manager.h" +#include "Scope.h" +#include "DebugLogger.h" #include "Reporter.h" #include "Desc.h" diff --git a/src/zeekygen/ScriptInfo.h b/src/zeekygen/ScriptInfo.h index 153fb0e54c..2ebd9b3968 100644 --- a/src/zeekygen/ScriptInfo.h +++ b/src/zeekygen/ScriptInfo.h @@ -3,7 +3,6 @@ #pragma once #include "Info.h" -#include "IdentifierInfo.h" #include #include @@ -11,6 +10,8 @@ #include #include +#include // for time_t + namespace zeekygen { class IdentifierInfo; diff --git a/src/zeekygen/Target.cc b/src/zeekygen/Target.cc index afb96cbd8b..85071b31e7 100644 --- a/src/zeekygen/Target.cc +++ b/src/zeekygen/Target.cc @@ -2,6 +2,9 @@ #include "Target.h" #include "Manager.h" +#include "IdentifierInfo.h" +#include "PackageInfo.h" +#include "ScriptInfo.h" #include "util.h" #include "Reporter.h" diff --git a/src/zeekygen/Target.h b/src/zeekygen/Target.h index 9d18f6e4c5..9aa95e58cf 100644 --- a/src/zeekygen/Target.h +++ b/src/zeekygen/Target.h @@ -2,11 +2,6 @@ #pragma once -#include "Info.h" -#include "PackageInfo.h" -#include "ScriptInfo.h" -#include "IdentifierInfo.h" - #include #include #include @@ -14,6 +9,11 @@ namespace zeekygen { +class Info; +class PackageInfo; +class ScriptInfo; +class IdentifierInfo; + /** * Helper class to create files in arbitrary file paths and automatically * close it on destruction. diff --git a/src/zeekygen/utils.cc b/src/zeekygen/utils.cc index b04790ee92..c661b84539 100644 --- a/src/zeekygen/utils.cc +++ b/src/zeekygen/utils.cc @@ -1,6 +1,8 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "utils.h" +#include "ID.h" +#include "Scope.h" #include "Reporter.h" diff --git a/src/zeekygen/utils.h b/src/zeekygen/utils.h index 27975207a3..e5ddcc49a4 100644 --- a/src/zeekygen/utils.h +++ b/src/zeekygen/utils.h @@ -2,10 +2,12 @@ #pragma once -#include "ID.h" - #include +#include // for time_t + +class ID; + namespace zeekygen { /** diff --git a/src/zeekygen/zeekygen.bif b/src/zeekygen/zeekygen.bif index d97cd782bd..289054f076 100644 --- a/src/zeekygen/zeekygen.bif +++ b/src/zeekygen/zeekygen.bif @@ -4,6 +4,9 @@ %%{ #include "zeekygen/Manager.h" +#include "zeekygen/IdentifierInfo.h" +#include "zeekygen/PackageInfo.h" +#include "zeekygen/ScriptInfo.h" #include "util.h" static StringVal* comments_to_val(const vector& comments) diff --git a/testing/btest/plugins/file-plugin/src/Foo.cc b/testing/btest/plugins/file-plugin/src/Foo.cc index 5dde4634ab..3c4fb403ab 100644 --- a/testing/btest/plugins/file-plugin/src/Foo.cc +++ b/testing/btest/plugins/file-plugin/src/Foo.cc @@ -1,5 +1,7 @@ #include "Foo.h" +#include "file_analysis/File.h" + #include #include diff --git a/testing/btest/plugins/file-plugin/src/Plugin.cc b/testing/btest/plugins/file-plugin/src/Plugin.cc index 5c61d28e28..ea38628aaa 100644 --- a/testing/btest/plugins/file-plugin/src/Plugin.cc +++ b/testing/btest/plugins/file-plugin/src/Plugin.cc @@ -1,7 +1,8 @@ -#include "Plugin.h" - #include "Foo.h" +#include "Plugin.h" +#include "file_analysis/Component.h" +#include "file_analysis/File.h" namespace plugin { namespace Demo_Foo { Plugin plugin; } } diff --git a/testing/btest/plugins/hooks-plugin/src/Plugin.cc b/testing/btest/plugins/hooks-plugin/src/Plugin.cc index 52aea76bda..473de5f5c0 100644 --- a/testing/btest/plugins/hooks-plugin/src/Plugin.cc +++ b/testing/btest/plugins/hooks-plugin/src/Plugin.cc @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace plugin { namespace Demo_Hooks { Plugin plugin; } } diff --git a/testing/btest/plugins/logging-hooks-plugin/src/Plugin.cc b/testing/btest/plugins/logging-hooks-plugin/src/Plugin.cc index eb06d5a27d..95a669aa22 100644 --- a/testing/btest/plugins/logging-hooks-plugin/src/Plugin.cc +++ b/testing/btest/plugins/logging-hooks-plugin/src/Plugin.cc @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace plugin { namespace Log_Hooks { Plugin plugin; } } diff --git a/testing/btest/plugins/pktdumper-plugin/src/Foo.cc b/testing/btest/plugins/pktdumper-plugin/src/Foo.cc index c68eec809a..99a269c02e 100644 --- a/testing/btest/plugins/pktdumper-plugin/src/Foo.cc +++ b/testing/btest/plugins/pktdumper-plugin/src/Foo.cc @@ -1,9 +1,10 @@ +#include "Foo.h" +#include "iosource/Packet.h" + #include #include -#include "Foo.h" - using namespace plugin::Demo_Foo; Foo::Foo(const std::string& path, bool is_live) diff --git a/testing/btest/plugins/pktdumper-plugin/src/Plugin.cc b/testing/btest/plugins/pktdumper-plugin/src/Plugin.cc index f4417ff6a2..1836ad7b4a 100644 --- a/testing/btest/plugins/pktdumper-plugin/src/Plugin.cc +++ b/testing/btest/plugins/pktdumper-plugin/src/Plugin.cc @@ -2,6 +2,8 @@ #include "Plugin.h" #include "Foo.h" +#include "iosource/Component.h" + namespace plugin { namespace Demo_Foo { Plugin plugin; } } diff --git a/testing/btest/plugins/pktsrc-plugin/src/Foo.cc b/testing/btest/plugins/pktsrc-plugin/src/Foo.cc index b2b768b4ba..012b1f226a 100644 --- a/testing/btest/plugins/pktsrc-plugin/src/Foo.cc +++ b/testing/btest/plugins/pktsrc-plugin/src/Foo.cc @@ -1,9 +1,13 @@ +#include "Foo.h" + +extern "C" { +#include +} + #include #include -#include "Foo.h" - using namespace plugin::Demo_Foo; Foo::Foo(const std::string& path, bool is_live) diff --git a/testing/btest/plugins/pktsrc-plugin/src/Plugin.cc b/testing/btest/plugins/pktsrc-plugin/src/Plugin.cc index 088a4dd36d..6723651cce 100644 --- a/testing/btest/plugins/pktsrc-plugin/src/Plugin.cc +++ b/testing/btest/plugins/pktsrc-plugin/src/Plugin.cc @@ -2,6 +2,7 @@ #include "Plugin.h" #include "Foo.h" +#include "iosource/Component.h" namespace plugin { namespace Demo_Foo { Plugin plugin; } } diff --git a/testing/btest/plugins/protocol-plugin/src/Plugin.cc b/testing/btest/plugins/protocol-plugin/src/Plugin.cc index bd2662d67c..5a7bfb63b2 100644 --- a/testing/btest/plugins/protocol-plugin/src/Plugin.cc +++ b/testing/btest/plugins/protocol-plugin/src/Plugin.cc @@ -1,5 +1,6 @@ #include "Plugin.h" +#include "analyzer/Component.h" #include "Foo.h" diff --git a/testing/btest/plugins/reporter-hook-plugin/src/Plugin.cc b/testing/btest/plugins/reporter-hook-plugin/src/Plugin.cc index d9c856966a..375d9951aa 100644 --- a/testing/btest/plugins/reporter-hook-plugin/src/Plugin.cc +++ b/testing/btest/plugins/reporter-hook-plugin/src/Plugin.cc @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace plugin { namespace Reporter_Hook { Plugin plugin; } } diff --git a/testing/btest/plugins/writer-plugin/src/Foo.h b/testing/btest/plugins/writer-plugin/src/Foo.h index 64ca166b63..5a3e336fbf 100644 --- a/testing/btest/plugins/writer-plugin/src/Foo.h +++ b/testing/btest/plugins/writer-plugin/src/Foo.h @@ -3,6 +3,7 @@ #include "logging/WriterBackend.h" #include "threading/formatters/Ascii.h" +#include "Desc.h" namespace logging { namespace writer { From 6a815b4b060913d0bc8c8ae4f9b09951727eb445 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Sun, 2 Feb 2020 16:29:27 +0100 Subject: [PATCH 49/53] UID, ..: un-inline methods to reduce header dependencies Only 1% build time speedup, but still, it declutters the headers a bit. Before this patch: 2565.17user 141.83system 2:25.46elapsed 1860%CPU (0avgtext+0avgdata 1489076maxresident)k 72576inputs+9130920outputs (1667major+49400430minor)pagefaults 0swaps After this patch: 2537.19user 142.94system 2:26.90elapsed 1824%CPU (0avgtext+0avgdata 1434268maxresident)k 16240inputs+8887152outputs (1931major+48728888minor)pagefaults 0swaps --- src/Anon.cc | 8 ++++ src/Anon.h | 9 +---- src/Base64.cc | 11 +++++- src/Base64.h | 15 +++---- src/BroString.cc | 6 +++ src/BroString.h | 5 +-- src/CCL.cc | 5 +++ src/CCL.h | 3 +- src/Event.cc | 21 ++++++++++ src/Event.h | 20 ++-------- src/Expr.cc | 25 ++++++++++++ src/Expr.h | 17 +++----- src/Frame.cc | 9 +++++ src/Frame.h | 10 +---- src/ID.cc | 22 +++++++++++ src/ID.h | 11 ++---- src/IP.cc | 71 +++++++++++++++++++++++++++++++++ src/IP.h | 72 ++++++++-------------------------- src/IPAddr.cc | 25 ++++++++++++ src/IPAddr.h | 27 +++---------- src/RuleAction.cc | 11 ++++++ src/RuleAction.h | 7 +--- src/Stmt.cc | 15 +++++++ src/Stmt.h | 16 +------- src/Trigger.cc | 5 +++ src/Trigger.h | 5 +-- src/Type.cc | 7 ++++ src/Type.h | 7 +--- src/UID.cc | 14 +++++++ src/UID.h | 13 ------ src/Val.cc | 15 +++++++ src/Val.h | 11 +----- src/broker/Data.cc | 8 ++++ src/broker/Data.h | 10 ++--- src/iosource/Packet.cc | 7 +++- src/iosource/Packet.h | 7 ++-- src/zeekygen/IdentifierInfo.cc | 5 +++ src/zeekygen/IdentifierInfo.h | 4 +- 38 files changed, 348 insertions(+), 211 deletions(-) diff --git a/src/Anon.cc b/src/Anon.cc index 583d11016b..0c511ee158 100644 --- a/src/Anon.cc +++ b/src/Anon.cc @@ -9,6 +9,7 @@ #include "net_util.h" #include "Val.h" #include "NetVar.h" +#include "Reporter.h" AnonymizeIPAddr* ip_anonymizer[NUM_ADDR_ANONYMIZATION_METHODS] = {0}; @@ -67,6 +68,13 @@ ipaddr32_t AnonymizeIPAddr::Anonymize(ipaddr32_t addr) } } +// Keep the specified prefix unchanged. +int AnonymizeIPAddr::PreservePrefix(ipaddr32_t /* input */, int /* num_bits */) + { + reporter->InternalError("prefix preserving is not supported for the anonymizer"); + return 0; + } + int AnonymizeIPAddr::PreserveNet(ipaddr32_t input) { switch ( addr_to_class(ntohl(input)) ) { diff --git a/src/Anon.h b/src/Anon.h index cce4ab1922..6c3125f94c 100644 --- a/src/Anon.h +++ b/src/Anon.h @@ -13,8 +13,6 @@ #include #include -#include "Reporter.h" - using std::map; // TODO: Anon.h may not be the right place to put these functions ... @@ -46,12 +44,7 @@ public: ipaddr32_t Anonymize(ipaddr32_t addr); - // Keep the specified prefix unchanged. - virtual int PreservePrefix(ipaddr32_t /* input */, int /* num_bits */) - { - reporter->InternalError("prefix preserving is not supported for the anonymizer"); - return 0; - } + virtual int PreservePrefix(ipaddr32_t input, int num_bits); virtual ipaddr32_t anonymize(ipaddr32_t addr) = 0; diff --git a/src/Base64.cc b/src/Base64.cc index 11455ef150..95e140e6c1 100644 --- a/src/Base64.cc +++ b/src/Base64.cc @@ -1,6 +1,8 @@ #include "zeek-config.h" #include "Base64.h" #include "BroString.h" +#include "Reporter.h" +#include "Conn.h" #include @@ -217,6 +219,14 @@ int Base64Converter::Done(int* pblen, char** pbuf) return 0; } +void Base64Converter::IllegalEncoding(const char* msg) + { + // strncpy(error_msg, msg, sizeof(error_msg)); + if ( conn ) + conn->Weird("base64_illegal_encoding", msg); + else + reporter->Error("%s", msg); + } BroString* decode_base64(const BroString* s, const BroString* a, Connection* conn) { @@ -268,4 +278,3 @@ BroString* encode_base64(const BroString* s, const BroString* a, Connection* con return new BroString(1, (u_char*)outbuf, outlen); } - diff --git a/src/Base64.h b/src/Base64.h index c829203851..eaa6921c48 100644 --- a/src/Base64.h +++ b/src/Base64.h @@ -1,9 +1,11 @@ #pragma once -#include "Reporter.h" -#include "Conn.h" +#include + +using std::string; class BroString; +class Connection; // Maybe we should have a base class for generic decoders? class Base64Converter { @@ -36,14 +38,7 @@ public: int Errored() const { return errored; } const char* ErrorMsg() const { return error_msg; } - void IllegalEncoding(const char* msg) - { - // strncpy(error_msg, msg, sizeof(error_msg)); - if ( conn ) - conn->Weird("base64_illegal_encoding", msg); - else - reporter->Error("%s", msg); - } + void IllegalEncoding(const char* msg); protected: char error_msg[256]; diff --git a/src/BroString.cc b/src/BroString.cc index 788bbf6d75..ddd1b8093f 100644 --- a/src/BroString.cc +++ b/src/BroString.cc @@ -10,6 +10,7 @@ #include "Val.h" #include "Var.h" #include "Reporter.h" +#include "util.h" #ifdef DEBUG #define DEBUG_STR(msg) DBG_LOG(DBG_STRING, msg) @@ -274,6 +275,11 @@ void BroString::ToUpper() b[i] = toupper(b[i]); } +unsigned int BroString::MemoryAllocation() const + { + return padded_sizeof(*this) + pad_size(n + final_NUL); + } + BroString* BroString::GetSubstring(int start, int len) const { // This code used to live in zeek.bif's sub_bytes() routine. diff --git a/src/BroString.h b/src/BroString.h index df9f216ecf..40a1029291 100644 --- a/src/BroString.h +++ b/src/BroString.h @@ -2,8 +2,6 @@ #pragma once -#include "util.h" - #include #include #include @@ -114,8 +112,7 @@ public: // XXX and to_upper; the latter doesn't use BroString::ToUpper(). void ToUpper(); - unsigned int MemoryAllocation() const - { return padded_sizeof(*this) + pad_size(n + final_NUL); } + unsigned int MemoryAllocation() const; // Returns new string containing the substring of this string, // starting at @start >= 0 for going up to @length elements, diff --git a/src/CCL.cc b/src/CCL.cc index 34809843ef..732d386163 100644 --- a/src/CCL.cc +++ b/src/CCL.cc @@ -43,3 +43,8 @@ void CCL::Sort() { std::sort(syms->begin(), syms->end()); } + +unsigned int CCL::MemoryAllocation() const + { + return padded_sizeof(*this) + padded_sizeof(*syms) + pad_size(syms->size() * sizeof(int_list::value_type)); + } diff --git a/src/CCL.h b/src/CCL.h index 51fa901be8..58c04688d9 100644 --- a/src/CCL.h +++ b/src/CCL.h @@ -25,8 +25,7 @@ public: void ReplaceSyms(int_list* new_syms) { delete syms; syms = new_syms; } - unsigned int MemoryAllocation() const - { return padded_sizeof(*this) + padded_sizeof(*syms) + pad_size(syms->size() * sizeof(int_list::value_type)); } + unsigned int MemoryAllocation() const; protected: int_list* syms; diff --git a/src/Event.cc b/src/Event.cc index 98b4f5455c..89fdd5d907 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -7,6 +7,7 @@ #include "Func.h" #include "NetVar.h" #include "Trigger.h" +#include "Val.h" #include "plugin/Manager.h" EventMgr mgr; @@ -102,6 +103,19 @@ EventMgr::~EventMgr() Unref(src_val); } +void EventMgr::QueueEvent(const EventHandlerPtr &h, val_list vl, + SourceID src, analyzer::ID aid, + TimerMgr* mgr, BroObj* obj) + { + if ( h ) + QueueEvent(new Event(h, std::move(vl), src, aid, mgr, obj)); + else + { + for ( const auto& v : vl ) + Unref(v); + } + } + void EventMgr::QueueEvent(Event* event) { bool done = PLUGIN_HOOK_WITH_RESULT(HOOK_QUEUE_EVENT, HookQueueEvent(event), false); @@ -120,6 +134,13 @@ void EventMgr::QueueEvent(Event* event) ++num_events_queued; } +void EventMgr::Dispatch(Event* event, bool no_remote) + { + current_src = event->Source(); + event->Dispatch(no_remote); + Unref(event); + } + void EventMgr::Drain() { if ( event_queue_flush_point ) diff --git a/src/Event.h b/src/Event.h index 3d367ef034..1482ecb83e 100644 --- a/src/Event.h +++ b/src/Event.h @@ -2,8 +2,8 @@ #pragma once +#include "BroList.h" #include "analyzer/Analyzer.h" -#include "Val.h" class EventMgr; @@ -77,16 +77,7 @@ public: // existence check. void QueueEvent(const EventHandlerPtr &h, val_list vl, SourceID src = SOURCE_LOCAL, analyzer::ID aid = 0, - TimerMgr* mgr = 0, BroObj* obj = 0) - { - if ( h ) - QueueEvent(new Event(h, std::move(vl), src, aid, mgr, obj)); - else - { - for ( const auto& v : vl ) - Unref(v); - } - } + TimerMgr* mgr = 0, BroObj* obj = 0); // Same as QueueEvent, except taking the event's argument list via a // pointer instead of by value. This function takes ownership of the @@ -100,12 +91,7 @@ public: delete vl; } - void Dispatch(Event* event, bool no_remote = false) - { - current_src = event->Source(); - event->Dispatch(no_remote); - Unref(event); - } + void Dispatch(Event* event, bool no_remote = false); void Drain(); bool IsDraining() const { return draining; } diff --git a/src/Expr.cc b/src/Expr.cc index b21029d945..9aff73cdb1 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -135,12 +135,32 @@ Val* Expr::InitVal(const BroType* t, Val* aggr) const return check_and_promote(Eval(0), t, 1); } +int Expr::IsError() const + { + return type && type->Tag() == TYPE_ERROR; + } + +void Expr::SetError() + { + SetType(error_type()); + } + void Expr::SetError(const char* msg) { Error(msg); SetError(); } +int Expr::IsZero() const + { + return IsConst() && ExprVal()->IsZero(); + } + +int Expr::IsOne() const + { + return IsConst() && ExprVal()->IsOne(); + } + void Expr::Describe(ODesc* d) const { if ( IsParen() && ! d->IsBinary() ) @@ -2079,6 +2099,11 @@ AssignExpr::AssignExpr(Expr* arg_op1, Expr* arg_op2, int arg_is_init, SetLocationInfo(arg_op1->GetLocationInfo(), arg_op2->GetLocationInfo()); } +AssignExpr::~AssignExpr() + { + Unref(val); + } + bool AssignExpr::TypeCheck(attr_list* attrs) { TypeTag bt1 = op1->Type()->Tag(); diff --git a/src/Expr.h b/src/Expr.h index 52bc32b743..701e826b19 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -7,7 +7,6 @@ #include "BroList.h" #include "Timer.h" #include "Type.h" -#include "Val.h" #include "EventHandler.h" #include "TraverseTypes.h" @@ -119,10 +118,10 @@ public: int IsConst() const { return tag == EXPR_CONST; } // True if the expression is in error (to alleviate error propagation). - int IsError() const { return type && type->Tag() == TYPE_ERROR; } + int IsError() const; // Mark expression as in error. - void SetError() { SetType(error_type()); } + void SetError(); void SetError(const char* msg); // Returns the expression's constant value, or complains @@ -130,16 +129,10 @@ public: inline Val* ExprVal() const; // True if the expression is a constant zero, false otherwise. - int IsZero() const - { - return IsConst() && ExprVal()->IsZero(); - } + int IsZero() const; // True if the expression is a constant one, false otherwise. - int IsOne() const - { - return IsConst() && ExprVal()->IsOne(); - } + int IsOne() const; // True if the expression supports the "add" or "delete" operations, // false otherwise. @@ -605,7 +598,7 @@ public: // If val is given, evaluating this expression will always yield the val // yet still perform the assignment. Used for triggers. AssignExpr(Expr* op1, Expr* op2, int is_init, Val* val = 0, attr_list* attrs = 0); - ~AssignExpr() override { Unref(val); } + ~AssignExpr() override; Val* Eval(Frame* f) const override; void EvalIntoAggregate(const BroType* t, Val* aggr, Frame* f) const override; diff --git a/src/Frame.cc b/src/Frame.cc index a8e6ac1e65..e6b42ddd04 100644 --- a/src/Frame.cc +++ b/src/Frame.cc @@ -9,6 +9,7 @@ #include "Desc.h" #include "IntrusivePtr.h" #include "Trigger.h" +#include "Val.h" vector g_frame_stack; @@ -535,6 +536,14 @@ void Frame::ClearTrigger() trigger = nullptr; } +void Frame::UnrefElement(int n) + { + if ( weak_refs && weak_refs[n] ) + return; + + Unref(frame[n]); + } + bool Frame::IsOuterID(const ID* in) const { return std::any_of(outer_ids.begin(), outer_ids.end(), diff --git a/src/Frame.h b/src/Frame.h index fabe2a9e19..8e9cbdc4e7 100644 --- a/src/Frame.h +++ b/src/Frame.h @@ -3,7 +3,7 @@ #pragma once #include "BroList.h" // for typedef val_list -#include "Val.h" +#include "Obj.h" #include #include @@ -235,13 +235,7 @@ private: /** * Unrefs the value at offset 'n' frame unless it's a weak reference. */ - void UnrefElement(int n) - { - if ( weak_refs && weak_refs[n] ) - return; - - Unref(frame[n]); - } + void UnrefElement(int n); /** Have we captured this id? */ bool IsOuterID(const ID* in) const; diff --git a/src/ID.cc b/src/ID.cc index 793c7bc7ff..d6b949fc59 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -3,12 +3,14 @@ #include "zeek-config.h" #include "ID.h" +#include "Attr.h" #include "Desc.h" #include "Expr.h" #include "Dict.h" #include "EventRegistry.h" #include "Func.h" #include "Scope.h" +#include "Type.h" #include "File.h" #include "Scope.h" #include "Traverse.h" @@ -56,6 +58,11 @@ string ID::ModuleName() const return extract_module_name(name); } +void ID::SetType(BroType* t) + { + Unref(type); type = t; + } + void ID::ClearVal() { if ( ! weak_ref ) @@ -148,6 +155,11 @@ void ID::SetVal(Expr* ev, init_class c) EvalFunc(a->AttrExpr(), ev); } +bool ID::IsRedefinable() const + { + return FindAttr(ATTR_REDEF) != 0; + } + void ID::SetAttrs(Attributes* a) { Unref(attrs); @@ -194,6 +206,16 @@ void ID::UpdateValAttrs() } } +Attr* ID::FindAttr(attr_tag t) const + { + return attrs ? attrs->FindAttr(t) : 0; + } + +bool ID::IsDeprecated() const + { + return FindAttr(ATTR_DEPRECATED) != 0; + } + void ID::MakeDeprecated(Expr* deprecation) { if ( IsDeprecated() ) diff --git a/src/ID.h b/src/ID.h index f9d7f01e40..7b6a1ada6c 100644 --- a/src/ID.h +++ b/src/ID.h @@ -3,7 +3,6 @@ #pragma once #include "Obj.h" -#include "Type.h" #include "Attr.h" #include "Notifier.h" #include "TraverseTypes.h" @@ -36,7 +35,7 @@ public: std::string ModuleName() const; - void SetType(BroType* t) { Unref(type); type = t; } + void SetType(BroType* t); BroType* Type() { return type; } const BroType* Type() const { return type; } @@ -74,7 +73,7 @@ public: void SetOffset(int arg_offset) { offset = arg_offset; } int Offset() const { return offset; } - bool IsRedefinable() const { return FindAttr(ATTR_REDEF) != 0; } + bool IsRedefinable() const; void SetAttrs(Attributes* attr); void AddAttrs(Attributes* attr); @@ -82,11 +81,9 @@ public: void UpdateValAttrs(); Attributes* Attrs() const { return attrs; } - Attr* FindAttr(attr_tag t) const - { return attrs ? attrs->FindAttr(t) : 0; } + Attr* FindAttr(attr_tag t) const; - bool IsDeprecated() const - { return FindAttr(ATTR_DEPRECATED) != 0; } + bool IsDeprecated() const; void MakeDeprecated(Expr* deprecation); diff --git a/src/IP.cc b/src/IP.cc index bc7c90262a..24ffac2b17 100644 --- a/src/IP.cc +++ b/src/IP.cc @@ -6,9 +6,11 @@ #include #include +#include "IPAddr.h" #include "Type.h" #include "Val.h" #include "Var.h" +#include "Reporter.h" static RecordType* ip4_hdr_type = 0; static RecordType* ip6_hdr_type = 0; @@ -305,6 +307,26 @@ RecordVal* IPv6_Hdr::BuildRecordVal(VectorVal* chain) const return rv; } +IPAddr IP_Hdr::IPHeaderSrcAddr() const + { + return ip4 ? IPAddr(ip4->ip_src) : IPAddr(ip6->ip6_src); + } + +IPAddr IP_Hdr::IPHeaderDstAddr() const + { + return ip4 ? IPAddr(ip4->ip_dst) : IPAddr(ip6->ip6_dst); + } + +IPAddr IP_Hdr::SrcAddr() const + { + return ip4 ? IPAddr(ip4->ip_src) : ip6_hdrs->SrcAddr(); + } + +IPAddr IP_Hdr::DstAddr() const + { + return ip4 ? IPAddr(ip4->ip_dst) : ip6_hdrs->DstAddr(); + } + RecordVal* IP_Hdr::BuildIPHdrVal() const { RecordVal* rval = 0; @@ -447,6 +469,15 @@ static inline bool isIPv6ExtHeader(uint8_t type) } } +IPv6_Hdr_Chain::~IPv6_Hdr_Chain() + { + for ( size_t i = 0; i < chain.size(); ++i ) delete chain[i]; +#ifdef ENABLE_MOBILE_IPV6 + delete homeAddr; +#endif + delete finalDst; + } + void IPv6_Hdr_Chain::Init(const struct ip6_hdr* ip6, int total_len, bool set_next, uint16_t next) { @@ -511,6 +542,46 @@ void IPv6_Hdr_Chain::Init(const struct ip6_hdr* ip6, int total_len, isIPv6ExtHeader(next_type) ); } +bool IPv6_Hdr_Chain::IsFragment() const + { + if ( chain.empty() ) + { + reporter->InternalWarning("empty IPv6 header chain"); + return false; + } + + return chain[chain.size()-1]->Type() == IPPROTO_FRAGMENT; + } + +IPAddr IPv6_Hdr_Chain::SrcAddr() const + { +#ifdef ENABLE_MOBILE_IPV6 + if ( homeAddr ) + return IPAddr(*homeAddr); +#endif + if ( chain.empty() ) + { + reporter->InternalWarning("empty IPv6 header chain"); + return IPAddr(); + } + + return IPAddr(((const struct ip6_hdr*)(chain[0]->Data()))->ip6_src); + } + +IPAddr IPv6_Hdr_Chain::DstAddr() const + { + if ( finalDst ) + return IPAddr(*finalDst); + + if ( chain.empty() ) + { + reporter->InternalWarning("empty IPv6 header chain"); + return IPAddr(); + } + + return IPAddr(((const struct ip6_hdr*)(chain[0]->Data()))->ip6_dst); + } + void IPv6_Hdr_Chain::ProcessRoutingHeader(const struct ip6_rthdr* r, uint16_t len) { if ( finalDst ) diff --git a/src/IP.h b/src/IP.h index 985f23c0b4..9067e45c43 100644 --- a/src/IP.h +++ b/src/IP.h @@ -3,16 +3,23 @@ #pragma once #include "zeek-config.h" -#include "IPAddr.h" -#include "Reporter.h" + #include #include // for u_char #include #include +#ifdef HAVE_NETINET_IP6_H +#include +#endif + using std::vector; +class IPAddr; +class RecordVal; +class VectorVal; + #ifdef ENABLE_MOBILE_IPV6 #ifndef IPPROTO_MOBILITY @@ -148,14 +155,7 @@ public: finalDst(0) { Init(ip6, len, false); } - ~IPv6_Hdr_Chain() - { - for ( size_t i = 0; i < chain.size(); ++i ) delete chain[i]; -#ifdef ENABLE_MOBILE_IPV6 - delete homeAddr; -#endif - delete finalDst; - } + ~IPv6_Hdr_Chain(); /** * @return a copy of the header chain, but with pointers to individual @@ -181,16 +181,7 @@ public: /** * Returns whether the header chain indicates a fragmented packet. */ - bool IsFragment() const - { - if ( chain.empty() ) - { - reporter->InternalWarning("empty IPv6 header chain"); - return false; - } - - return chain[chain.size()-1]->Type() == IPPROTO_FRAGMENT; - } + bool IsFragment() const; /** * Returns pointer to fragment header structure if the chain contains one. @@ -225,39 +216,14 @@ public: * option as defined by Mobile IPv6 (RFC 6275), then return it, else * return the source address in the main IPv6 header. */ - IPAddr SrcAddr() const - { -#ifdef ENABLE_MOBILE_IPV6 - if ( homeAddr ) - return IPAddr(*homeAddr); -#endif - if ( chain.empty() ) - { - reporter->InternalWarning("empty IPv6 header chain"); - return IPAddr(); - } - - return IPAddr(((const struct ip6_hdr*)(chain[0]->Data()))->ip6_src); - } + IPAddr SrcAddr() const; /** * If the chain contains a Routing header with non-zero segments left, * then return the last address of the first such header, else return * the destination address of the main IPv6 header. */ - IPAddr DstAddr() const - { - if ( finalDst ) - return IPAddr(*finalDst); - - if ( chain.empty() ) - { - reporter->InternalWarning("empty IPv6 header chain"); - return IPAddr(); - } - - return IPAddr(((const struct ip6_hdr*)(chain[0]->Data()))->ip6_dst); - } + IPAddr DstAddr() const; /** * Returns a vector of ip6_ext_hdr RecordVals that includes script-layer @@ -401,22 +367,19 @@ public: /** * Returns the source address held in the IP header. */ - IPAddr IPHeaderSrcAddr() const - { return ip4 ? IPAddr(ip4->ip_src) : IPAddr(ip6->ip6_src); } + IPAddr IPHeaderSrcAddr() const; /** * Returns the destination address held in the IP header. */ - IPAddr IPHeaderDstAddr() const - { return ip4 ? IPAddr(ip4->ip_dst) : IPAddr(ip6->ip6_dst); } + IPAddr IPHeaderDstAddr() const; /** * For IPv4 or IPv6 headers that don't contain a Home Address option * (Mobile IPv6, RFC 6275), return source address held in the IP header. * For IPv6 headers that contain a Home Address option, return that address. */ - IPAddr SrcAddr() const - { return ip4 ? IPAddr(ip4->ip_src) : ip6_hdrs->SrcAddr(); } + IPAddr SrcAddr() const; /** * For IPv4 or IPv6 headers that don't contain a Routing header with @@ -424,8 +387,7 @@ public: * For IPv6 headers with a Routing header that has non-zero segments left, * return the last address in the first such Routing header. */ - IPAddr DstAddr() const - { return ip4 ? IPAddr(ip4->ip_dst) : ip6_hdrs->DstAddr(); } + IPAddr DstAddr() const; /** * Returns a pointer to the payload of the IP packet, usually an diff --git a/src/IPAddr.cc b/src/IPAddr.cc index 405d8b335a..cbcb8dc867 100644 --- a/src/IPAddr.cc +++ b/src/IPAddr.cc @@ -5,7 +5,9 @@ #include #include "IPAddr.h" #include "Reporter.h" +#include "BroString.h" #include "Conn.h" +#include "Hash.h" #include "bro_inet_ntop.h" #include "analyzer/Manager.h" @@ -45,6 +47,16 @@ ConnIDKey BuildConnIDKey(const ConnID& id) return key; } +IPAddr::IPAddr(const BroString& s) + { + Init(s.CheckString()); + } + +HashKey* IPAddr::GetHashKey() const + { + return new HashKey((void*)in6.s6_addr, sizeof(in6.s6_addr)); + } + static inline uint32_t bit_mask32(int bottom_bits) { if ( bottom_bits >= 32 ) @@ -290,6 +302,19 @@ string IPPrefix::AsString() const return prefix.AsString() +"/" + l; } +HashKey* IPPrefix::GetHashKey() const + { + struct { + in6_addr ip; + uint32_t len; + } key; + + key.ip = prefix.in6; + key.len = Length(); + + return new HashKey(&key, sizeof(key)); + } + bool IPPrefix::ConvertString(const char* text, IPPrefix* result) { string s(text); diff --git a/src/IPAddr.h b/src/IPAddr.h index 238ba10407..04ecf5e33e 100644 --- a/src/IPAddr.h +++ b/src/IPAddr.h @@ -2,8 +2,6 @@ #pragma once -#include "BroString.h" -#include "Hash.h" #include "threading/SerialTypes.h" #include @@ -13,6 +11,8 @@ using std::string; struct ConnID; +class BroString; +class HashKey; namespace analyzer { class ExpectedConn; } typedef in_addr in4_addr; @@ -112,10 +112,7 @@ public: * @param s String containing an IP address as either a dotted IPv4 * address or a hex IPv6 address. */ - explicit IPAddr(const BroString& s) - { - Init(s.CheckString()); - } + explicit IPAddr(const BroString& s); /** * Constructs an address instance from a raw byte representation. @@ -254,10 +251,7 @@ public: * Returns a key that can be used to lookup the IP Address in a hash * table. Passes ownership to caller. */ - HashKey* GetHashKey() const - { - return new HashKey((void*)in6.s6_addr, sizeof(in6.s6_addr)); - } + HashKey* GetHashKey() const; /** * Masks out lower bits of the address. @@ -639,18 +633,7 @@ public: * Returns a key that can be used to lookup the IP Prefix in a hash * table. Passes ownership to caller. */ - HashKey* GetHashKey() const - { - struct { - in6_addr ip; - uint32_t len; - } key; - - key.ip = prefix.in6; - key.len = Length(); - - return new HashKey(&key, sizeof(key)); - } + HashKey* GetHashKey() const; /** Converts the prefix into the type used internally by the * inter-thread communication. diff --git a/src/RuleAction.cc b/src/RuleAction.cc index edfe2497a2..5419e68a4c 100644 --- a/src/RuleAction.cc +++ b/src/RuleAction.cc @@ -12,6 +12,11 @@ using std::string; #include "analyzer/Manager.h" +RuleActionEvent::RuleActionEvent(const char* arg_msg) + { + msg = copy_string(arg_msg); + } + void RuleActionEvent::DoAction(const Rule* parent, RuleEndpointState* state, const u_char* data, int len) { @@ -30,6 +35,12 @@ void RuleActionEvent::PrintDebug() fprintf(stderr, " RuleActionEvent: |%s|\n", msg); } +RuleActionMIME::RuleActionMIME(const char* arg_mime, int arg_strength) + { + mime = copy_string(arg_mime); + strength = arg_strength; + } + void RuleActionMIME::PrintDebug() { fprintf(stderr, " RuleActionMIME: |%s|\n", mime); diff --git a/src/RuleAction.h b/src/RuleAction.h index 8ffc097fc6..8604fa89a8 100644 --- a/src/RuleAction.h +++ b/src/RuleAction.h @@ -1,7 +1,5 @@ #pragma once -#include "util.h" - #include "analyzer/Tag.h" #include @@ -27,7 +25,7 @@ public: // Implements the "event" keyword. class RuleActionEvent : public RuleAction { public: - explicit RuleActionEvent(const char* arg_msg) { msg = copy_string(arg_msg); } + explicit RuleActionEvent(const char* arg_msg); ~RuleActionEvent() override { delete [] msg; } void DoAction(const Rule* parent, RuleEndpointState* state, @@ -41,8 +39,7 @@ private: class RuleActionMIME : public RuleAction { public: - explicit RuleActionMIME(const char* arg_mime, int arg_strength = 0) - { mime = copy_string(arg_mime); strength = arg_strength; } + explicit RuleActionMIME(const char* arg_mime, int arg_strength = 0); ~RuleActionMIME() override { delete [] mime; } diff --git a/src/Stmt.cc b/src/Stmt.cc index 6b98b0322e..8766a1a75a 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -93,6 +93,14 @@ void Stmt::Describe(ODesc* d) const AddTag(d); } +void Stmt::DecrBPCount() + { + if ( breakpoint_count ) + --breakpoint_count; + else + reporter->InternalError("breakpoint count decremented below 0"); + } + void Stmt::AddTag(ODesc* d) const { if ( d->IsBinary() ) @@ -1643,6 +1651,13 @@ void EventBodyList::Describe(ODesc* d) const StmtList::Describe(d); } +InitStmt::InitStmt(id_list* arg_inits) : Stmt(STMT_INIT) + { + inits = arg_inits; + if ( arg_inits && arg_inits->length() ) + SetLocationInfo((*arg_inits)[0]->GetLocationInfo()); + } + InitStmt::~InitStmt() { for ( const auto& init : *inits ) diff --git a/src/Stmt.h b/src/Stmt.h index a0ac3a4543..a65af2ee88 100644 --- a/src/Stmt.h +++ b/src/Stmt.h @@ -8,7 +8,6 @@ #include "Dict.h" #include "ID.h" #include "Obj.h" -#include "Reporter.h" #include "StmtEnums.h" @@ -63,13 +62,7 @@ public: void Describe(ODesc* d) const override; virtual void IncrBPCount() { ++breakpoint_count; } - virtual void DecrBPCount() - { - if ( breakpoint_count ) - --breakpoint_count; - else - reporter->InternalError("breakpoint count decremented below 0"); - } + virtual void DecrBPCount(); virtual unsigned int BPCount() const { return breakpoint_count; } @@ -436,12 +429,7 @@ protected: class InitStmt : public Stmt { public: - explicit InitStmt(id_list* arg_inits) : Stmt(STMT_INIT) - { - inits = arg_inits; - if ( arg_inits && arg_inits->length() ) - SetLocationInfo((*arg_inits)[0]->GetLocationInfo()); - } + explicit InitStmt(id_list* arg_inits); ~InitStmt() override; diff --git a/src/Trigger.cc b/src/Trigger.cc index 357bd5836f..5a600074a6 100644 --- a/src/Trigger.cc +++ b/src/Trigger.cc @@ -475,6 +475,11 @@ void Trigger::Disable() disabled = true; } +void Trigger::Describe(ODesc* d) const + { + d->Add(""); + } + void Trigger::Modified(notifier::Modifiable* m) { trigger_mgr->Queue(this); diff --git a/src/Trigger.h b/src/Trigger.h index 9296449778..685f8ec129 100644 --- a/src/Trigger.h +++ b/src/Trigger.h @@ -1,7 +1,6 @@ #pragma once #include "Obj.h" -#include "Desc.h" #include "Notifier.h" #include "iosource/IOSource.h" @@ -15,6 +14,7 @@ class Stmt; class Frame; class Val; class ID; +class ODesc; namespace trigger { // Triggers are the heart of "when" statements: expressions that when @@ -69,8 +69,7 @@ public: bool Disabled() const { return disabled; } - void Describe(ODesc* d) const override - { d->Add(""); } + void Describe(ODesc* d) const override; // Overidden from Notifier. We queue the trigger and evaluate it // later to avoid race conditions. diff --git a/src/Type.cc b/src/Type.cc index 501a27ad8d..f9bb36db01 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -273,6 +273,13 @@ void TypeList::Describe(ODesc* d) const } } +unsigned int TypeList::MemoryAllocation() const + { + return BroType::MemoryAllocation() + + padded_sizeof(*this) - padded_sizeof(BroType) + + types.MemoryAllocation() - padded_sizeof(types); + } + IndexType::~IndexType() { Unref(indices); diff --git a/src/Type.h b/src/Type.h index 02c4fbb592..4ea3caf318 100644 --- a/src/Type.h +++ b/src/Type.h @@ -318,12 +318,7 @@ public: void Describe(ODesc* d) const override; - unsigned int MemoryAllocation() const override - { - return BroType::MemoryAllocation() - + padded_sizeof(*this) - padded_sizeof(BroType) - + types.MemoryAllocation() - padded_sizeof(types); - } + unsigned int MemoryAllocation() const override; protected: BroType* pure_type; diff --git a/src/UID.cc b/src/UID.cc index cff534edfb..73d61873be 100644 --- a/src/UID.cc +++ b/src/UID.cc @@ -1,6 +1,8 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "UID.h" +#include "Reporter.h" +#include "util.h" #include @@ -26,3 +28,15 @@ void UID::Set(bro_uint_t bits, const uint64_t* v, size_t n) if ( res.rem ) uid[0] >>= 64 - res.rem; } + +std::string UID::Base62(std::string prefix) const + { + if ( ! initialized ) + reporter->InternalError("use of uninitialized UID"); + + char tmp[sizeof(uid) * 8 + 1]; // enough for even binary representation + for ( size_t i = 0; i < BRO_UID_LEN; ++i ) + prefix.append(uitoa_n(uid[i], tmp, sizeof(tmp), 62)); + + return prefix; + } diff --git a/src/UID.h b/src/UID.h index 2fa39badc2..711541e283 100644 --- a/src/UID.h +++ b/src/UID.h @@ -2,7 +2,6 @@ #pragma once -#include "Reporter.h" #include "util.h" // for bro_int_t #include @@ -99,16 +98,4 @@ inline UID& UID::operator=(const UID& other) return *this; } -inline std::string UID::Base62(std::string prefix) const - { - if ( ! initialized ) - reporter->InternalError("use of uninitialized UID"); - - char tmp[sizeof(uid) * 8 + 1]; // enough for even binary representation - for ( size_t i = 0; i < BRO_UID_LEN; ++i ) - prefix.append(uitoa_n(uid[i], tmp, sizeof(tmp), 62)); - - return prefix; - } - } // namespace Bro diff --git a/src/Val.cc b/src/Val.cc index 1d2d0588a2..aace4d5a6f 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -13,11 +13,13 @@ #include #include +#include "Attr.h" #include "Net.h" #include "File.h" #include "Func.h" #include "Desc.h" #include "IntrusivePtr.h" +#include "ID.h" #include "RE.h" #include "Scope.h" #include "NetVar.h" @@ -360,6 +362,14 @@ void Val::ValDescribeReST(ODesc* d) const } +#ifdef DEBUG +void Val::SetID(ID* id) + { + delete [] bound_id; + bound_id = id ? copy_string(id->Name()) : 0; + } +#endif + bool Val::WouldOverflow(const BroType* from_type, const BroType* to_type, const Val* val) { if ( !to_type || !from_type ) @@ -1993,6 +2003,11 @@ ListVal* TableVal::ConvertToPureList() const return ConvertToList((*tl)[0]->Tag()); } +Attr* TableVal::FindAttr(attr_tag t) const + { + return attrs ? attrs->FindAttr(t) : 0; + } + void TableVal::Describe(ODesc* d) const { const PDict* tbl = AsTable(); diff --git a/src/Val.h b/src/Val.h index c0cd79e126..e7323f2965 100644 --- a/src/Val.h +++ b/src/Val.h @@ -6,9 +6,7 @@ #include "Dict.h" #include "CompHash.h" #include "BroString.h" -#include "Attr.h" #include "Timer.h" -#include "ID.h" #include "Scope.h" #include "Notifier.h" #include "RE.h" @@ -293,11 +291,7 @@ public: return bound_id ? global_scope()->Lookup(bound_id) : 0; } - void SetID(ID* id) - { - delete [] bound_id; - bound_id = id ? copy_string(id->Name()) : 0; - } + void SetID(ID* id); #endif static bool WouldOverflow(const BroType* from_type, const BroType* to_type, const Val* val); @@ -792,8 +786,7 @@ public: ListVal* ConvertToPureList() const; // must be single index type void SetAttrs(Attributes* attrs); - Attr* FindAttr(attr_tag t) const - { return attrs ? attrs->FindAttr(t) : 0; } + Attr* FindAttr(attr_tag t) const; Attributes* Attrs() { return attrs; } // Returns the size of the table. diff --git a/src/broker/Data.cc b/src/broker/Data.cc index dc5f829082..dd231e44d0 100644 --- a/src/broker/Data.cc +++ b/src/broker/Data.cc @@ -1,5 +1,6 @@ #include "Data.h" #include "File.h" +#include "Desc.h" #include "IntrusivePtr.h" #include "module_util.h" #include "3rdparty/doctest.h" @@ -1150,6 +1151,13 @@ broker::data& bro_broker::opaque_field_to_data(RecordVal* v, Frame* f) return static_cast(d)->data; } +void bro_broker::DataVal::ValDescribe(ODesc* d) const + { + d->Add("broker::data{"); + d->Add(broker::to_string(data)); + d->Add("}"); + } + bool bro_broker::DataVal::canCastTo(BroType* t) const { return data_type_check(data, t); diff --git a/src/broker/Data.h b/src/broker/Data.h index e31d27e1c7..5e8b9cc0fd 100644 --- a/src/broker/Data.h +++ b/src/broker/Data.h @@ -4,12 +4,13 @@ #include "Reporter.h" #include "Frame.h" #include "Expr.h" -#include "Desc.h" #include "Var.h" // for internal_type() template class IntrusivePtr; +class ODesc; + namespace bro_broker { extern OpaqueType* opaque_of_data_type; @@ -102,12 +103,7 @@ public: : OpaqueVal(bro_broker::opaque_of_data_type), data(std::move(arg_data)) {} - void ValDescribe(ODesc* d) const override - { - d->Add("broker::data{"); - d->Add(broker::to_string(data)); - d->Add("}"); - } + void ValDescribe(ODesc* d) const override; IntrusivePtr castTo(BroType* t); bool canCastTo(BroType* t) const; diff --git a/src/iosource/Packet.cc b/src/iosource/Packet.cc index f498fe0402..49d0cf35b0 100644 --- a/src/iosource/Packet.cc +++ b/src/iosource/Packet.cc @@ -1,7 +1,7 @@ - #include "Packet.h" #include "Sessions.h" #include "Desc.h" +#include "IP.h" #include "iosource/Manager.h" extern "C" { @@ -62,6 +62,11 @@ void Packet::Init(int arg_link_type, pkt_timeval *arg_ts, uint32_t arg_caplen, ProcessLayer2(); } +const IP_Hdr Packet::IP() const + { + return IP_Hdr((struct ip *) (data + hdr_size), false); + } + void Packet::Weird(const char* name) { sessions->Weird(name, this); diff --git a/src/iosource/Packet.h b/src/iosource/Packet.h index 2017f11677..79e82c3227 100644 --- a/src/iosource/Packet.h +++ b/src/iosource/Packet.h @@ -1,7 +1,5 @@ #pragma once -#include "IP.h" - #include #include @@ -16,6 +14,8 @@ typedef struct timeval pkt_timeval; class Val; class ODesc; +class IP_Hdr; +class RecordVal; /** * The Layer 3 type of a packet, as determined by the parsing code in Packet. @@ -119,8 +119,7 @@ public: * Interprets the Layer 3 of the packet as IP and returns a * correspondign object. */ - const IP_Hdr IP() const - { return IP_Hdr((struct ip *) (data + hdr_size), false); } + const IP_Hdr IP() const; /** * Returns a \c raw_pkt_hdr RecordVal, which includes layer 2 and diff --git a/src/zeekygen/IdentifierInfo.cc b/src/zeekygen/IdentifierInfo.cc index cc8cd13c61..5ff5b5dfa3 100644 --- a/src/zeekygen/IdentifierInfo.cc +++ b/src/zeekygen/IdentifierInfo.cc @@ -180,3 +180,8 @@ IdentifierInfo::Redefinition::~Redefinition() { Unref(init_expr); } + +IdentifierInfo::RecordField::~RecordField() + { + delete field; + } diff --git a/src/zeekygen/IdentifierInfo.h b/src/zeekygen/IdentifierInfo.h index 69e7e10d1c..56ae00f0b8 100644 --- a/src/zeekygen/IdentifierInfo.h +++ b/src/zeekygen/IdentifierInfo.h @@ -4,7 +4,6 @@ #include "Info.h" #include "ID.h" -#include "Type.h" #include #include @@ -167,8 +166,7 @@ private: std::string DoReStructuredText(bool roles_only) const override; struct RecordField { - ~RecordField() - { delete field; } + ~RecordField(); TypeDecl* field; std::string from_script; From 99b1dccb8de6fe2c9fbcfc558c271f0e15610752 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 4 Feb 2020 12:08:13 -0800 Subject: [PATCH 50/53] Updating submodule(s). [nomail] --- aux/zeekctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/zeekctl b/aux/zeekctl index db74e7cd16..3ef627f10a 160000 --- a/aux/zeekctl +++ b/aux/zeekctl @@ -1 +1 @@ -Subproject commit db74e7cd16eb000c26c793b5501f7119b1e7a23c +Subproject commit 3ef627f10ab700c4d3f5e373dd0c8b3866ac0378 From 5171f25e3a26166d0b00b9d09e8d75f096e01982 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 4 Feb 2020 14:15:14 -0700 Subject: [PATCH 51/53] Update NEWS about plugin breakage from the include-file changes --- NEWS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NEWS b/NEWS index f4e147fc06..8c067241c1 100644 --- a/NEWS +++ b/NEWS @@ -118,6 +118,12 @@ Changed Functionality packet sources, which should remain functional with little to no changes, since the entirety of the changes should be in ``PktSrc``. +- Remove a large number of headers from being included by various files across + the entire code base, which leads to a sizeable build time improvement. This + set of changes has the potential to cause plugins to not build anymore. The + fixes for this potential breakage should just be a matter of including the + necessary headers in the plugin code. + Removed Functionality --------------------- From deaab421873f68970b4e26ac8ae824007b385d92 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 4 Feb 2020 16:10:10 -0800 Subject: [PATCH 52/53] Change threading formatter errors to warnings. This change standardizes threading formatter error handling and moves the remaining error calls to be warnings instead. This is in line with already existing code - in most cases warnings were raised, only a few cases raised errors. These cases do not differ significantly from other cases in which warnings are raised. This also fixes GH-692, in which misformatted lines prevent future file parsing. This commit also moves the FailWarn method that is used by both the config and the ascii reader up to the ReaderBackend. Furthermore it makes the Warning method of ReaderBackend respect the warning suppression that is introduced by the FailWarn method. --- src/input/ReaderBackend.cc | 18 ++++++++++++ src/input/ReaderBackend.h | 28 ++++++++++++++++++- src/input/readers/ascii/Ascii.cc | 22 ++------------- src/input/readers/ascii/Ascii.h | 8 ------ src/input/readers/config/Config.cc | 21 ++------------ src/input/readers/config/Config.h | 7 ----- src/threading/Formatter.cc | 6 ++-- src/threading/formatters/Ascii.cc | 4 +-- .../.stderr | 8 +++--- .../.stderrwithoutfirstline | 9 ++++++ .../out | 14 +++++++++- .../base/frameworks/input/invalid-lines.zeek | 15 ++++++---- 12 files changed, 90 insertions(+), 70 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.invalid-lines/.stderrwithoutfirstline diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index cb25cfda3e..af46cade81 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -336,8 +336,26 @@ void ReaderBackend::Info(const char* msg) MsgThread::Info(msg); } +void ReaderBackend::FailWarn(bool is_error, const char *msg, bool suppress_future) + { + if ( is_error ) + Error(msg); + else + { + // suppress error message when we are already in error mode. + // There is no reason to repeat it every second. + if ( ! suppress_warnings ) + Warning(msg); + + if ( suppress_future ) + suppress_warnings = true; + } + } void ReaderBackend::Warning(const char* msg) { + if ( suppress_warnings ) + return; + SendOut(new ReaderErrorMessage(frontend, ReaderErrorMessage::WARNING, msg)); MsgThread::Warning(msg); } diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index 0d38edb40e..3219359f71 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -138,7 +138,7 @@ public: /** * One-time initialization of the reader to define the input source. * - * @param @param info Meta information for the writer. + * @param info Meta information for the writer. * * @param num_fields Number of fields contained in \a fields. * @@ -184,11 +184,34 @@ public: */ int NumFields() const { return num_fields; } + /** + * Convenience function that calls Warning or Error, depending on the + * is_error parameter. In case of a warning, setting suppress_future to + * true will suppress all future warnings until StopWarningSuppression() + * is called. + * + * @param is_error If set to true, an error is generated. Else a warning + * is generate. + * + * @param msg The error/warning message. + * + * @param suppress_future If set to true, future warnings are suppressed + * until StopWarningSuppression is called. + */ + void FailWarn(bool is_error, const char *msg, bool suppress_future = false); + + inline void StopWarningSuppression() { suppress_warnings = false; }; + // Overridden from MsgThread. bool OnHeartbeat(double network_time, double current_time) override; bool OnFinish(double network_time) override; void Info(const char* msg) override; + + /** + * Reports a warning in the child thread. For input readers, warning suppression + * that is caused by calling FailWarn() is respected by the Warning function. + */ void Warning(const char* msg) override; /** @@ -348,6 +371,9 @@ private: const threading::Field* const * fields; // raw mapping bool disabled; + // this is an internal indicator in case the read is currently in a failed state + // it's used to suppress duplicate error messages. + bool suppress_warnings = false; }; } diff --git a/src/input/readers/ascii/Ascii.cc b/src/input/readers/ascii/Ascii.cc index 7003c519a0..4a0cfea41e 100644 --- a/src/input/readers/ascii/Ascii.cc +++ b/src/input/readers/ascii/Ascii.cc @@ -50,7 +50,6 @@ Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) { mtime = 0; ino = 0; - suppress_warnings = false; fail_on_file_problem = false; fail_on_invalid_lines = false; } @@ -65,7 +64,7 @@ void Ascii::DoClose() bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fields) { - suppress_warnings = false; + StopWarningSuppression(); separator.assign( (const char*) BifConst::InputAscii::separator->Bytes(), BifConst::InputAscii::separator->Len()); @@ -119,21 +118,6 @@ bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* f return DoUpdate(); } -void Ascii::FailWarn(bool is_error, const char *msg, bool suppress_future) - { - if ( is_error ) - Error(msg); - else - { - // suppress error message when we are already in error mode. - // There is no reason to repeat it every second. - if ( ! suppress_warnings ) - Warning(msg); - - if ( suppress_future ) - suppress_warnings = true; - } - } bool Ascii::OpenFile() { @@ -173,7 +157,7 @@ bool Ascii::OpenFile() return ! fail_on_file_problem; } - suppress_warnings = false; + StopWarningSuppression(); return true; } @@ -309,7 +293,7 @@ bool Ascii::DoUpdate() // is to suppress an extra warning that we'd otherwise get on the initial // inode assignment. if ( ino != 0 ) - suppress_warnings = false; + StopWarningSuppression(); mtime = sb.st_mtime; ino = sb.st_ino; diff --git a/src/input/readers/ascii/Ascii.h b/src/input/readers/ascii/Ascii.h index 251ec05226..019e215c4f 100644 --- a/src/input/readers/ascii/Ascii.h +++ b/src/input/readers/ascii/Ascii.h @@ -56,10 +56,6 @@ private: bool ReadHeader(bool useCached); bool GetLine(string& str); bool OpenFile(); - // Call Warning or Error, depending on the is_error boolean. - // In case of a warning, setting suppress_future to true will suppress all future warnings - // (by setting suppress_warnings to true, until suppress_warnings is set back to false) - void FailWarn(bool is_error, const char *msg, bool suppress_future = false); ifstream file; time_t mtime; @@ -85,10 +81,6 @@ private: bool fail_on_file_problem; string path_prefix; - // this is an internal indicator in case the read is currently in a failed state - // it's used to suppress duplicate error messages. - bool suppress_warnings; - std::unique_ptr formatter; }; diff --git a/src/input/readers/config/Config.cc b/src/input/readers/config/Config.cc index 307ce3cddc..c215a44733 100644 --- a/src/input/readers/config/Config.cc +++ b/src/input/readers/config/Config.cc @@ -24,7 +24,6 @@ Config::Config(ReaderFrontend *frontend) : ReaderBackend(frontend) { mtime = 0; ino = 0; - suppress_warnings = false; fail_on_file_problem = false; // find all option names and their types. @@ -91,26 +90,10 @@ bool Config::OpenFile() return ! fail_on_file_problem; } - suppress_warnings = false; + StopWarningSuppression(); return true; } -void Config::FailWarn(bool is_error, const char *msg, bool suppress_future) - { - if ( is_error ) - Error(msg); - else - { - // suppress error message when we are already in error mode. - // There is no reason to repeat it every second. - if ( ! suppress_warnings ) - Warning(msg); - - if ( suppress_future ) - suppress_warnings = true; - } - } - bool Config::GetLine(string& str) { while ( getline(file, str) ) @@ -155,7 +138,7 @@ bool Config::DoUpdate() // is to suppress an extra warning that we'd otherwise get on the initial // inode assignment. if ( ino != 0 ) - suppress_warnings = false; + StopWarningSuppression(); mtime = sb.st_mtime; ino = sb.st_ino; diff --git a/src/input/readers/config/Config.h b/src/input/readers/config/Config.h index 4abddfb185..fde24e29d9 100644 --- a/src/input/readers/config/Config.h +++ b/src/input/readers/config/Config.h @@ -39,19 +39,12 @@ protected: private: bool GetLine(string& str); bool OpenFile(); - // Call Warning or Error, depending on the is_error boolean. - // In case of a warning, setting suppress_future to true will suppress all future warnings - // (by setting suppress_warnings to true, until suppress_warnings is set back to false) - void FailWarn(bool is_error, const char *msg, bool suppress_future = false); ifstream file; time_t mtime; ino_t ino; bool fail_on_file_problem; - // this is an internal indicator in case the read is currently in a failed state - // it's used to suppress duplicate error messages. - bool suppress_warnings; string set_separator; string empty_field; diff --git a/src/threading/Formatter.cc b/src/threading/Formatter.cc index 462067303a..ec89ebed97 100644 --- a/src/threading/Formatter.cc +++ b/src/threading/Formatter.cc @@ -55,7 +55,7 @@ TransportProto Formatter::ParseProto(const string &proto) const else if ( proto == "icmp" ) return TRANSPORT_ICMP; - thread->Error(thread->Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str())); + thread->Warning(thread->Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str())); return TRANSPORT_UNKNOWN; } @@ -72,7 +72,7 @@ threading::Value::addr_t Formatter::ParseAddr(const string &s) const if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 ) { - thread->Error(thread->Fmt("Bad address: %s", s.c_str())); + thread->Warning(thread->Fmt("Bad address: %s", s.c_str())); memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr)); } } @@ -82,7 +82,7 @@ threading::Value::addr_t Formatter::ParseAddr(const string &s) const val.family = IPv6; if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 ) { - thread->Error(thread->Fmt("Bad address: %s", s.c_str())); + thread->Warning(thread->Fmt("Bad address: %s", s.c_str())); memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr)); } } diff --git a/src/threading/formatters/Ascii.cc b/src/threading/formatters/Ascii.cc index a8dd8efa69..17c7a9e876 100644 --- a/src/threading/formatters/Ascii.cc +++ b/src/threading/formatters/Ascii.cc @@ -199,7 +199,7 @@ bool Ascii::Describe(ODesc* desc, threading::Value* val, const string& name) con } default: - GetThread()->Error(GetThread()->Fmt("Ascii writer unsupported field format %d", val->type)); + GetThread()->Warning(GetThread()->Fmt("Ascii writer unsupported field format %d", val->type)); return false; } @@ -346,7 +346,7 @@ threading::Value* Ascii::ParseValue(const string& s, const string& name, TypeTag } } - GetThread()->Error(GetThread()->Fmt("String '%s' contained no parseable pattern.", candidate.c_str())); + GetThread()->Warning(GetThread()->Fmt("String '%s' contained no parseable pattern.", candidate.c_str())); goto parse_error; } diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.bad_patterns/.stderr b/testing/btest/Baseline/scripts.base.frameworks.input.bad_patterns/.stderr index e0a7be2cc3..06ec4e063e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.bad_patterns/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.input.bad_patterns/.stderr @@ -1,9 +1,9 @@ -error: input.log/Input::READER_ASCII: String '/cat/sss' contained no parseable pattern. +warning: input.log/Input::READER_ASCII: String '/cat/sss' contained no parseable pattern. warning: input.log/Input::READER_ASCII: Could not convert line '2 /cat/sss' of input.log to Val. Ignoring line. -error: input.log/Input::READER_ASCII: String '/foo|bar' contained no parseable pattern. +warning: input.log/Input::READER_ASCII: String '/foo|bar' contained no parseable pattern. warning: input.log/Input::READER_ASCII: Could not convert line '3 /foo|bar' of input.log to Val. Ignoring line. -error: input.log/Input::READER_ASCII: String 'this is not a pattern' contained no parseable pattern. +warning: input.log/Input::READER_ASCII: String 'this is not a pattern' contained no parseable pattern. warning: input.log/Input::READER_ASCII: Could not convert line '4 this is not a pattern' of input.log to Val. Ignoring line. -error: input.log/Input::READER_ASCII: String '/5' contained no parseable pattern. +warning: input.log/Input::READER_ASCII: String '/5' contained no parseable pattern. warning: input.log/Input::READER_ASCII: Could not convert line '5 /5' of input.log to Val. Ignoring line. received termination signal diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalid-lines/.stderrwithoutfirstline b/testing/btest/Baseline/scripts.base.frameworks.input.invalid-lines/.stderrwithoutfirstline new file mode 100644 index 0000000000..282dc1a964 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalid-lines/.stderrwithoutfirstline @@ -0,0 +1,9 @@ +warning: ../input.log/Input::READER_ASCII: Not enough fields in line 'T -41 SSH::LOG 21 123 tcp 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30' of ../input.log. Found 15 fields, want positions 17 and -1 +warning: ../input.log/Input::READER_ASCII: Tried to parse invalid/unknown protocol: whatever +warning: ../input.log/Input::READER_ASCII: Bad address: 342.2.3.4 +warning: ../input.log/Input::READER_ASCII: Not enough fields in line 'T -41' of ../input.log. Found 1 fields, want positions 2 and -1 +error: ../input.log/Input::READER_ASCII: Not enough fields in line 'T -41 SSH::LOG 21 123 tcp 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30' of ../input.log. Found 15 fields, want positions 17 and -1 +error: ../input.log/Input::READER_ASCII: Init failed +error: ../input.log/Input::READER_ASCII: terminating thread +received termination signal +>>> diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalid-lines/out b/testing/btest/Baseline/scripts.base.frameworks.input.invalid-lines/out index 3406639d29..7aaf9232bc 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.invalid-lines/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalid-lines/out @@ -1,4 +1,16 @@ { +[-44] = [b=T, e=SSH::LOG, c=21, p=123/udp, sn=10.0.0.0/24, a=0.0.0.0, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, ns=4242 HOHOHO, sc={ +2, +4, +1, +3 +}, ss={ +BB, +AA, +CC +}, se={ + +}, vc=[10, 20, 30], ve=[]], [-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, ns=4242 HOHOHO, sc={ 2, 4, @@ -11,7 +23,7 @@ CC }, se={ }, vc=[10, 20, 30], ve=[]], -[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, ns=4242, sc={ +[-42] = [b=T, e=SSH::LOG, c=21, p=123/tcp, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, ns=4242, sc={ 2, 4, 1, diff --git a/testing/btest/scripts/base/frameworks/input/invalid-lines.zeek b/testing/btest/scripts/base/frameworks/input/invalid-lines.zeek index 86ace59204..f3625e127e 100644 --- a/testing/btest/scripts/base/frameworks/input/invalid-lines.zeek +++ b/testing/btest/scripts/base/frameworks/input/invalid-lines.zeek @@ -1,6 +1,8 @@ # @TEST-EXEC: btest-bg-run zeek zeek -b %INPUT # @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: btest-diff out +# @TEST-EXEC: sed 1d .stderr > .stderrwithoutfirstline +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff .stderrwithoutfirstline redef exit_only_after_terminate = T; redef InputAscii::fail_on_invalid_lines = F; @@ -8,11 +10,12 @@ redef InputAscii::fail_on_invalid_lines = F; @TEST-START-FILE input.log #separator \x09 #path ssh -#fields b i e c p sn a d t iv s sc ss se vc ve ns -#types bool int enum count port subnet addr double time interval string table table table vector vector string -T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 -T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY 4242 -T -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY 4242 HOHOHO +#fields b i e c p pt sn a d t iv s sc ss se vc ve ns +#types bool int enum count port string subnet addr double time interval string table table table vector vector string +T -41 SSH::LOG 21 123 tcp 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 +T -42 SSH::LOG 21 123 tcp 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY 4242 +T -43 SSH::LOG 21 123 whatever 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY 4242 HOHOHO +T -44 SSH::LOG 21 123 udp 10.0.0.0/24 342.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY 4242 HOHOHO T -41 @TEST-END-FILE @@ -32,7 +35,7 @@ type Val: record { b: bool; e: Log::ID; c: count; - p: port; + p: port &type_column="pt"; sn: subnet; a: addr; d: double; From 50ebbc90add103ed3cf21506841a9e39550eb883 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Wed, 5 Feb 2020 09:34:29 -0800 Subject: [PATCH 53/53] Resolve race condition in test --- testing/btest/scripts/base/frameworks/input/invalid-lines.zeek | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/btest/scripts/base/frameworks/input/invalid-lines.zeek b/testing/btest/scripts/base/frameworks/input/invalid-lines.zeek index f3625e127e..025d802823 100644 --- a/testing/btest/scripts/base/frameworks/input/invalid-lines.zeek +++ b/testing/btest/scripts/base/frameworks/input/invalid-lines.zeek @@ -1,7 +1,7 @@ # @TEST-EXEC: btest-bg-run zeek zeek -b %INPUT # @TEST-EXEC: btest-bg-wait 10 # @TEST-EXEC: btest-diff out -# @TEST-EXEC: sed 1d .stderr > .stderrwithoutfirstline +# @TEST-EXEC: sed 1d .stderr | grep -v "queued" > .stderrwithoutfirstline # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff .stderrwithoutfirstline redef exit_only_after_terminate = T;