From 29bc84e1d64a9550fe0e95eb580bc422de1539b4 Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Mon, 12 Aug 2024 11:59:30 +0200 Subject: [PATCH 1/5] Add packet analyzer history --- scripts/policy/misc/unknown-protocols.zeek | 9 ++++++++- src/event.bif | 6 +++++- src/packet_analysis/Analyzer.cc | 2 ++ src/packet_analysis/Manager.cc | 14 +++++++++++++- src/packet_analysis/Manager.h | 17 +++++++++++++++++ 5 files changed, 45 insertions(+), 3 deletions(-) diff --git a/scripts/policy/misc/unknown-protocols.zeek b/scripts/policy/misc/unknown-protocols.zeek index df510d9846..c532f7b0f5 100644 --- a/scripts/policy/misc/unknown-protocols.zeek +++ b/scripts/policy/misc/unknown-protocols.zeek @@ -25,16 +25,23 @@ export { ## A certain number of bytes at the start of the unknown protocol's ## header. first_bytes: string &log; + + ## The chain of packet analyzers that processed the packet up to this + ## point. This includes the history of encapsulating packets in case + ## of tunneling. + analyzer_history: vector of string &log; }; } -event unknown_protocol(analyzer_name: string, protocol: count, first_bytes: string) +event unknown_protocol(analyzer_name: string, protocol: count, first_bytes: string, + analyzer_history: string_vec) { local info : Info; info$ts = network_time(); info$analyzer = analyzer_name; info$protocol_id = fmt("0x%x", protocol); info$first_bytes = bytestring_to_hexstr(first_bytes); + info$analyzer_history = analyzer_history; Log::write(LOG, info); } diff --git a/src/event.bif b/src/event.bif index 38e15bf09f..b925f02144 100644 --- a/src/event.bif +++ b/src/event.bif @@ -885,8 +885,12 @@ event Pcap::file_done%(path: string%); ## ## first_bytes: A certain number of bytes at the start of the unknown protocol's header. ## +## analyzer_history: The chain of packet analyzers that processed the packet up to this +## point. This includes the history of encapsulating packets in case +## of tunneling. +## ## .. zeek:see:: UnknownProtocol::first_bytes_count -event unknown_protocol%(analyzer_name: string, protocol: count, first_bytes: string%); +event unknown_protocol%(analyzer_name: string, protocol: count, first_bytes: string, analyzer_history: string_vec%); ## An event for handling packets that reached the end of processing without ## being marked as processed. Note that this event may lead to unpredictable diff --git a/src/packet_analysis/Analyzer.cc b/src/packet_analysis/Analyzer.cc index ee067e4513..b272ec4965 100644 --- a/src/packet_analysis/Analyzer.cc +++ b/src/packet_analysis/Analyzer.cc @@ -114,6 +114,7 @@ bool Analyzer::ForwardPacket(size_t len, const uint8_t* data, Packet* packet, ui DBG_LOG(DBG_PACKET_ANALYSIS, "Analysis in %s succeeded, next layer identifier is %#x.", GetAnalyzerName(), identifier); + packet_mgr->TrackAnalyzer(inner_analyzer); return inner_analyzer->AnalyzePacket(len, data, packet); } @@ -129,6 +130,7 @@ bool Analyzer::ForwardPacket(size_t len, const uint8_t* data, Packet* packet) co return false; } + packet_mgr->TrackAnalyzer(inner_analyzer); return inner_analyzer->AnalyzePacket(len, data, packet); } diff --git a/src/packet_analysis/Manager.cc b/src/packet_analysis/Manager.cc index 259aefd37d..c04cb7b17e 100644 --- a/src/packet_analysis/Manager.cc +++ b/src/packet_analysis/Manager.cc @@ -113,6 +113,7 @@ void Manager::ProcessPacket(Packet* packet) { } // Start packet analysis + analyzer_stack.clear(); root_analyzer->ForwardPacket(packet->cap_len, packet->data, packet, packet->link_type); if ( ! packet->processed ) { @@ -227,13 +228,24 @@ bool Manager::PermitUnknownProtocol(const std::string& analyzer, uint32_t protoc return false; } +zeek::VectorValPtr Manager::BuildAnalyzerHistory() const { + auto history = zeek::make_intrusive(zeek::id::string_vec); + + for ( unsigned int i = 0; i < analyzer_stack.size(); i++ ) { + auto analyzer_name = analyzer_stack[i]->GetAnalyzerName(); + history->Assign(i, make_intrusive(analyzer_name)); + } + + return history; +} + void Manager::ReportUnknownProtocol(const std::string& analyzer, uint32_t protocol, const uint8_t* data, size_t len) { if ( unknown_protocol ) { if ( PermitUnknownProtocol(analyzer, protocol) ) { int bytes_len = std::min(unknown_first_bytes_count, static_cast(len)); event_mgr.Enqueue(unknown_protocol, make_intrusive(analyzer), val_mgr->Count(protocol), - make_intrusive(bytes_len, (const char*)data)); + make_intrusive(bytes_len, (const char*)data), BuildAnalyzerHistory()); } } } diff --git a/src/packet_analysis/Manager.h b/src/packet_analysis/Manager.h index dac81c99d9..833c7ccde2 100644 --- a/src/packet_analysis/Manager.h +++ b/src/packet_analysis/Manager.h @@ -176,6 +176,13 @@ public: */ uint64_t GetUnprocessedCount() const { return total_not_processed; } + /** + * Tracks the given analyzer for the current packet's analyzer history. + * The packet analyzer history is implemented in form of a stack, which is reset on a + * call to ProcessPacket() but maintained throughout calls to ProcessInnerPacket(). + */ + void TrackAnalyzer(AnalyzerPtr analyzer) { analyzer_stack.push_back(std::move(analyzer)); } + private: /** * Instantiates a new analyzer instance. @@ -197,6 +204,14 @@ private: */ AnalyzerPtr InstantiateAnalyzer(const std::string& name); + /** + * Generates a string vector that represents the analyzer history of the + * current packet based on the analyzers' tags. + * + * @return A vector of strings representing the packet analyzer history. + */ + VectorValPtr BuildAnalyzerHistory() const; + bool PermitUnknownProtocol(const std::string& analyzer, uint32_t protocol); std::map analyzers; @@ -216,6 +231,8 @@ private: uint64_t total_not_processed = 0; iosource::PktDumper* unprocessed_dumper = nullptr; + + std::vector analyzer_stack; }; } // namespace packet_analysis From f18c28cfe59438c14c7407bc17d6857aa61d2dee Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Mon, 12 Aug 2024 12:02:31 +0200 Subject: [PATCH 2/5] Update btest baselines for analyzer history --- testing/btest/Baseline/core.unknown-protocol-event/out | 2 +- .../btest/Baseline/core.vntag/unknown_protocols.log | 10 +++++----- .../btest/Baseline/plugins.packet-protocol/output_orig | 6 +++--- testing/btest/core/unknown-protocol-event.zeek | 9 ++++++--- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/testing/btest/Baseline/core.unknown-protocol-event/out b/testing/btest/Baseline/core.unknown-protocol-event/out index 0cdfa29213..7f9b60fb5d 100644 --- a/testing/btest/Baseline/core.unknown-protocol-event/out +++ b/testing/btest/Baseline/core.unknown-protocol-event/out @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -ETHERNET, 35020, 02070400222d81db1004 +ETHERNET, 35020, 02070400222d81db1004, [ETHERNET] diff --git a/testing/btest/Baseline/core.vntag/unknown_protocols.log b/testing/btest/Baseline/core.vntag/unknown_protocols.log index c7ddb819f3..e0e8fa8956 100644 --- a/testing/btest/Baseline/core.vntag/unknown_protocols.log +++ b/testing/btest/Baseline/core.vntag/unknown_protocols.log @@ -5,9 +5,9 @@ #unset_field - #path unknown_protocols #open XXXX-XX-XX-XX-XX-XX -#fields ts analyzer protocol_id first_bytes -#types time string string string -XXXXXXXXXX.XXXXXX IP 0xfd 1b794b175fac06aba658 -XXXXXXXXXX.XXXXXX IP 0xfd 9d6c1f9e20274bb66385 -XXXXXXXXXX.XXXXXX IP 0xfd 06ffb64ded001f65f818 +#fields ts analyzer protocol_id first_bytes analyzer_history +#types time string string string vector[string] +XXXXXXXXXX.XXXXXX IP 0xfd 1b794b175fac06aba658 ETHERNET,VNTAG,VLAN,IP +XXXXXXXXXX.XXXXXX IP 0xfd 9d6c1f9e20274bb66385 ETHERNET,VNTAG,VLAN,IP +XXXXXXXXXX.XXXXXX IP 0xfd 06ffb64ded001f65f818 ETHERNET,VNTAG,VLAN,IP #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/plugins.packet-protocol/output_orig b/testing/btest/Baseline/plugins.packet-protocol/output_orig index 7a832e4072..b60ee2f067 100644 --- a/testing/btest/Baseline/plugins.packet-protocol/output_orig +++ b/testing/btest/Baseline/plugins.packet-protocol/output_orig @@ -5,7 +5,7 @@ #unset_field - #path unknown_protocols #open XXXX-XX-XX-XX-XX-XX -#fields ts analyzer protocol_id first_bytes -#types time string string string -XXXXXXXXXX.XXXXXX ETHERNET 0x88b5 4920616d20656e636170 +#fields ts analyzer protocol_id first_bytes analyzer_history +#types time string string string vector[string] +XXXXXXXXXX.XXXXXX ETHERNET 0x88b5 4920616d20656e636170 ETHERNET #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/core/unknown-protocol-event.zeek b/testing/btest/core/unknown-protocol-event.zeek index c043217da4..ba49a28c16 100644 --- a/testing/btest/core/unknown-protocol-event.zeek +++ b/testing/btest/core/unknown-protocol-event.zeek @@ -1,6 +1,9 @@ # @TEST-EXEC: zeek -b -r $TRACES/lldp.pcap %INPUT >out # @TEST-EXEC: btest-diff out -event unknown_protocol(analyzer_name: string, protocol: count, first_bytes: string) - { print analyzer_name, protocol, bytestring_to_hexstr(first_bytes); } - +event unknown_protocol(analyzer_name: string, protocol: count, first_bytes: string, + analyzer_history: string_vec) + { + print analyzer_name, protocol, bytestring_to_hexstr(first_bytes), + analyzer_history; + } From b4e83aca8ca72bada15e779435019b7e4a3c5084 Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Mon, 12 Aug 2024 12:03:23 +0200 Subject: [PATCH 3/5] Update external baselines for analyzer history --- testing/external/commit-hash.zeek-testing | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/external/commit-hash.zeek-testing b/testing/external/commit-hash.zeek-testing index 64e24e5e4b..6c7400555f 100644 --- a/testing/external/commit-hash.zeek-testing +++ b/testing/external/commit-hash.zeek-testing @@ -1 +1 @@ -3577ad0657c0cd61222d329a840e881dacf93f07 +2662f0a809ff1f1c8fe581f23a4968c4d46a0484 From c73fcdec3d25c4ba49291f9ca3f374526a0c6aab Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Mon, 12 Aug 2024 16:08:18 +0200 Subject: [PATCH 4/5] Use raw pointer for packet analyzer history --- src/packet_analysis/Analyzer.cc | 4 ++-- src/packet_analysis/Manager.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/packet_analysis/Analyzer.cc b/src/packet_analysis/Analyzer.cc index b272ec4965..e61ef83238 100644 --- a/src/packet_analysis/Analyzer.cc +++ b/src/packet_analysis/Analyzer.cc @@ -114,7 +114,7 @@ bool Analyzer::ForwardPacket(size_t len, const uint8_t* data, Packet* packet, ui DBG_LOG(DBG_PACKET_ANALYSIS, "Analysis in %s succeeded, next layer identifier is %#x.", GetAnalyzerName(), identifier); - packet_mgr->TrackAnalyzer(inner_analyzer); + packet_mgr->TrackAnalyzer(inner_analyzer.get()); return inner_analyzer->AnalyzePacket(len, data, packet); } @@ -130,7 +130,7 @@ bool Analyzer::ForwardPacket(size_t len, const uint8_t* data, Packet* packet) co return false; } - packet_mgr->TrackAnalyzer(inner_analyzer); + packet_mgr->TrackAnalyzer(inner_analyzer.get()); return inner_analyzer->AnalyzePacket(len, data, packet); } diff --git a/src/packet_analysis/Manager.h b/src/packet_analysis/Manager.h index 833c7ccde2..010cef27b8 100644 --- a/src/packet_analysis/Manager.h +++ b/src/packet_analysis/Manager.h @@ -181,7 +181,7 @@ public: * The packet analyzer history is implemented in form of a stack, which is reset on a * call to ProcessPacket() but maintained throughout calls to ProcessInnerPacket(). */ - void TrackAnalyzer(AnalyzerPtr analyzer) { analyzer_stack.push_back(std::move(analyzer)); } + void TrackAnalyzer(Analyzer* analyzer) { analyzer_stack.push_back(analyzer); } private: /** @@ -232,7 +232,7 @@ private: uint64_t total_not_processed = 0; iosource::PktDumper* unprocessed_dumper = nullptr; - std::vector analyzer_stack; + std::vector analyzer_stack; }; } // namespace packet_analysis From e14f3bae705ba05e767e646605a8eaee627526c5 Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Mon, 12 Aug 2024 21:44:02 +0200 Subject: [PATCH 5/5] Deprecate old unknown_protocol event signature --- src/event.bif | 1 + src/packet_analysis/Manager.h | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/event.bif b/src/event.bif index b925f02144..02e55c4f14 100644 --- a/src/event.bif +++ b/src/event.bif @@ -891,6 +891,7 @@ event Pcap::file_done%(path: string%); ## ## .. zeek:see:: UnknownProtocol::first_bytes_count event unknown_protocol%(analyzer_name: string, protocol: count, first_bytes: string, analyzer_history: string_vec%); +event unknown_protocol%(analyzer_name: string, protocol: count, first_bytes: string%) &deprecated="Remove in v8.1. Use the version that includes analyzer_history."; ## An event for handling packets that reached the end of processing without ## being marked as processed. Note that this event may lead to unpredictable diff --git a/src/packet_analysis/Manager.h b/src/packet_analysis/Manager.h index 010cef27b8..5497382714 100644 --- a/src/packet_analysis/Manager.h +++ b/src/packet_analysis/Manager.h @@ -180,6 +180,8 @@ public: * Tracks the given analyzer for the current packet's analyzer history. * The packet analyzer history is implemented in form of a stack, which is reset on a * call to ProcessPacket() but maintained throughout calls to ProcessInnerPacket(). + * + * @param analyzer The analyzer to track. */ void TrackAnalyzer(Analyzer* analyzer) { analyzer_stack.push_back(analyzer); }