diff --git a/CHANGES b/CHANGES index a176d02e60..3252f6165c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,15 @@ +7.1.0-dev.147 | 2024-08-13 12:37:17 +0200 + + * Deprecate old unknown_protocol event signature (Jan Grashoefer, Corelight) + + * Use raw pointer for packet analyzer history (Jan Grashoefer, Corelight) + + * Update external baselines for analyzer history (Jan Grashoefer, Corelight) + + * Update btest baselines for analyzer history (Jan Grashoefer, Corelight) + + * Add packet analyzer history (Jan Grashoefer, Corelight) + 7.1.0-dev.141 | 2024-08-12 11:07:32 +0200 * spicyz: Add back message about removed support for port / ports in evt (Arne Welzel, Corelight) diff --git a/NEWS b/NEWS index bfb2deb0a4..d148272c12 100644 --- a/NEWS +++ b/NEWS @@ -25,6 +25,10 @@ New Functionality for connections where client and server negotiate to TLS through the extended request/response mechanism. +* The ``unknown_protocols()`` event now includes the name of all packet + analyzer used for processing the packet when the event is raised. The + ``unknown_protocol.log`` file was extended to include this information. + Changed Functionality --------------------- diff --git a/VERSION b/VERSION index e2bf3bdfd6..cc5acca03d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -7.1.0-dev.141 +7.1.0-dev.147 diff --git a/scripts/policy/misc/unknown-protocols.zeek b/scripts/policy/misc/unknown-protocols.zeek index df510d9846..c532f7b0f5 100644 --- a/scripts/policy/misc/unknown-protocols.zeek +++ b/scripts/policy/misc/unknown-protocols.zeek @@ -25,16 +25,23 @@ export { ## A certain number of bytes at the start of the unknown protocol's ## header. first_bytes: string &log; + + ## The chain of packet analyzers that processed the packet up to this + ## point. This includes the history of encapsulating packets in case + ## of tunneling. + analyzer_history: vector of string &log; }; } -event unknown_protocol(analyzer_name: string, protocol: count, first_bytes: string) +event unknown_protocol(analyzer_name: string, protocol: count, first_bytes: string, + analyzer_history: string_vec) { local info : Info; info$ts = network_time(); info$analyzer = analyzer_name; info$protocol_id = fmt("0x%x", protocol); info$first_bytes = bytestring_to_hexstr(first_bytes); + info$analyzer_history = analyzer_history; Log::write(LOG, info); } diff --git a/src/event.bif b/src/event.bif index 38e15bf09f..02e55c4f14 100644 --- a/src/event.bif +++ b/src/event.bif @@ -885,8 +885,13 @@ event Pcap::file_done%(path: string%); ## ## first_bytes: A certain number of bytes at the start of the unknown protocol's header. ## +## analyzer_history: The chain of packet analyzers that processed the packet up to this +## point. This includes the history of encapsulating packets in case +## of tunneling. +## ## .. zeek:see:: UnknownProtocol::first_bytes_count -event unknown_protocol%(analyzer_name: string, protocol: count, first_bytes: string%); +event unknown_protocol%(analyzer_name: string, protocol: count, first_bytes: string, analyzer_history: string_vec%); +event unknown_protocol%(analyzer_name: string, protocol: count, first_bytes: string%) &deprecated="Remove in v8.1. Use the version that includes analyzer_history."; ## An event for handling packets that reached the end of processing without ## being marked as processed. Note that this event may lead to unpredictable diff --git a/src/packet_analysis/Analyzer.cc b/src/packet_analysis/Analyzer.cc index ee067e4513..e61ef83238 100644 --- a/src/packet_analysis/Analyzer.cc +++ b/src/packet_analysis/Analyzer.cc @@ -114,6 +114,7 @@ bool Analyzer::ForwardPacket(size_t len, const uint8_t* data, Packet* packet, ui DBG_LOG(DBG_PACKET_ANALYSIS, "Analysis in %s succeeded, next layer identifier is %#x.", GetAnalyzerName(), identifier); + packet_mgr->TrackAnalyzer(inner_analyzer.get()); return inner_analyzer->AnalyzePacket(len, data, packet); } @@ -129,6 +130,7 @@ bool Analyzer::ForwardPacket(size_t len, const uint8_t* data, Packet* packet) co return false; } + packet_mgr->TrackAnalyzer(inner_analyzer.get()); return inner_analyzer->AnalyzePacket(len, data, packet); } diff --git a/src/packet_analysis/Manager.cc b/src/packet_analysis/Manager.cc index 259aefd37d..c04cb7b17e 100644 --- a/src/packet_analysis/Manager.cc +++ b/src/packet_analysis/Manager.cc @@ -113,6 +113,7 @@ void Manager::ProcessPacket(Packet* packet) { } // Start packet analysis + analyzer_stack.clear(); root_analyzer->ForwardPacket(packet->cap_len, packet->data, packet, packet->link_type); if ( ! packet->processed ) { @@ -227,13 +228,24 @@ bool Manager::PermitUnknownProtocol(const std::string& analyzer, uint32_t protoc return false; } +zeek::VectorValPtr Manager::BuildAnalyzerHistory() const { + auto history = zeek::make_intrusive(zeek::id::string_vec); + + for ( unsigned int i = 0; i < analyzer_stack.size(); i++ ) { + auto analyzer_name = analyzer_stack[i]->GetAnalyzerName(); + history->Assign(i, make_intrusive(analyzer_name)); + } + + return history; +} + void Manager::ReportUnknownProtocol(const std::string& analyzer, uint32_t protocol, const uint8_t* data, size_t len) { if ( unknown_protocol ) { if ( PermitUnknownProtocol(analyzer, protocol) ) { int bytes_len = std::min(unknown_first_bytes_count, static_cast(len)); event_mgr.Enqueue(unknown_protocol, make_intrusive(analyzer), val_mgr->Count(protocol), - make_intrusive(bytes_len, (const char*)data)); + make_intrusive(bytes_len, (const char*)data), BuildAnalyzerHistory()); } } } diff --git a/src/packet_analysis/Manager.h b/src/packet_analysis/Manager.h index dac81c99d9..5497382714 100644 --- a/src/packet_analysis/Manager.h +++ b/src/packet_analysis/Manager.h @@ -176,6 +176,15 @@ public: */ uint64_t GetUnprocessedCount() const { return total_not_processed; } + /** + * Tracks the given analyzer for the current packet's analyzer history. + * The packet analyzer history is implemented in form of a stack, which is reset on a + * call to ProcessPacket() but maintained throughout calls to ProcessInnerPacket(). + * + * @param analyzer The analyzer to track. + */ + void TrackAnalyzer(Analyzer* analyzer) { analyzer_stack.push_back(analyzer); } + private: /** * Instantiates a new analyzer instance. @@ -197,6 +206,14 @@ private: */ AnalyzerPtr InstantiateAnalyzer(const std::string& name); + /** + * Generates a string vector that represents the analyzer history of the + * current packet based on the analyzers' tags. + * + * @return A vector of strings representing the packet analyzer history. + */ + VectorValPtr BuildAnalyzerHistory() const; + bool PermitUnknownProtocol(const std::string& analyzer, uint32_t protocol); std::map analyzers; @@ -216,6 +233,8 @@ private: uint64_t total_not_processed = 0; iosource::PktDumper* unprocessed_dumper = nullptr; + + std::vector analyzer_stack; }; } // namespace packet_analysis diff --git a/testing/btest/Baseline/core.unknown-protocol-event/out b/testing/btest/Baseline/core.unknown-protocol-event/out index 0cdfa29213..7f9b60fb5d 100644 --- a/testing/btest/Baseline/core.unknown-protocol-event/out +++ b/testing/btest/Baseline/core.unknown-protocol-event/out @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -ETHERNET, 35020, 02070400222d81db1004 +ETHERNET, 35020, 02070400222d81db1004, [ETHERNET] diff --git a/testing/btest/Baseline/core.vntag/unknown_protocols.log b/testing/btest/Baseline/core.vntag/unknown_protocols.log index c7ddb819f3..e0e8fa8956 100644 --- a/testing/btest/Baseline/core.vntag/unknown_protocols.log +++ b/testing/btest/Baseline/core.vntag/unknown_protocols.log @@ -5,9 +5,9 @@ #unset_field - #path unknown_protocols #open XXXX-XX-XX-XX-XX-XX -#fields ts analyzer protocol_id first_bytes -#types time string string string -XXXXXXXXXX.XXXXXX IP 0xfd 1b794b175fac06aba658 -XXXXXXXXXX.XXXXXX IP 0xfd 9d6c1f9e20274bb66385 -XXXXXXXXXX.XXXXXX IP 0xfd 06ffb64ded001f65f818 +#fields ts analyzer protocol_id first_bytes analyzer_history +#types time string string string vector[string] +XXXXXXXXXX.XXXXXX IP 0xfd 1b794b175fac06aba658 ETHERNET,VNTAG,VLAN,IP +XXXXXXXXXX.XXXXXX IP 0xfd 9d6c1f9e20274bb66385 ETHERNET,VNTAG,VLAN,IP +XXXXXXXXXX.XXXXXX IP 0xfd 06ffb64ded001f65f818 ETHERNET,VNTAG,VLAN,IP #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/plugins.packet-protocol/output_orig b/testing/btest/Baseline/plugins.packet-protocol/output_orig index 7a832e4072..b60ee2f067 100644 --- a/testing/btest/Baseline/plugins.packet-protocol/output_orig +++ b/testing/btest/Baseline/plugins.packet-protocol/output_orig @@ -5,7 +5,7 @@ #unset_field - #path unknown_protocols #open XXXX-XX-XX-XX-XX-XX -#fields ts analyzer protocol_id first_bytes -#types time string string string -XXXXXXXXXX.XXXXXX ETHERNET 0x88b5 4920616d20656e636170 +#fields ts analyzer protocol_id first_bytes analyzer_history +#types time string string string vector[string] +XXXXXXXXXX.XXXXXX ETHERNET 0x88b5 4920616d20656e636170 ETHERNET #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/core/unknown-protocol-event.zeek b/testing/btest/core/unknown-protocol-event.zeek index c043217da4..ba49a28c16 100644 --- a/testing/btest/core/unknown-protocol-event.zeek +++ b/testing/btest/core/unknown-protocol-event.zeek @@ -1,6 +1,9 @@ # @TEST-EXEC: zeek -b -r $TRACES/lldp.pcap %INPUT >out # @TEST-EXEC: btest-diff out -event unknown_protocol(analyzer_name: string, protocol: count, first_bytes: string) - { print analyzer_name, protocol, bytestring_to_hexstr(first_bytes); } - +event unknown_protocol(analyzer_name: string, protocol: count, first_bytes: string, + analyzer_history: string_vec) + { + print analyzer_name, protocol, bytestring_to_hexstr(first_bytes), + analyzer_history; + } diff --git a/testing/external/commit-hash.zeek-testing b/testing/external/commit-hash.zeek-testing index 64e24e5e4b..c6c71cea50 100644 --- a/testing/external/commit-hash.zeek-testing +++ b/testing/external/commit-hash.zeek-testing @@ -1 +1 @@ -3577ad0657c0cd61222d329a840e881dacf93f07 +4f808b8e1a4e99e738af85f15a5534c0ee99cdba