From 8d19fa23ef11351a8d7aaace3636f5bb564e3fd0 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Fri, 19 Aug 2022 12:51:36 +0200 Subject: [PATCH] Remove unified2 file analyzer --- NEWS | 7 + doc | 2 +- scripts/base/init-bare.zeek | 36 --- scripts/policy/files/unified2/README | 1 - scripts/policy/files/unified2/__load__.zeek | 1 - scripts/policy/files/unified2/main.zeek | 298 ------------------ scripts/test-all-policy.zeek | 2 - src/file_analysis/analyzer/CMakeLists.txt | 1 - .../analyzer/unified2/CMakeLists.txt | 11 - src/file_analysis/analyzer/unified2/Plugin.cc | 28 -- .../analyzer/unified2/Unified2.cc | 41 --- .../analyzer/unified2/Unified2.h | 37 --- .../analyzer/unified2/events.bif | 17 - src/file_analysis/analyzer/unified2/types.bif | 2 - .../analyzer/unified2/unified2-analyzer.pac | 173 ---------- .../analyzer/unified2/unified2-file.pac | 91 ------ .../analyzer/unified2/unified2.pac | 21 -- .../canonified_loaded_scripts.log | 2 - .../canonified_loaded_scripts.log | 2 - .../btest/Baseline/coverage.find-bro-logs/out | 1 - testing/btest/Baseline/plugins.hooks/output | 12 - .../unified2.log | 12 - testing/btest/Files/unified2.u2 | Bin 576 -> 0 bytes .../scripts/base/files/unified2/alert.zeek | 76 ----- 24 files changed, 8 insertions(+), 866 deletions(-) delete mode 100644 scripts/policy/files/unified2/README delete mode 100644 scripts/policy/files/unified2/__load__.zeek delete mode 100644 scripts/policy/files/unified2/main.zeek delete mode 100644 src/file_analysis/analyzer/unified2/CMakeLists.txt delete mode 100644 src/file_analysis/analyzer/unified2/Plugin.cc delete mode 100644 src/file_analysis/analyzer/unified2/Unified2.cc delete mode 100644 src/file_analysis/analyzer/unified2/Unified2.h delete mode 100644 src/file_analysis/analyzer/unified2/events.bif delete mode 100644 src/file_analysis/analyzer/unified2/types.bif delete mode 100644 src/file_analysis/analyzer/unified2/unified2-analyzer.pac delete mode 100644 src/file_analysis/analyzer/unified2/unified2-file.pac delete mode 100644 src/file_analysis/analyzer/unified2/unified2.pac delete mode 100644 testing/btest/Baseline/scripts.base.files.unified2.alert/unified2.log delete mode 100755 testing/btest/Files/unified2.u2 delete mode 100644 testing/btest/scripts/base/files/unified2/alert.zeek diff --git a/NEWS b/NEWS index 2b0e1ec6ce..925cb78cfa 100644 --- a/NEWS +++ b/NEWS @@ -47,6 +47,8 @@ Breaking Changes continue to represent the total number across all connections seen by the current instance of Zeek. +- The unified2 analyzer and accompanying scripts have been removed without + deprecation. It has been assumed there have been no users. New Functionality ----------------- @@ -111,6 +113,11 @@ Changed Functionality This prevents callbacks into script-land through change handlers when parts of the environment have already been torn down. +Removed Functionality +--------------------- + +- The unified2 analyzer and accompanying scripts have been removed. + Deprecated Functionality ------------------------ diff --git a/doc b/doc index 79238280e1..497bcf29bc 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 79238280e1a29ab232af84e4bed354ae37c7e6f6 +Subproject commit 497bcf29bc7bec2ef4eb076c8e4364ff45078858 diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 6519c8c968..1ca106a1b9 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -4251,42 +4251,6 @@ type ModbusHeaders: record { function_code: count; }; -module Unified2; -export { - type Unified2::IDSEvent: record { - sensor_id: count; - event_id: count; - ts: time; - signature_id: count; - generator_id: count; - signature_revision: count; - classification_id: count; - priority_id: count; - src_ip: addr; - dst_ip: addr; - src_p: port; - dst_p: port; - impact_flag: count; - impact: count; - blocked: count; - ## Not available in "legacy" IDS events. - mpls_label: count &optional; - ## Not available in "legacy" IDS events. - vlan_id: count &optional; - ## Only available in "legacy" IDS events. - packet_action: count &optional; - }; - - type Unified2::Packet: record { - sensor_id: count; - event_id: count; - event_second: count; - packet_ts: time; - link_type: count; - data: string; - }; -} - module SSL; export { type SignatureAndHashAlgorithm: record { diff --git a/scripts/policy/files/unified2/README b/scripts/policy/files/unified2/README deleted file mode 100644 index 08b5014db6..0000000000 --- a/scripts/policy/files/unified2/README +++ /dev/null @@ -1 +0,0 @@ -Support for Unified2 files in the file analysis framework. diff --git a/scripts/policy/files/unified2/__load__.zeek b/scripts/policy/files/unified2/__load__.zeek deleted file mode 100644 index a10fe855df..0000000000 --- a/scripts/policy/files/unified2/__load__.zeek +++ /dev/null @@ -1 +0,0 @@ -@load ./main diff --git a/scripts/policy/files/unified2/main.zeek b/scripts/policy/files/unified2/main.zeek deleted file mode 100644 index 2930f483a0..0000000000 --- a/scripts/policy/files/unified2/main.zeek +++ /dev/null @@ -1,298 +0,0 @@ -@load base/utils/dir -@load base/utils/paths - -module Unified2; - -export { - redef enum Log::ID += { LOG }; - - global log_policy: Log::PolicyHook; - - ## File to watch for Unified2 files. - const watch_file = "" &redef; - - ## Directory to watch for Unified2 records. - const watch_dir = "" &redef; - - ## The sid-msg.map file you would like to use for your alerts. - const sid_msg = "" &redef; - - ## The gen-msg.map file you would like to use for your alerts. - const gen_msg = "" &redef; - - ## The classification.config file you would like to use for your alerts. - const classification_config = "" &redef; - - ## Reconstructed "alert" which combines related events - ## and packets. - global alert: event(f: fa_file, ev: Unified2::IDSEvent, pkt: Unified2::Packet); - - type PacketID: record { - src_ip: addr; - src_p: port; - dst_ip: addr; - dst_p: port; - } &log; - - type Info: record { - ## Timestamp attached to the alert. - ts: time &log; - ## Addresses and ports for the connection. - id: PacketID &log; - ## Sensor that originated this event. - sensor_id: count &log; - ## Sig id for this generator. - signature_id: count &log; - ## A string representation of the *signature_id* field if a sid_msg.map file was loaded. - signature: string &log &optional; - ## Which generator generated the alert? - generator_id: count &log; - ## A string representation of the *generator_id* field if a gen_msg.map file was loaded. - generator: string &log &optional; - ## Sig revision for this id. - signature_revision: count &log; - ## Event classification. - classification_id: count &log; - ## Descriptive classification string. - classification: string &log &optional; - ## Event priority. - priority_id: count &log; - ## Event ID. - event_id: count &log; - ## Some of the packet data. - packet: string &log &optional; - } &log; - - ## The event for accessing logged records. - global log_unified2: event(rec: Info); -} - -# Mappings for extended information from alerts. -global classification_map: table[count] of string; -global sid_map: table[count] of string; -global gen_map: table[count] of string; - -global num_classification_map_reads = 0; -global num_sid_map_reads = 0; -global num_gen_map_reads = 0; -global watching = F; - -# For reading in config files. -type OneLine: record { - line: string; -}; - -function mappings_initialized(): bool - { - return num_classification_map_reads > 0 && - num_sid_map_reads > 0 && - num_gen_map_reads > 0; - } - -function start_watching() - { - if ( watching ) - return; - - watching = T; - - if ( watch_dir != "" ) - { - Dir::monitor(watch_dir, function(fname: string) - { - Input::add_analysis([$source=fname, - $reader=Input::READER_BINARY, - $mode=Input::STREAM, - $name=fname]); - }, 10secs); - } - - if ( watch_file != "" ) - { - Input::add_analysis([$source=watch_file, - $reader=Input::READER_BINARY, - $mode=Input::STREAM, - $name=watch_file]); - } - } - -function create_info(ev: IDSEvent): Info - { - local info = Info($ts=ev$ts, - $id=PacketID($src_ip=ev$src_ip, $src_p=ev$src_p, - $dst_ip=ev$dst_ip, $dst_p=ev$dst_p), - $sensor_id=ev$sensor_id, - $signature_id=ev$signature_id, - $generator_id=ev$generator_id, - $signature_revision=ev$signature_revision, - $classification_id=ev$classification_id, - $priority_id=ev$priority_id, - $event_id=ev$event_id); - - if ( ev$signature_id in sid_map ) - info$signature=sid_map[ev$signature_id]; - if ( ev$generator_id in gen_map ) - info$generator=gen_map[ev$generator_id]; - if ( ev$classification_id in classification_map ) - info$classification=classification_map[ev$classification_id]; - - return info; - } - -redef record fa_file += { - ## Recently received IDS events. This is primarily used - ## for tying together Unified2 events and packets. - u2_events: table[count] of Unified2::IDSEvent - &optional &create_expire=5sec - &expire_func=function(t: table[count] of Unified2::IDSEvent, event_id: count): interval - { - Log::write(LOG, create_info(t[event_id])); - return 0secs; - }; -}; - -event Unified2::read_sid_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string) - { - local parts = split_string_n(line, / \|\| /, F, 100); - if ( |parts| >= 2 && /^[0-9]+$/ in parts[0] ) - sid_map[to_count(parts[0])] = parts[1]; - } - -event Unified2::read_gen_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string) - { - local parts = split_string_n(line, / \|\| /, F, 3); - if ( |parts| >= 2 && /^[0-9]+$/ in parts[0] ) - gen_map[to_count(parts[0])] = parts[2]; - } - -event Unified2::read_classification_line(desc: Input::EventDescription, tpe: Input::Event, line: string) - { - local parts = split_string_n(line, /: /, F, 2); - if ( |parts| == 2 ) - { - local parts2 = split_string_n(parts[1], /,/, F, 4); - if ( |parts2| > 1 ) - classification_map[|classification_map|+1] = parts2[0]; - } - } - -event Input::end_of_data(name: string, source: string) - { - if ( name == classification_config ) - ++num_classification_map_reads; - else if ( name == sid_msg ) - ++num_sid_map_reads; - else if ( name == gen_msg ) - ++num_gen_map_reads; - else - return; - - if ( watching ) - return; - - if ( mappings_initialized() ) - start_watching(); - } - -event zeek_init() &priority=5 - { - Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2, $path="unified2", $policy=log_policy]); - - if ( sid_msg == "" ) - { - num_sid_map_reads = 1; - } - else - { - Input::add_event([$source=sid_msg, - $reader=Input::READER_RAW, - $mode=Input::REREAD, - $name=sid_msg, - $fields=Unified2::OneLine, - $want_record=F, - $ev=Unified2::read_sid_msg_line]); - } - - if ( gen_msg == "" ) - { - num_gen_map_reads = 1; - } - else - { - Input::add_event([$source=gen_msg, - $name=gen_msg, - $reader=Input::READER_RAW, - $mode=Input::REREAD, - $fields=Unified2::OneLine, - $want_record=F, - $ev=Unified2::read_gen_msg_line]); - } - - if ( classification_config == "" ) - { - num_classification_map_reads = 1; - } - else - { - Input::add_event([$source=classification_config, - $name=classification_config, - $reader=Input::READER_RAW, - $mode=Input::REREAD, - $fields=Unified2::OneLine, - $want_record=F, - $ev=Unified2::read_classification_line]); - } - - if ( mappings_initialized() ) - start_watching(); - } - -event file_new(f: fa_file) - { - local file_dir = ""; - local parts = split_string_all(f$source, /\/[^\/]*$/); - if ( |parts| == 3 ) - file_dir = parts[0]; - - if ( (watch_file != "" && f$source == watch_file) || - (watch_dir != "" && compress_path(watch_dir) == file_dir) ) - { - Files::add_analyzer(f, Files::ANALYZER_UNIFIED2); - f$u2_events = table(); - } - } - -event unified2_event(f: fa_file, ev: Unified2::IDSEvent) - { - f$u2_events[ev$event_id] = ev; - } - -event unified2_packet(f: fa_file, pkt: Unified2::Packet) - { - if ( f?$u2_events && pkt$event_id in f$u2_events) - { - local ev = f$u2_events[pkt$event_id]; - event Unified2::alert(f, ev, pkt); - delete f$u2_events[pkt$event_id]; - } - } - -event Unified2::alert(f: fa_file, ev: IDSEvent, pkt: Packet) - { - local info = create_info(ev); - info$packet=pkt$data; - Log::write(LOG, info); - } - -event file_state_remove(f: fa_file) - { - if ( f?$u2_events ) - { - # In case any events never had matching packets, flush - # the extras to the log. - for ( i, ev in f$u2_events ) - { - Log::write(LOG, create_info(ev)); - } - } - } diff --git a/scripts/test-all-policy.zeek b/scripts/test-all-policy.zeek index cd7638eea4..467eacbf54 100644 --- a/scripts/test-all-policy.zeek +++ b/scripts/test-all-policy.zeek @@ -65,8 +65,6 @@ @load frameworks/notice/__load__.zeek @load frameworks/notice/actions/drop.zeek @load frameworks/notice/extend-email/hostnames.zeek -@load files/unified2/__load__.zeek -@load files/unified2/main.zeek @load files/x509/disable-certificate-events-known-certs.zeek @load frameworks/packet-filter/shunt.zeek @load frameworks/software/version-changes.zeek diff --git a/src/file_analysis/analyzer/CMakeLists.txt b/src/file_analysis/analyzer/CMakeLists.txt index ef17247997..95f6cbe85c 100644 --- a/src/file_analysis/analyzer/CMakeLists.txt +++ b/src/file_analysis/analyzer/CMakeLists.txt @@ -3,5 +3,4 @@ add_subdirectory(entropy) add_subdirectory(extract) add_subdirectory(hash) add_subdirectory(pe) -add_subdirectory(unified2) add_subdirectory(x509) diff --git a/src/file_analysis/analyzer/unified2/CMakeLists.txt b/src/file_analysis/analyzer/unified2/CMakeLists.txt deleted file mode 100644 index 68c3d2712c..0000000000 --- a/src/file_analysis/analyzer/unified2/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ - -include(ZeekPlugin) - -include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_BINARY_DIR}) - -zeek_plugin_begin(Zeek Unified2) -zeek_plugin_cc(Unified2.cc Plugin.cc) -zeek_plugin_bif(events.bif types.bif) -zeek_plugin_pac(unified2.pac unified2-file.pac unified2-analyzer.pac) -zeek_plugin_end() diff --git a/src/file_analysis/analyzer/unified2/Plugin.cc b/src/file_analysis/analyzer/unified2/Plugin.cc deleted file mode 100644 index e2cd970360..0000000000 --- a/src/file_analysis/analyzer/unified2/Plugin.cc +++ /dev/null @@ -1,28 +0,0 @@ -// See the file in the main distribution directory for copyright. - -// See the file "COPYING" in the main distribution directory for copyright. - -#include "zeek/plugin/Plugin.h" - -#include "zeek/file_analysis/Component.h" -#include "zeek/file_analysis/analyzer/unified2/Unified2.h" - -namespace zeek::plugin::detail::Zeek_Unified2 - { - -class Plugin : public zeek::plugin::Plugin - { -public: - zeek::plugin::Configuration Configure() override - { - AddComponent(new zeek::file_analysis::Component( - "UNIFIED2", zeek::file_analysis::detail::Unified2::Instantiate)); - - zeek::plugin::Configuration config; - config.name = "Zeek::Unified2"; - config.description = "Analyze Unified2 alert files."; - return config; - } - } plugin; - - } // namespace zeek::plugin::detail::Zeek_Unified2 diff --git a/src/file_analysis/analyzer/unified2/Unified2.cc b/src/file_analysis/analyzer/unified2/Unified2.cc deleted file mode 100644 index a675cd8344..0000000000 --- a/src/file_analysis/analyzer/unified2/Unified2.cc +++ /dev/null @@ -1,41 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#include "zeek/file_analysis/analyzer/unified2/Unified2.h" - -#include "zeek/file_analysis/Manager.h" - -namespace zeek::file_analysis::detail - { - -Unified2::Unified2(RecordValPtr args, file_analysis::File* file) - : file_analysis::Analyzer(file_mgr->GetComponentTag("UNIFIED2"), std::move(args), file) - { - interp = new binpac::Unified2::Unified2_Analyzer(this); - } - -Unified2::~Unified2() - { - delete interp; - } - -file_analysis::Analyzer* Unified2::Instantiate(RecordValPtr args, file_analysis::File* file) - { - return new Unified2(std::move(args), file); - } - -bool Unified2::DeliverStream(const u_char* data, uint64_t len) - { - try - { - interp->NewData(true, data, data + len); - } - catch ( const binpac::Exception& e ) - { - printf("Binpac exception: %s\n", e.c_msg()); - return false; - } - - return true; - } - - } // namespace zeek::file_analysis::detail diff --git a/src/file_analysis/analyzer/unified2/Unified2.h b/src/file_analysis/analyzer/unified2/Unified2.h deleted file mode 100644 index 62f883105d..0000000000 --- a/src/file_analysis/analyzer/unified2/Unified2.h +++ /dev/null @@ -1,37 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#pragma once - -#include - -#include "zeek/Val.h" -#include "zeek/file_analysis/Analyzer.h" -#include "zeek/file_analysis/File.h" - -#include "file_analysis/analyzer/unified2/unified2_pac.h" - -namespace zeek::file_analysis::detail - { - -/** - * An analyzer to extract content of files from local disk. - */ -class Unified2 : public file_analysis::Analyzer - { -public: - ~Unified2() override; - - bool DeliverStream(const u_char* data, uint64_t len) override; - - static file_analysis::Analyzer* Instantiate(RecordValPtr args, file_analysis::File* file); - -protected: - Unified2(RecordValPtr args, file_analysis::File* file); - -private: - binpac::Unified2::Unified2_Analyzer* interp; - - string filename; - }; - - } // namespace zeek::file_analysis::detail diff --git a/src/file_analysis/analyzer/unified2/events.bif b/src/file_analysis/analyzer/unified2/events.bif deleted file mode 100644 index a9134e5285..0000000000 --- a/src/file_analysis/analyzer/unified2/events.bif +++ /dev/null @@ -1,17 +0,0 @@ - -## Abstract all of the various Unified2 event formats into -## a single event. -## -## f: The file. -## -## ev: TODO. -## -event unified2_event%(f: fa_file, ev: Unified2::IDSEvent%); - -## The Unified2 packet format event. -## -## f: The file. -## -## pkt: TODO. -## -event unified2_packet%(f: fa_file, pkt: Unified2::Packet%); diff --git a/src/file_analysis/analyzer/unified2/types.bif b/src/file_analysis/analyzer/unified2/types.bif deleted file mode 100644 index 20965f7038..0000000000 --- a/src/file_analysis/analyzer/unified2/types.bif +++ /dev/null @@ -1,2 +0,0 @@ -type Unified2::IDSEvent: record; -type Unified2::Packet: record; diff --git a/src/file_analysis/analyzer/unified2/unified2-analyzer.pac b/src/file_analysis/analyzer/unified2/unified2-analyzer.pac deleted file mode 100644 index d30af665e2..0000000000 --- a/src/file_analysis/analyzer/unified2/unified2-analyzer.pac +++ /dev/null @@ -1,173 +0,0 @@ - -%extern{ -#include "zeek/Event.h" -#include "zeek/file_analysis/File.h" -#include "zeek/IPAddr.h" - -#include "zeek/file_analysis/analyzer/unified2/events.bif.h" -#include "zeek/file_analysis/analyzer/unified2/types.bif.h" -%} - -%code{ -zeek::AddrValPtr binpac::Unified2::Flow::unified2_addr_to_zeek_addr(std::vector* a) - { - if ( a->size() == 1 ) - { - return zeek::make_intrusive(zeek::IPAddr(IPv4, &(a->at(0)), zeek::IPAddr::Host)); - } - else if ( a->size() == 4 ) - { - uint32 tmp[4] = { a->at(0), a->at(1), a->at(2), a->at(3) }; - return zeek::make_intrusive(zeek::IPAddr(IPv6, tmp, zeek::IPAddr::Host)); - } - else - { - // Should never reach here. - return zeek::make_intrusive(1); - } - } - -zeek::ValPtr binpac::Unified2::Flow::to_port(uint16_t n, uint8_t p) - { - TransportProto proto = TRANSPORT_UNKNOWN; - switch ( p ) { - case 1: proto = TRANSPORT_ICMP; break; - case 6: proto = TRANSPORT_TCP; break; - case 17: proto = TRANSPORT_UDP; break; - } - - return zeek::val_mgr->Port(n, proto); - } -%} - -refine flow Flow += { - - %member{ - zeek::AddrValPtr unified2_addr_to_zeek_addr(std::vector* a); - zeek::ValPtr to_port(uint16_t n, uint8_t p); - %} - - %init{ - %} - - %eof{ - %} - - %cleanup{ - %} - - function ts_to_double(ts: Time): double - %{ - double t = ${ts.seconds} + (${ts.microseconds} / 1000000); - return t; - %} - - - #function proc_record(rec: Record) : bool - # %{ - # return true; - # %} - - function proc_ids_event(ev: IDS_Event) : bool - %{ - if ( ::unified2_event ) - { - auto ids_event = zeek::make_intrusive(zeek::BifType::Record::Unified2::IDSEvent); - ids_event->Assign(0, ${ev.sensor_id}); - ids_event->Assign(1, ${ev.event_id}); - ids_event->AssignTime(2, ts_to_double(${ev.ts})); - ids_event->Assign(3, ${ev.signature_id}); - ids_event->Assign(4, ${ev.generator_id}); - ids_event->Assign(5, ${ev.signature_revision}); - ids_event->Assign(6, ${ev.classification_id}); - ids_event->Assign(7, ${ev.priority_id}); - ids_event->Assign(8, unified2_addr_to_zeek_addr(${ev.src_ip})); - ids_event->Assign(9, unified2_addr_to_zeek_addr(${ev.dst_ip})); - ids_event->Assign(10, to_port(${ev.src_p}, ${ev.protocol})); - ids_event->Assign(11, to_port(${ev.dst_p}, ${ev.protocol})); - ids_event->Assign(17, ${ev.packet_action}); - - zeek::event_mgr.Enqueue(::unified2_event, - connection()->zeek_analyzer()->GetFile()->ToVal(), - std::move(ids_event)); - } - return true; - %} - - function proc_ids_event_2(ev: IDS_Event_2) : bool - %{ - if ( ::unified2_event ) - { - auto ids_event = zeek::make_intrusive(zeek::BifType::Record::Unified2::IDSEvent); - ids_event->Assign(0, ${ev.sensor_id}); - ids_event->Assign(1, ${ev.event_id}); - ids_event->AssignTime(2, ts_to_double(${ev.ts})); - ids_event->Assign(3, ${ev.signature_id}); - ids_event->Assign(4, ${ev.generator_id}); - ids_event->Assign(5, ${ev.signature_revision}); - ids_event->Assign(6, ${ev.classification_id}); - ids_event->Assign(7, ${ev.priority_id}); - ids_event->Assign(8, unified2_addr_to_zeek_addr(${ev.src_ip})); - ids_event->Assign(9, unified2_addr_to_zeek_addr(${ev.dst_ip})); - ids_event->Assign(10, to_port(${ev.src_p}, ${ev.protocol})); - ids_event->Assign(11, to_port(${ev.dst_p}, ${ev.protocol})); - ids_event->Assign(12, ${ev.impact_flag}); - ids_event->Assign(13, ${ev.impact}); - ids_event->Assign(14, ${ev.blocked}); - ids_event->Assign(15, ${ev.mpls_label}); - ids_event->Assign(16, ${ev.vlan_id}); - - zeek::event_mgr.Enqueue(::unified2_event, - connection()->zeek_analyzer()->GetFile()->ToVal(), - std::move(ids_event)); - } - - return true; - %} - - function proc_packet(pkt: Packet) : bool - %{ - if ( ::unified2_packet ) - { - auto packet = zeek::make_intrusive(zeek::BifType::Record::Unified2::Packet); - packet->Assign(0, ${pkt.sensor_id}); - packet->Assign(1, ${pkt.event_id}); - packet->Assign(2, ${pkt.event_second}); - packet->AssignTime(3, ts_to_double(${pkt.packet_ts})); - packet->Assign(4, ${pkt.link_type}); - packet->Assign(5, to_stringval(${pkt.packet_data})); - - zeek::event_mgr.Enqueue(::unified2_packet, - connection()->zeek_analyzer()->GetFile()->ToVal(), - std::move(packet)); - } - - return true; - %} - - #function proc_unknown_record_type(rec: UnknownRecordType) : bool - # %{ - # printf("unknown packet type\n"); - # return true; - # %} -}; - -#refine typeattr Record += &let { -# proc : bool = $context.flow.proc_record(this); -#}; - -refine typeattr IDS_Event += &let { - proc : bool = $context.flow.proc_ids_event(this); -}; - -refine typeattr IDS_Event_2 += &let { - proc : bool = $context.flow.proc_ids_event_2(this); -}; - -refine typeattr Packet += &let { - proc : bool = $context.flow.proc_packet(this); -}; - -#refine typeattr UnknownRecordType += &let { -# proc : bool = $context.flow.proc_unknown_record_type(this); -#}; diff --git a/src/file_analysis/analyzer/unified2/unified2-file.pac b/src/file_analysis/analyzer/unified2/unified2-file.pac deleted file mode 100644 index f0d8ca9187..0000000000 --- a/src/file_analysis/analyzer/unified2/unified2-file.pac +++ /dev/null @@ -1,91 +0,0 @@ - -enum Types { - PACKET = 2, - IDS_EVENT = 7, - IDS_EVENT_IPV6 = 72, - IDS_EVENT_2 = 104, - IDS_EVENT_IPV6_2 = 105, - EXTRA_DATA = 110, -}; - -type Time = record { - seconds: uint32; - microseconds: uint32; -} &byteorder=bigendian; - -type Record = record { - rtype: uint32; - length: uint32; - data: case rtype of { - PACKET -> packet: Packet(this); - IDS_EVENT -> ids_event: IDS_Event(this, 1); - IDS_EVENT_IPV6 -> ids_event_ipv6: IDS_Event(this, 4); - IDS_EVENT_2 -> ids_event_vlan: IDS_Event_2(this, 1); - IDS_EVENT_IPV6_2 -> ids_event_ipv6_vlan: IDS_Event_2(this, 4); - #EXTRA_DATA -> extra_data: ExtraData(this); - default -> unknown_record_type: UnknownRecordType(this); - }; -} &byteorder=bigendian &length=length+8; - -type IDS_Event(rec: Record, ip_len: int) = record { - sensor_id: uint32; - event_id: uint32; - ts: Time; - signature_id: uint32; - generator_id: uint32; - signature_revision: uint32; - classification_id: uint32; - priority_id: uint32; - src_ip: uint32[ip_len]; - dst_ip: uint32[ip_len]; - src_p: uint16; - dst_p: uint16; - protocol: uint8; - packet_action: uint8; -} &byteorder=bigendian; - -type IDS_Event_2(rec: Record, ip_len: int) = record { - sensor_id: uint32; - event_id: uint32; - ts: Time; - signature_id: uint32; - generator_id: uint32; - signature_revision: uint32; - classification_id: uint32; - priority_id: uint32; - src_ip: uint32[ip_len]; - dst_ip: uint32[ip_len]; - src_p: uint16; - dst_p: uint16; - protocol: uint8; - impact_flag: uint8; - impact: uint8; - blocked: uint8; - mpls_label: uint32; - vlan_id: uint16; - pad: uint16; -} &byteorder=bigendian; - -type Packet(rec: Record) = record { - sensor_id: uint32; - event_id: uint32; - event_second: uint32; - packet_ts: Time; - link_type: uint32; - packet_len: uint32; - packet_data: bytestring &length=packet_len; -} &byteorder=bigendian; - -type ExtraData(rec: Record) = record { - sensor_id: uint32; - event_id: uint32; - event_second: uint32; - extra_type: uint32; - data_type: uint32; - blob_len: uint32; - blob: bytestring &length=blob_len; -} &byteorder=bigendian &length=rec.length; - -type UnknownRecordType(rec: Record) = record { - data: bytestring &transient &length=rec.length; -} &byteorder=bigendian &length=rec.length; diff --git a/src/file_analysis/analyzer/unified2/unified2.pac b/src/file_analysis/analyzer/unified2/unified2.pac deleted file mode 100644 index f8a4fb993d..0000000000 --- a/src/file_analysis/analyzer/unified2/unified2.pac +++ /dev/null @@ -1,21 +0,0 @@ - -%include binpac.pac -%include zeek.pac - -analyzer Unified2 withcontext { - analyzer: Unified2_Analyzer; - flow: Flow; -}; - -analyzer Unified2_Analyzer(zeek_analyzer: ZeekFileAnalyzer) { - downflow = Flow; - upflow = Flow; -}; - -%include unified2-file.pac - -flow Flow { - flowunit = Record withcontext(connection, this); -}; - -%include unified2-analyzer.pac diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 5ce56d1f46..0bb0cb3630 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -228,8 +228,6 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_FileExtract.functions.bif.zeek build/scripts/base/bif/plugins/Zeek_FileHash.events.bif.zeek build/scripts/base/bif/plugins/Zeek_PE.events.bif.zeek - build/scripts/base/bif/plugins/Zeek_Unified2.events.bif.zeek - build/scripts/base/bif/plugins/Zeek_Unified2.types.bif.zeek build/scripts/base/bif/plugins/Zeek_X509.events.bif.zeek build/scripts/base/bif/plugins/Zeek_X509.types.bif.zeek build/scripts/base/bif/plugins/Zeek_X509.functions.bif.zeek diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 307c31c4ac..754e4fa866 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -228,8 +228,6 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_FileExtract.functions.bif.zeek build/scripts/base/bif/plugins/Zeek_FileHash.events.bif.zeek build/scripts/base/bif/plugins/Zeek_PE.events.bif.zeek - build/scripts/base/bif/plugins/Zeek_Unified2.events.bif.zeek - build/scripts/base/bif/plugins/Zeek_Unified2.types.bif.zeek build/scripts/base/bif/plugins/Zeek_X509.events.bif.zeek build/scripts/base/bif/plugins/Zeek_X509.types.bif.zeek build/scripts/base/bif/plugins/Zeek_X509.functions.bif.zeek diff --git a/testing/btest/Baseline/coverage.find-bro-logs/out b/testing/btest/Baseline/coverage.find-bro-logs/out index b2cb315069..a5175aa450 100644 --- a/testing/btest/Baseline/coverage.find-bro-logs/out +++ b/testing/btest/Baseline/coverage.find-bro-logs/out @@ -61,7 +61,6 @@ telemetry telemetry_histogram traceroute tunnel -unified2 unknown_protocols weird weird_stats diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 2333adc739..831f460d87 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -850,8 +850,6 @@ 0.000000 MetaHookPost LoadFile(0, ./Zeek_Teredo.events.bif.zeek, <...>/Zeek_Teredo.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_Teredo.functions.bif.zeek, <...>/Zeek_Teredo.functions.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_UDP.events.bif.zeek, <...>/Zeek_UDP.events.bif.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, ./Zeek_Unified2.events.bif.zeek, <...>/Zeek_Unified2.events.bif.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, ./Zeek_Unified2.types.bif.zeek, <...>/Zeek_Unified2.types.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_VXLAN.events.bif.zeek, <...>/Zeek_VXLAN.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_X509.events.bif.zeek, <...>/Zeek_X509.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_X509.functions.bif.zeek, <...>/Zeek_X509.functions.bif.zeek) -> -1 @@ -1236,8 +1234,6 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Teredo.events.bif.zeek, <...>/Zeek_Teredo.events.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Teredo.functions.bif.zeek, <...>/Zeek_Teredo.functions.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_UDP.events.bif.zeek, <...>/Zeek_UDP.events.bif.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Unified2.events.bif.zeek, <...>/Zeek_Unified2.events.bif.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Unified2.types.bif.zeek, <...>/Zeek_Unified2.types.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_VXLAN.events.bif.zeek, <...>/Zeek_VXLAN.events.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_X509.events.bif.zeek, <...>/Zeek_X509.events.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_X509.functions.bif.zeek, <...>/Zeek_X509.functions.bif.zeek) -> (-1, ) @@ -2357,8 +2353,6 @@ 0.000000 MetaHookPre LoadFile(0, ./Zeek_Teredo.events.bif.zeek, <...>/Zeek_Teredo.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_Teredo.functions.bif.zeek, <...>/Zeek_Teredo.functions.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_UDP.events.bif.zeek, <...>/Zeek_UDP.events.bif.zeek) -0.000000 MetaHookPre LoadFile(0, ./Zeek_Unified2.events.bif.zeek, <...>/Zeek_Unified2.events.bif.zeek) -0.000000 MetaHookPre LoadFile(0, ./Zeek_Unified2.types.bif.zeek, <...>/Zeek_Unified2.types.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_VXLAN.events.bif.zeek, <...>/Zeek_VXLAN.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_X509.events.bif.zeek, <...>/Zeek_X509.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_X509.functions.bif.zeek, <...>/Zeek_X509.functions.bif.zeek) @@ -2743,8 +2737,6 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Teredo.events.bif.zeek, <...>/Zeek_Teredo.events.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Teredo.functions.bif.zeek, <...>/Zeek_Teredo.functions.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_UDP.events.bif.zeek, <...>/Zeek_UDP.events.bif.zeek) -0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Unified2.events.bif.zeek, <...>/Zeek_Unified2.events.bif.zeek) -0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Unified2.types.bif.zeek, <...>/Zeek_Unified2.types.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_VXLAN.events.bif.zeek, <...>/Zeek_VXLAN.events.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_X509.events.bif.zeek, <...>/Zeek_X509.events.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_X509.functions.bif.zeek, <...>/Zeek_X509.functions.bif.zeek) @@ -3863,8 +3855,6 @@ 0.000000 | HookLoadFile ./Zeek_Teredo.events.bif.zeek <...>/Zeek_Teredo.events.bif.zeek 0.000000 | HookLoadFile ./Zeek_Teredo.functions.bif.zeek <...>/Zeek_Teredo.functions.bif.zeek 0.000000 | HookLoadFile ./Zeek_UDP.events.bif.zeek <...>/Zeek_UDP.events.bif.zeek -0.000000 | HookLoadFile ./Zeek_Unified2.events.bif.zeek <...>/Zeek_Unified2.events.bif.zeek -0.000000 | HookLoadFile ./Zeek_Unified2.types.bif.zeek <...>/Zeek_Unified2.types.bif.zeek 0.000000 | HookLoadFile ./Zeek_VXLAN.events.bif.zeek <...>/Zeek_VXLAN.events.bif.zeek 0.000000 | HookLoadFile ./Zeek_X509.events.bif.zeek <...>/Zeek_X509.events.bif.zeek 0.000000 | HookLoadFile ./Zeek_X509.functions.bif.zeek <...>/Zeek_X509.functions.bif.zeek @@ -4249,8 +4239,6 @@ 0.000000 | HookLoadFileExtended ./Zeek_Teredo.events.bif.zeek <...>/Zeek_Teredo.events.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_Teredo.functions.bif.zeek <...>/Zeek_Teredo.functions.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_UDP.events.bif.zeek <...>/Zeek_UDP.events.bif.zeek -0.000000 | HookLoadFileExtended ./Zeek_Unified2.events.bif.zeek <...>/Zeek_Unified2.events.bif.zeek -0.000000 | HookLoadFileExtended ./Zeek_Unified2.types.bif.zeek <...>/Zeek_Unified2.types.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_VXLAN.events.bif.zeek <...>/Zeek_VXLAN.events.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_X509.events.bif.zeek <...>/Zeek_X509.events.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_X509.functions.bif.zeek <...>/Zeek_X509.functions.bif.zeek diff --git a/testing/btest/Baseline/scripts.base.files.unified2.alert/unified2.log b/testing/btest/Baseline/scripts.base.files.unified2.alert/unified2.log deleted file mode 100644 index 6b99c4356d..0000000000 --- a/testing/btest/Baseline/scripts.base.files.unified2.alert/unified2.log +++ /dev/null @@ -1,12 +0,0 @@ -### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path unified2 -#open XXXX-XX-XX-XX-XX-XX -#fields ts id.src_ip id.src_p id.dst_ip id.dst_p sensor_id signature_id signature generator_id generator signature_revision classification_id classification priority_id event_id packet -#types time addr port addr port count count string count string count count string count count string -XXXXXXXXXX.XXXXXX 192.168.1.72 50185 74.125.225.49 80 0 2003058 ET MALWARE 180solutions (Zango) Spyware Installer Download 1 snort general alert 5 21 trojan-activity 1 2 \xd80bH\xc5\xb5x\xca9\xb7\xe4r\x08\x00E\x10\x00\\\x1a\xce@\x00@\x062\x1f\xc0\xa8\x01HJ}\xe11\xc4\x09\x00P*\xa8bv]z/\xde\x80\x18\x82+\x88,\x00\x00\x01\x01\x08\x0a\x17J\x83Q\xfe\xad\xac\x1aGET /Zango/ZangoInstaller.exe HTTP/1.0\x0d\x0a -XXXXXXXXXX.XXXXXX 192.168.1.72 49862 199.47.216.144 80 0 2012647 ET POLICY Dropbox.com Offsite File Backup in Use 1 snort general alert 3 33 policy-violation 1 3 \xd80bH\xc5\xb5x\xca9\xb7\xe4r\x08\x00E\x00\x00\xf8Q\xdf@\x00@\x06\x86p\xc0\xa8\x01H\xc7/\xd8\x90\xc2\xc6\x00P\x9cm\x97U\xf07\x084\x80\x18\x82\x18%<\x00\x00\x01\x01\x08\x0a\x17J\xd7\xde\x00\x92\x81\xc5GET /subscribe?host_int=43112345&ns_map=123456_1234524412104916591&ts=1323827344 HTTP/1.1\x0d\x0aHost: notify1.dropbox.com\x0d\x0aAccept-Encoding: identity\x0d\x0aConnection: keep-alive\x0d\x0aX-Dropbox-Locale: en_US\x0d\x0a\x0d\x0a -#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Files/unified2.u2 b/testing/btest/Files/unified2.u2 deleted file mode 100755 index 2619152e1449dcba038027753738d44954a1eed5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 576 zcmZQzV8~!#V6XvV5Mc6q!BEY>6lTjHH=zi~X9QwaAQlC&53FGH@Tz@ic!ZN7fDI^s z1ek#AHYgu#2HYHo8Cf?Bl01%XtvF@5{YeoAgR1~TjMO;?1_w4Hd1O?iO;=z=WiHV_+p@E5|p_!?rp;}3?t)a1z zv4xSjv55&R5Da;_Jb-3dDdgpsWTsUb>ZKIr7bN9Z=q2ap@^U#QC#M#a=(^@5=ci=m zrCTXvrljVTWR_I&ayjSc<)tQ