diff --git a/configure b/configure index 48c35e64a5..a54e1248d2 100755 --- a/configure +++ b/configure @@ -47,6 +47,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]... Optional Features: --enable-debug compile in debugging mode (like --build-type=Debug) --enable-coverage compile with code coverage support (implies debugging mode) + --enable-fuzzing build fuzzing targets --enable-mobile-ipv6 analyze mobile IPv6 features defined by RFC 6275 --enable-perftools enable use of Google perftools (use tcmalloc) --enable-perftools-debug use Google's perftools for debugging @@ -229,6 +230,9 @@ while [ $# -ne 0 ]; do append_cache_entry ENABLE_COVERAGE BOOL true append_cache_entry ENABLE_DEBUG BOOL true ;; + --enable-fuzzing) + append_cache_entry ZEEK_ENABLE_FUZZING BOOL true + ;; --enable-debug) append_cache_entry ENABLE_DEBUG BOOL true ;; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8e533f5733..9072966a54 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -153,6 +153,8 @@ add_subdirectory(iosource) add_subdirectory(logging) add_subdirectory(probabilistic) +add_subdirectory(fuzzers) + ######################################################################## ## bro target @@ -204,12 +206,12 @@ set_source_files_properties(nb_dns.c PROPERTIES COMPILE_FLAGS -fno-strict-aliasing) set(MAIN_SRCS - main.cc digest.cc net_util.cc util.cc module_util.cc zeek-affinity.cc + zeek-setup.cc Anon.cc Attr.cc Base64.cc @@ -316,7 +318,7 @@ set(THIRD_PARTY_SRCS 3rdparty/sqlite3.c ) -set(bro_SRCS +set(zeek_SRCS ${CMAKE_CURRENT_BINARY_DIR}/version.c ${BIF_SRCS} ${BINPAC_AUXSRC} @@ -336,9 +338,16 @@ set(bro_SRCS ${MAIN_SRCS} ) -collect_headers(bro_HEADERS ${bro_SRCS}) +collect_headers(zeek_HEADERS ${zeek_SRCS}) -add_executable(zeek ${bro_SRCS} ${bro_HEADERS} ${bro_SUBDIR_LIBS} ${bro_PLUGIN_LIBS}) +add_library(zeek_objs OBJECT ${zeek_SRCS}) + +add_executable(zeek main.cc + $ + ${zeek_HEADERS} + ${bro_SUBDIR_LIBS} + ${bro_PLUGIN_LIBS} +) target_link_libraries(zeek ${zeekdeps} ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) if ( NOT "${bro_LINKER_FLAGS}" STREQUAL "" ) @@ -381,12 +390,12 @@ add_dependencies(generate_outputs generate_outputs_stage2a generate_outputs_stag # Build __load__.zeek files for standard *.bif.zeek. bro_bif_create_loader(bif_loader "${bro_BASE_BIF_SCRIPTS}") add_dependencies(bif_loader ${bro_PLUGIN_DEPS} ${bro_SUBDIR_DEPS}) -add_dependencies(zeek bif_loader) +add_dependencies(zeek_objs bif_loader) # Build __load__.zeek files for plugins/*.bif.zeek. bro_bif_create_loader(bif_loader_plugins "${bro_PLUGIN_BIF_SCRIPTS}") add_dependencies(bif_loader_plugins ${bro_PLUGIN_DEPS} ${bro_SUBDIR_DEPS}) -add_dependencies(zeek bif_loader_plugins) +add_dependencies(zeek_objs bif_loader_plugins) # Install *.bif.zeek. install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/base/bif DESTINATION ${ZEEK_SCRIPT_INSTALL_PATH}/base) diff --git a/src/Options.cc b/src/Options.cc index a68f76595c..d7799cab84 100644 --- a/src/Options.cc +++ b/src/Options.cc @@ -41,6 +41,8 @@ void zeek::Options::filter_supervised_node_options() bare_mode = og.bare_mode; perftools_check_leaks = og.perftools_check_leaks; perftools_profile = og.perftools_profile; + deterministic_mode = og.deterministic_mode; + abort_on_scripting_errors = og.abort_on_scripting_errors; pcap_filter = og.pcap_filter; signature_files = og.signature_files; @@ -93,6 +95,7 @@ void zeek::usage(const char* prog, int code) fprintf(stderr, " -B|--debug | Enable debugging output for selected streams ('-B help' for help)\n"); #endif fprintf(stderr, " -C|--no-checksums | ignore checksums\n"); + fprintf(stderr, " -D|--deterministic | initialize random seeds to zero\n"); fprintf(stderr, " -F|--force-dns | force DNS\n"); fprintf(stderr, " -G|--load-seeds | load seeds from given file\n"); fprintf(stderr, " -H|--save-seeds | save seeds to given file\n"); @@ -202,6 +205,7 @@ zeek::Options zeek::parse_cmdline(int argc, char** argv) {"version", no_argument, nullptr, 'v'}, {"no-checksums", no_argument, nullptr, 'C'}, {"force-dns", no_argument, nullptr, 'F'}, + {"deterministic", no_argument, nullptr, 'D'}, {"load-seeds", required_argument, nullptr, 'G'}, {"save-seeds", required_argument, nullptr, 'H'}, {"print-plugins", no_argument, nullptr, 'N'}, @@ -232,7 +236,7 @@ zeek::Options zeek::parse_cmdline(int argc, char** argv) }; char opts[256]; - safe_strncpy(opts, "B:e:f:G:H:I:i:j::n:p:r:s:T:t:U:w:X:CFNPQSWabdhv", + safe_strncpy(opts, "B:e:f:G:H:I:i:j::n:p:r:s:T:t:U:w:X:CDFNPQSWabdhv", sizeof(opts)); #ifdef USE_PERFTOOLS_DEBUG @@ -329,6 +333,9 @@ zeek::Options zeek::parse_cmdline(int argc, char** argv) case 'C': rval.ignore_checksums = true; break; + case 'D': + rval.deterministic_mode = true; + break; case 'E': rval.pseudo_realtime = 1.0; if ( optarg ) diff --git a/src/Options.h b/src/Options.h index 496a5fa203..2dcbdea853 100644 --- a/src/Options.h +++ b/src/Options.h @@ -53,6 +53,8 @@ struct Options { bool debug_scripts = false; bool perftools_check_leaks = false; bool perftools_profile = false; + bool deterministic_mode = false; + bool abort_on_scripting_errors = false; bool run_unit_tests = false; std::vector doctest_args; diff --git a/src/Reporter.cc b/src/Reporter.cc index e49c9eca75..a4ce65b88b 100644 --- a/src/Reporter.cc +++ b/src/Reporter.cc @@ -32,8 +32,9 @@ int closelog(); Reporter* reporter = nullptr; -Reporter::Reporter() +Reporter::Reporter(bool arg_abort_on_scripting_errors) { + abort_on_scripting_errors = arg_abort_on_scripting_errors; errors = 0; via_events = false; in_error_handler = 0; @@ -157,6 +158,10 @@ void Reporter::ExprRuntimeError(const Expr* expr, const char* fmt, ...) d.Description(), fmt, ap); va_end(ap); PopLocation(); + + if ( abort_on_scripting_errors ) + abort(); + throw InterpreterException(); } @@ -170,6 +175,10 @@ void Reporter::RuntimeError(const Location* location, const char* fmt, ...) DoLog("runtime error", reporter_error, out, nullptr, nullptr, true, true, "", fmt, ap); va_end(ap); PopLocation(); + + if ( abort_on_scripting_errors ) + abort(); + throw InterpreterException(); } diff --git a/src/Reporter.h b/src/Reporter.h index 97f1759963..9224b3e8b6 100644 --- a/src/Reporter.h +++ b/src/Reporter.h @@ -44,7 +44,7 @@ public: using WeirdFlowMap = std::map; using WeirdSet = std::unordered_set; - Reporter(); + Reporter(bool abort_on_scripting_errors); ~Reporter(); // Initialize reporter-sepcific options that are defined in script-layer. @@ -262,6 +262,7 @@ private: bool warnings_to_stderr; bool errors_to_stderr; bool after_zeek_init; + bool abort_on_scripting_errors = false; std::list > locations; diff --git a/src/Sessions.cc b/src/Sessions.cc index 2970a5383f..44f50b6e29 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -1107,6 +1107,23 @@ void NetSessions::Drain() } } +void NetSessions::Clear() + { + for ( const auto& entry : tcp_conns ) + Unref(entry.second); + for ( const auto& entry : udp_conns ) + Unref(entry.second); + for ( const auto& entry : icmp_conns ) + Unref(entry.second); + for ( const auto& entry : fragments ) + Unref(entry.second); + + tcp_conns.clear(); + udp_conns.clear(); + icmp_conns.clear(); + fragments.clear(); + } + void NetSessions::GetStats(SessionStats& s) const { s.num_TCP_conns = tcp_conns.size(); diff --git a/src/Sessions.h b/src/Sessions.h index 9cefd6156a..67589d8742 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -72,6 +72,9 @@ public: // that are still active. void Drain(); + // Clears the session maps. + void Clear(); + void GetStats(SessionStats& s) const; void Weird(const char* name, const Packet* pkt, diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 8c72ca7cc6..05a10397b2 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -124,10 +124,10 @@ static std::string RenderMessage(const broker::error& e) #endif -Manager::Manager(bool arg_reading_pcaps) +Manager::Manager(bool arg_use_real_time) { bound_port = 0; - reading_pcaps = arg_reading_pcaps; + use_real_time = arg_use_real_time; after_zeek_init = false; peer_count = 0; log_batch_size = 0; @@ -166,7 +166,7 @@ void Manager::InitPostScript() broker::broker_options options; options.disable_ssl = get_option("Broker::disable_ssl")->AsBool(); options.forward = get_option("Broker::forward_messages")->AsBool(); - options.use_real_time = ! reading_pcaps; + options.use_real_time = use_real_time; BrokerConfig config{std::move(options)}; @@ -283,6 +283,14 @@ void Manager::FlushPendingQueries() } } +void Manager::ClearStores() + { + FlushPendingQueries(); + + for ( const auto& [name, handle] : data_stores ) + handle->store.clear(); + } + uint16_t Manager::Listen(const string& addr, uint16_t port) { if ( bstate->endpoint.is_shutdown() ) diff --git a/src/broker/Manager.h b/src/broker/Manager.h index 2a38967458..bbd38f88be 100644 --- a/src/broker/Manager.h +++ b/src/broker/Manager.h @@ -64,7 +64,7 @@ public: /** * Constructor. */ - Manager(bool reading_pcaps); + Manager(bool use_real_time); /** * Destructor. @@ -316,6 +316,11 @@ public: */ size_t FlushLogBuffers(); + /** + * Flushes all pending data store queries and also clears all contents. + */ + void ClearStores(); + /** * @return communication statistics. */ @@ -383,7 +388,7 @@ private: Stats statistics; uint16_t bound_port; - bool reading_pcaps; + bool use_real_time; bool after_zeek_init; int peer_count; diff --git a/src/fuzzers/CMakeLists.txt b/src/fuzzers/CMakeLists.txt new file mode 100644 index 0000000000..6e6d3576b2 --- /dev/null +++ b/src/fuzzers/CMakeLists.txt @@ -0,0 +1,30 @@ +######################################################################## +## Fuzzing targets + +if ( NOT ZEEK_ENABLE_FUZZING ) + return() +endif () + +macro(ADD_FUZZ_TARGET _name) + set(_fuzz_target zeek_fuzzer_${_name}) + set(_fuzz_source ${_name}.cc) + + add_executable(${_fuzz_target} ${_fuzz_source} ${ARGN} + $ + $ + ${zeek_HEADERS} + ${bro_SUBDIR_LIBS} + ${bro_PLUGIN_LIBS}) + + target_link_libraries(${_fuzz_target} ${zeekdeps} + ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) + + # TODO: Use LIB_FUZZING_ENGINE env. var. if it exists + set_target_properties(${_fuzz_target} PROPERTIES LINK_FLAGS + "-fsanitize=fuzzer") +endmacro () + +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}) +add_library(zeek_fuzzer_objs OBJECT FuzzBuffer.cc) + +add_fuzz_target(pop3) diff --git a/src/fuzzers/FuzzBuffer.cc b/src/fuzzers/FuzzBuffer.cc new file mode 100644 index 0000000000..cc25550983 --- /dev/null +++ b/src/fuzzers/FuzzBuffer.cc @@ -0,0 +1,56 @@ +#if !defined(_GNU_SOURCE) +#define _GNU_SOURCE +#endif + +#include + +#include "FuzzBuffer.h" + +bool zeek::FuzzBuffer::Valid() const + { + if ( end - begin < PKT_MAGIC_LEN + 2 ) + return false; + + if ( memcmp(begin, PKT_MAGIC, PKT_MAGIC_LEN) != 0) + return false; + + return true; + } + +int zeek::FuzzBuffer::Next(const unsigned char** chunk, size_t* len, bool* is_orig) + { + if ( begin == end ) + { + *chunk = nullptr; + *len = 0; + return 0; + } + + auto pos = (const unsigned char*)memmem(begin, end - begin, + PKT_MAGIC, PKT_MAGIC_LEN); + + if ( ! pos ) + return -1; + + begin += PKT_MAGIC_LEN; + auto remaining = end - begin; + + if ( remaining < 2 ) + return -2; + + *is_orig = begin[0] & 0x01; + begin += 1; + + *chunk = begin; + + auto next = (const unsigned char*)memmem(begin, end - begin, + PKT_MAGIC, PKT_MAGIC_LEN); + + if ( next ) + begin = next; + else + begin = end; + + *len = begin - *chunk; + return 0; + } diff --git a/src/fuzzers/FuzzBuffer.h b/src/fuzzers/FuzzBuffer.h new file mode 100644 index 0000000000..b677e2ac5c --- /dev/null +++ b/src/fuzzers/FuzzBuffer.h @@ -0,0 +1,24 @@ +#pragma once + +#include + +namespace zeek { + +struct FuzzBuffer { + + static constexpr int PKT_MAGIC_LEN = 4; + static constexpr unsigned char PKT_MAGIC[PKT_MAGIC_LEN + 1] = "\1PKT"; + + FuzzBuffer(const unsigned char* data, size_t size) + : begin(data), end(data + size) + { } + + bool Valid() const; + + int Next(const unsigned char** chunk, size_t* len, bool* is_orig); + + const unsigned char* begin; + const unsigned char* end; +}; + +} // namespace zeek diff --git a/src/fuzzers/fuzz-setup.h b/src/fuzzers/fuzz-setup.h new file mode 100644 index 0000000000..1ad8faff6a --- /dev/null +++ b/src/fuzzers/fuzz-setup.h @@ -0,0 +1,41 @@ +#pragma once + +#include + +#include "zeek-setup.h" + +#include "Event.h" +#include "Sessions.h" +#include "broker/Manager.h" +#include "file_analysis/Manager.h" + +extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) + { + zeek::Options options; + options.scripts_to_load.emplace_back("local.zeek"); + options.script_options_to_set.emplace_back("Site::local_nets={10.0.0.0/8}"); + options.script_options_to_set.emplace_back("Log::default_writer=Log::WRITER_NONE"); + options.deterministic_mode = true; + options.abort_on_scripting_errors = true; + + if ( zeek::setup(*argc, *argv, &options).code ) + abort(); + + return 0; + } + +namespace zeek { + +void fuzz_cleanup_one_input() + { + broker_mgr->ClearStores(); + file_mgr->Terminate(); + timer_mgr->Expire(); + + mgr.Drain(); + sessions->Drain(); + mgr.Drain(); + sessions->Clear(); + } + +} // namespace zeek diff --git a/src/fuzzers/pop3.cc b/src/fuzzers/pop3.cc new file mode 100644 index 0000000000..e98afec36f --- /dev/null +++ b/src/fuzzers/pop3.cc @@ -0,0 +1,83 @@ +#include "binpac.h" + +#include "Net.h" +#include "Conn.h" +#include "Sessions.h" +#include "analyzer/Analyzer.h" +#include "analyzer/Manager.h" +#include "analyzer/protocol/pia/PIA.h" +#include "analyzer/protocol/tcp/TCP.h" + +#include "FuzzBuffer.h" +#include "fuzz-setup.h" + +static constexpr auto ZEEK_FUZZ_ANALYZER = "pop3"; + +static Connection* add_connection() + { + static constexpr double network_time_start = 1439471031; + net_update_time(network_time_start); + + Packet p; + ConnID conn_id; + conn_id.src_addr = IPAddr("1.2.3.4"); + conn_id.dst_addr = IPAddr("5.6.7.8"); + conn_id.src_port = htons(23132); + conn_id.dst_port = htons(80); + ConnIDKey key = BuildConnIDKey(conn_id); + Connection* conn = new Connection(sessions, key, network_time_start, + &conn_id, 1, &p, nullptr); + conn->SetTransport(TRANSPORT_TCP); + sessions->Insert(conn); + return conn; + } + +static analyzer::Analyzer* add_analyzer(Connection* conn) + { + analyzer::tcp::TCP_Analyzer* tcp = new analyzer::tcp::TCP_Analyzer(conn); + analyzer::pia::PIA* pia = new analyzer::pia::PIA_TCP(conn); + auto a = analyzer_mgr->InstantiateAnalyzer(ZEEK_FUZZ_ANALYZER, conn); + tcp->AddChildAnalyzer(a); + tcp->AddChildAnalyzer(pia->AsAnalyzer()); + conn->SetRootAnalyzer(tcp, pia); + return a; + } + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) + { + zeek::FuzzBuffer fb{data, size}; + + if ( ! fb.Valid() ) + return 0; + + auto conn = add_connection(); + auto a = add_analyzer(conn); + + const unsigned char* chunk; + size_t chunk_size; + bool is_orig; + + for ( ; ; ) + { + auto err = fb.Next(&chunk, &chunk_size, &is_orig); + + if ( err ) + break; + + if ( chunk_size == 0 ) + break; + + try + { + a->DeliverStream(chunk_size, chunk, is_orig); + } + catch ( const binpac::Exception& e ) + { + } + + mgr.Drain(); + } + + zeek::fuzz_cleanup_one_input(); + return 0; + } diff --git a/src/main.cc b/src/main.cc index b44543173a..7d7e876f41 100644 --- a/src/main.cc +++ b/src/main.cc @@ -1,902 +1,26 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek-config.h" +#include "zeek-setup.h" -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef USE_IDMEF -extern "C" { -#include -} -#endif - -#include -#include - -#include "Options.h" -#include "input.h" -#include "DNS_Mgr.h" -#include "Frame.h" -#include "Scope.h" -#include "Event.h" -#include "File.h" -#include "Reporter.h" -#include "Net.h" -#include "NetVar.h" -#include "Var.h" -#include "Timer.h" -#include "Stmt.h" -#include "Desc.h" -#include "Debug.h" -#include "DFA.h" -#include "RuleMatcher.h" -#include "Anon.h" -#include "EventRegistry.h" -#include "Stats.h" -#include "Brofiler.h" -#include "Traverse.h" -#include "Trigger.h" - -#include "supervisor/Supervisor.h" -#include "threading/Manager.h" -#include "input/Manager.h" -#include "logging/Manager.h" -#include "input/readers/raw/Raw.h" -#include "analyzer/Manager.h" -#include "analyzer/Tag.h" -#include "plugin/Manager.h" -#include "file_analysis/Manager.h" -#include "zeekygen/Manager.h" #include "iosource/Manager.h" -#include "broker/Manager.h" - -#include "binpac_bro.h" - -#include "3rdparty/sqlite3.h" - -#define DOCTEST_CONFIG_IMPLEMENT -#include "3rdparty/doctest.h" - -Brofiler brofiler; - -#ifndef HAVE_STRSEP -extern "C" { -char* strsep(char**, const char*); -}; -#endif - -extern "C" { -#include "setsignal.h" -}; - -#ifdef USE_PERFTOOLS_DEBUG -HeapLeakChecker* heap_checker = 0; -int perftools_leaks = 0; -int perftools_profile = 0; -#endif - -DNS_Mgr* dns_mgr; -TimerMgr* timer_mgr; -ValManager* val_mgr = nullptr; -logging::Manager* log_mgr = nullptr; -threading::Manager* thread_mgr = nullptr; -input::Manager* input_mgr = nullptr; -plugin::Manager* plugin_mgr = nullptr; -analyzer::Manager* analyzer_mgr = nullptr; -file_analysis::Manager* file_mgr = nullptr; -zeekygen::Manager* zeekygen_mgr = nullptr; -iosource::Manager* iosource_mgr = nullptr; -bro_broker::Manager* broker_mgr = nullptr; -zeek::Supervisor* zeek::supervisor_mgr = nullptr; -trigger::Manager* trigger_mgr = nullptr; - -std::vector zeek_script_prefixes; -Stmt* stmts; -EventHandlerPtr net_done = nullptr; -RuleMatcher* rule_matcher = nullptr; -EventRegistry* event_registry = nullptr; -ProfileLogger* profiling_logger = nullptr; -ProfileLogger* segment_logger = nullptr; -SampleLogger* sample_logger = nullptr; -int signal_val = 0; -extern char version[]; -const char* command_line_policy = nullptr; -vector params; -set requested_plugins; -const char* proc_status_file = nullptr; - -OpaqueType* md5_type = nullptr; -OpaqueType* sha1_type = nullptr; -OpaqueType* sha256_type = nullptr; -OpaqueType* entropy_type = nullptr; -OpaqueType* cardinality_type = nullptr; -OpaqueType* topk_type = nullptr; -OpaqueType* bloomfilter_type = nullptr; -OpaqueType* x509_opaque_type = nullptr; -OpaqueType* ocsp_resp_opaque_type = nullptr; -OpaqueType* paraglob_type = nullptr; - -// Keep copy of command line -int bro_argc; -char** bro_argv; - -const char* zeek_version() - { -#ifdef DEBUG - static char* debug_version = nullptr; - - if ( ! debug_version ) - { - int n = strlen(version) + sizeof("-debug") + 1; - debug_version = new char[n]; - snprintf(debug_version, n, "%s%s", version, "-debug"); - } - - return debug_version; -#else - return version; -#endif - } - -static std::vector to_cargs(const std::vector& args) - { - std::vector rval; - rval.reserve(args.size()); - - for ( const auto& arg : args ) - rval.emplace_back(arg.data()); - - return rval; - } - -bool show_plugins(int level) - { - plugin::Manager::plugin_list plugins = plugin_mgr->ActivePlugins(); - - if ( ! plugins.size() ) - { - printf("No plugins registered, not even any built-ins. This is probably a bug.\n"); - return false; - } - - ODesc d; - - if ( level == 1 ) - d.SetShort(); - - int count = 0; - - for ( plugin::Manager::plugin_list::const_iterator i = plugins.begin(); i != plugins.end(); i++ ) - { - if ( requested_plugins.size() - && requested_plugins.find((*i)->Name()) == requested_plugins.end() ) - continue; - - (*i)->Describe(&d); - - if ( ! d.IsShort() ) - d.Add("\n"); - - ++count; - } - - printf("%s", d.Description()); - - plugin::Manager::inactive_plugin_list inactives = plugin_mgr->InactivePlugins(); - - if ( inactives.size() && ! requested_plugins.size() ) - { - printf("\nInactive dynamic plugins:\n"); - - for ( plugin::Manager::inactive_plugin_list::const_iterator i = inactives.begin(); i != inactives.end(); i++ ) - { - string name = (*i).first; - string path = (*i).second; - printf(" %s (%s)\n", name.c_str(), path.c_str()); - } - } - - return count != 0; - } - -void done_with_network() - { - set_processing_status("TERMINATING", "done_with_network"); - - // Cancel any pending alarms (watchdog, in particular). - (void) alarm(0); - - if ( net_done ) - { - mgr.Drain(); - // Don't propagate this event to remote clients. - mgr.Dispatch(new Event(net_done, - {make_intrusive(timer_mgr->Time(), TYPE_TIME)}), - true); - } - - if ( profiling_logger ) - profiling_logger->Log(); - - terminating = true; - - analyzer_mgr->Done(); - timer_mgr->Expire(); - dns_mgr->Flush(); - mgr.Drain(); - mgr.Drain(); - - net_finish(1); - -#ifdef USE_PERFTOOLS_DEBUG - - if ( perftools_profile ) - { - HeapProfilerDump("post net_run"); - HeapProfilerStop(); - } - - if ( heap_checker && ! heap_checker->NoLeaks() ) - { - fprintf(stderr, "Memory leaks - aborting.\n"); - abort(); - } -#endif - - ZEEK_LSAN_DISABLE(); - } - -void terminate_bro() - { - set_processing_status("TERMINATING", "terminate_bro"); - - terminating = true; - - iosource_mgr->Wakeup("terminate_bro"); - - // File analysis termination may produce events, so do it early on in - // the termination process. - file_mgr->Terminate(); - - brofiler.WriteStats(); - - EventHandlerPtr zeek_done = internal_handler("zeek_done"); - if ( zeek_done ) - mgr.Enqueue(zeek_done, zeek::Args{}); - - timer_mgr->Expire(); - mgr.Drain(); - - if ( profiling_logger ) - { - // FIXME: There are some occasional crashes in the memory - // allocation code when killing Bro. Disabling this for now. - if ( ! (signal_val == SIGTERM || signal_val == SIGINT) ) - profiling_logger->Log(); - - delete profiling_logger; - } - - mgr.Drain(); - - notifier::registry.Terminate(); - log_mgr->Terminate(); - input_mgr->Terminate(); - thread_mgr->Terminate(); - broker_mgr->Terminate(); - dns_mgr->Terminate(); - - mgr.Drain(); - - plugin_mgr->FinishPlugins(); - - delete zeekygen_mgr; - delete analyzer_mgr; - delete file_mgr; - // broker_mgr, timer_mgr, and supervisor are deleted via iosource_mgr - delete iosource_mgr; - delete event_registry; - delete log_mgr; - delete reporter; - delete plugin_mgr; - delete val_mgr; - - // free the global scope - pop_scope(); - - reporter = nullptr; - } - -void zeek_terminate_loop(const char* reason) - { - set_processing_status("TERMINATING", reason); - reporter->Info("%s", reason); - - net_get_final_stats(); - done_with_network(); - net_delete(); - - terminate_bro(); - - // Close files after net_delete(), because net_delete() - // might write to connection content files. - BroFile::CloseOpenFiles(); - - delete rule_matcher; - - exit(0); - } - -RETSIGTYPE sig_handler(int signo) - { - set_processing_status("TERMINATING", "sig_handler"); - signal_val = signo; - - if ( ! terminating ) - iosource_mgr->Wakeup("sig_handler"); - - return RETSIGVAL; - } - -static void atexit_handler() - { - set_processing_status("TERMINATED", "atexit"); - } - -static void bro_new_handler() - { - out_of_memory("new"); - } - -static std::vector get_script_signature_files() - { - std::vector rval; - - // Parse rule files defined on the script level. - char* script_signature_files = - copy_string(internal_val("signature_files")->AsString()->CheckString()); - - char* tmp = script_signature_files; - char* s; - while ( (s = strsep(&tmp, " \t")) ) - if ( *s ) - rval.emplace_back(s); - - delete [] script_signature_files; - return rval; - } - -static std::string get_exe_path(const std::string& invocation) - { - if ( invocation.empty() ) - return ""; - - if ( invocation[0] == '/' || invocation[0] == '~' ) - // Absolute path - return invocation; - - if ( invocation.find('/') != std::string::npos ) - { - // Relative path - char cwd[PATH_MAX]; - - if ( ! getcwd(cwd, sizeof(cwd)) ) - { - fprintf(stderr, "failed to get current directory: %s\n", - strerror(errno)); - exit(1); - } - - return std::string(cwd) + "/" + invocation; - } - - auto path = getenv("PATH"); - - if ( ! path ) - return ""; - - return find_file(invocation, path); - } +#include "supervisor/Supervisor.h" +#include "Net.h" int main(int argc, char** argv) { - ZEEK_LSAN_DISABLE(); - std::set_new_handler(bro_new_handler); + auto time_start = current_time(true); + auto setup_result = zeek::setup(argc, argv); - auto zeek_exe_path = get_exe_path(argv[0]); + if ( setup_result.code ) + return setup_result.code; - if ( zeek_exe_path.empty() ) - { - fprintf(stderr, "failed to get path to executable '%s'", argv[0]); - exit(1); - } + auto& options = setup_result.options; + auto do_net_run = iosource_mgr->Size() > 0 || + have_pending_timers || + BifConst::exit_only_after_terminate; - bro_argc = argc; - bro_argv = new char* [argc]; - - for ( int i = 0; i < argc; i++ ) - bro_argv[i] = copy_string(argv[i]); - - auto options = zeek::parse_cmdline(argc, argv); - - if ( options.print_usage ) - zeek::usage(argv[0], 0); - - if ( options.print_version ) - { - fprintf(stdout, "%s version %s\n", argv[0], zeek_version()); - exit(0); - } - - if ( options.run_unit_tests ) - { - doctest::Context context; - auto dargs = to_cargs(options.doctest_args); - context.applyCommandLine(dargs.size(), dargs.data()); - ZEEK_LSAN_ENABLE(); - return context.run(); - } - - auto stem_state = zeek::Supervisor::CreateStem(options.supervisor_mode); - - if ( zeek::Supervisor::ThisNode() ) - zeek::Supervisor::ThisNode()->Init(&options); - - double time_start = current_time(true); - - brofiler.ReadStats(); - - auto dns_type = options.dns_mode; - - if ( dns_type == DNS_DEFAULT && zeek::fake_dns() ) - dns_type = DNS_FAKE; - - RETSIGTYPE (*oldhandler)(int); - - zeek_script_prefixes = options.script_prefixes; - auto zeek_prefixes = zeekenv("ZEEK_PREFIXES"); - - if ( zeek_prefixes ) - tokenize_string(zeek_prefixes, ":", &zeek_script_prefixes); - - pseudo_realtime = options.pseudo_realtime; - -#ifdef USE_PERFTOOLS_DEBUG - perftools_leaks = options.perftools_check_leaks; - perftools_profile = options.perftools_profile; -#endif - - if ( options.debug_scripts ) - { - g_policy_debug = options.debug_scripts; - fprintf(stderr, "Zeek script debugging ON.\n"); - } - - if ( options.script_code_to_exec ) - command_line_policy = options.script_code_to_exec->data(); - - if ( options.debug_script_tracing_file ) - { - g_trace_state.SetTraceFile(options.debug_script_tracing_file->data()); - g_trace_state.TraceOn(); - } - - if ( options.process_status_file ) - proc_status_file = options.process_status_file->data(); - - atexit(atexit_handler); - set_processing_status("INITIALIZING", "main"); - - bro_start_time = current_time(true); - - val_mgr = new ValManager(); - reporter = new Reporter(); - thread_mgr = new threading::Manager(); - plugin_mgr = new plugin::Manager(); - -#ifdef DEBUG - if ( options.debug_log_streams ) - { - debug_logger.EnableStreams(options.debug_log_streams->data()); - - if ( getenv("ZEEK_DEBUG_LOG_STDERR") ) - debug_logger.OpenDebugLog(nullptr); - else - debug_logger.OpenDebugLog("debug"); - } -#endif - - if ( options.supervisor_mode ) - { - zeek::Supervisor::Config cfg = {}; - cfg.zeek_exe_path = zeek_exe_path; - options.filter_supervisor_options(); - zeek::supervisor_mgr = new zeek::Supervisor(std::move(cfg), - std::move(*stem_state)); - } - - const char* seed_load_file = zeekenv("ZEEK_SEED_FILE"); - - if ( options.random_seed_input_file ) - seed_load_file = options.random_seed_input_file->data(); - - init_random_seed((seed_load_file && *seed_load_file ? seed_load_file : nullptr), - options.random_seed_output_file ? options.random_seed_output_file->data() : nullptr); - // DEBUG_MSG("HMAC key: %s\n", md5_digest_print(shared_hmac_md5_key)); - init_hash_function(); - - ERR_load_crypto_strings(); - OPENSSL_add_all_algorithms_conf(); - SSL_library_init(); - SSL_load_error_strings(); - - // FIXME: On systems that don't provide /dev/urandom, OpenSSL doesn't - // seed the PRNG. We should do this here (but at least Linux, FreeBSD - // and Solaris provide /dev/urandom). - - int r = sqlite3_initialize(); - - if ( r != SQLITE_OK ) - reporter->Error("Failed to initialize sqlite3: %s", sqlite3_errstr(r)); - -#ifdef USE_IDMEF - char* libidmef_dtd_path_cstr = new char[options.libidmef_dtd_file.size() + 1]; - safe_strncpy(libidmef_dtd_path_cstr, options.libidmef_dtd_file.data(), - options.libidmef_dtd_file.size()); - globalsInit(libidmef_dtd_path_cstr); // Init LIBIDMEF globals - createCurrentDoc("1.0"); // Set a global XML document -#endif - - timer_mgr = new PQ_TimerMgr(); - - auto zeekygen_cfg = options.zeekygen_config_file.value_or(""); - zeekygen_mgr = new zeekygen::Manager(zeekygen_cfg, bro_argv[0]); - - add_essential_input_file("base/init-bare.zeek"); - add_essential_input_file("base/init-frameworks-and-bifs.zeek"); - - if ( ! options.bare_mode ) - add_input_file("base/init-default.zeek"); - - plugin_mgr->SearchDynamicPlugins(bro_plugin_path()); - - if ( options.plugins_to_load.empty() && options.scripts_to_load.empty() && - options.script_options_to_set.empty() && - ! options.pcap_file && ! options.interface && - ! options.identifier_to_print && - ! command_line_policy && ! options.print_plugins && - ! options.supervisor_mode && ! zeek::Supervisor::ThisNode() ) - add_input_file("-"); - - for ( const auto& script_option : options.script_options_to_set ) - params.push_back(script_option); - - for ( const auto& plugin : options.plugins_to_load ) - requested_plugins.insert(plugin); - - for ( const auto& script : options.scripts_to_load ) - add_input_file(script.data()); - - push_scope(nullptr, nullptr); - - dns_mgr = new DNS_Mgr(dns_type); - - // It would nice if this were configurable. This is similar to the - // chicken and the egg problem. It would be configurable by parsing - // policy, but we can't parse policy without DNS resolution. - dns_mgr->SetDir(".state"); - - iosource_mgr = new iosource::Manager(); - event_registry = new EventRegistry(); - analyzer_mgr = new analyzer::Manager(); - log_mgr = new logging::Manager(); - input_mgr = new input::Manager(); - file_mgr = new file_analysis::Manager(); - broker_mgr = new bro_broker::Manager(options.pcap_file.has_value()); - trigger_mgr = new trigger::Manager(); - - plugin_mgr->InitPreScript(); - analyzer_mgr->InitPreScript(); - file_mgr->InitPreScript(); - zeekygen_mgr->InitPreScript(); - - bool missing_plugin = false; - - for ( set::const_iterator i = requested_plugins.begin(); - i != requested_plugins.end(); i++ ) - { - if ( ! plugin_mgr->ActivateDynamicPlugin(*i) ) - missing_plugin = true; - } - - if ( missing_plugin ) - reporter->FatalError("Failed to activate requested dynamic plugin(s)."); - - plugin_mgr->ActivateDynamicPlugins(! options.bare_mode); - - init_event_handlers(); - - md5_type = new OpaqueType("md5"); - sha1_type = new OpaqueType("sha1"); - sha256_type = new OpaqueType("sha256"); - entropy_type = new OpaqueType("entropy"); - cardinality_type = new OpaqueType("cardinality"); - topk_type = new OpaqueType("topk"); - bloomfilter_type = new OpaqueType("bloomfilter"); - x509_opaque_type = new OpaqueType("x509"); - ocsp_resp_opaque_type = new OpaqueType("ocsp_resp"); - paraglob_type = new OpaqueType("paraglob"); - - // The leak-checker tends to produce some false - // positives (memory which had already been - // allocated before we start the checking is - // nevertheless reported; see perftools docs), thus - // we suppress some messages here. - -#ifdef USE_PERFTOOLS_DEBUG - { - HeapLeakChecker::Disabler disabler; -#endif - - is_parsing = true; - yyparse(); - is_parsing = false; - - RecordVal::DoneParsing(); - TableVal::DoneParsing(); - - init_general_global_var(); - init_net_var(); - init_builtin_funcs_subdirs(); - - // Must come after plugin activation (and also after hash - // initialization). - binpac::FlowBuffer::Policy flowbuffer_policy; - flowbuffer_policy.max_capacity = global_scope()->Lookup( - "BinPAC::flowbuffer_capacity_max")->ID_Val()->AsCount(); - flowbuffer_policy.min_capacity = global_scope()->Lookup( - "BinPAC::flowbuffer_capacity_min")->ID_Val()->AsCount(); - flowbuffer_policy.contract_threshold = global_scope()->Lookup( - "BinPAC::flowbuffer_contract_threshold")->ID_Val()->AsCount(); - binpac::init(&flowbuffer_policy); - - plugin_mgr->InitBifs(); - - if ( reporter->Errors() > 0 ) - exit(1); - - iosource_mgr->InitPostScript(); - plugin_mgr->InitPostScript(); - zeekygen_mgr->InitPostScript(); - broker_mgr->InitPostScript(); - timer_mgr->InitPostScript(); - mgr.InitPostScript(); - - if ( zeek::supervisor_mgr ) - zeek::supervisor_mgr->InitPostScript(); - - if ( options.print_plugins ) - { - bool success = show_plugins(options.print_plugins); - exit(success ? 0 : 1); - } - - analyzer_mgr->InitPostScript(); - file_mgr->InitPostScript(); - dns_mgr->InitPostScript(); - - if ( options.parse_only ) - { - int rc = (reporter->Errors() > 0 ? 1 : 0); - exit(rc); - } - -#ifdef USE_PERFTOOLS_DEBUG - } -#endif - - if ( reporter->Errors() > 0 ) - { - delete dns_mgr; - exit(1); - } - - reporter->InitOptions(); - zeekygen_mgr->GenerateDocs(); - - if ( options.pcap_filter ) - { - ID* id = global_scope()->Lookup("cmd_line_bpf_filter"); - - if ( ! id ) - reporter->InternalError("global cmd_line_bpf_filter not defined"); - - id->SetVal(make_intrusive(*options.pcap_filter)); - } - - auto all_signature_files = options.signature_files; - - // Append signature files defined in "signature_files" script option - for ( auto&& sf : get_script_signature_files() ) - all_signature_files.emplace_back(std::move(sf)); - - // Append signature files defined in @load-sigs - for ( const auto& sf : sig_files ) - all_signature_files.emplace_back(sf); - - if ( ! all_signature_files.empty() ) - { - rule_matcher = new RuleMatcher(options.signature_re_level); - if ( ! rule_matcher->ReadFiles(all_signature_files) ) - { - delete dns_mgr; - exit(1); - } - - if ( options.print_signature_debug_info ) - rule_matcher->PrintDebug(); - - file_mgr->InitMagic(); - } - - if ( g_policy_debug ) - // ### Add support for debug command file. - dbg_init_debugger(nullptr); - - if ( ! options.pcap_file && ! options.interface ) - { - Val* interfaces_val = internal_val("interfaces"); - if ( interfaces_val ) - { - char* interfaces_str = - interfaces_val->AsString()->Render(); - - if ( interfaces_str[0] != '\0' ) - options.interface = interfaces_str; - - delete [] interfaces_str; - } - } - - if ( dns_type != DNS_PRIME ) - net_init(options.interface, options.pcap_file, options.pcap_output_file, options.use_watchdog); - - net_done = internal_handler("net_done"); - - if ( ! g_policy_debug ) - { - (void) setsignal(SIGTERM, sig_handler); - (void) setsignal(SIGINT, sig_handler); - (void) setsignal(SIGPIPE, SIG_IGN); - } - - // Cooperate with nohup(1). - if ( (oldhandler = setsignal(SIGHUP, sig_handler)) != SIG_DFL ) - (void) setsignal(SIGHUP, oldhandler); - - if ( dns_type == DNS_PRIME ) - { - dns_mgr->Verify(); - dns_mgr->Resolve(); - - if ( ! dns_mgr->Save() ) - reporter->FatalError("can't update DNS cache"); - - mgr.Drain(); - delete dns_mgr; - exit(0); - } - - // Print the ID. - if ( options.identifier_to_print ) - { - ID* id = global_scope()->Lookup(*options.identifier_to_print); - if ( ! id ) - reporter->FatalError("No such ID: %s\n", options.identifier_to_print->data()); - - ODesc desc; - desc.SetQuotes(true); - desc.SetIncludeStats(true); - id->DescribeExtended(&desc); - - fprintf(stdout, "%s\n", desc.Description()); - exit(0); - } - - if ( profiling_interval > 0 ) - { - profiling_logger = new ProfileLogger(profiling_file->AsFile(), - profiling_interval); - - if ( segment_profiling ) - segment_logger = profiling_logger; - } - - if ( ! reading_live && ! reading_traces ) - // Set up network_time to track real-time, since - // we don't have any other source for it. - net_update_time(current_time()); - - EventHandlerPtr zeek_init = internal_handler("zeek_init"); - if ( zeek_init ) //### this should be a function - mgr.Enqueue(zeek_init, zeek::Args{}); - - EventRegistry::string_list dead_handlers = - event_registry->UnusedHandlers(); - - if ( ! dead_handlers.empty() && check_for_unused_event_handlers ) - { - for ( const string& handler : dead_handlers ) - reporter->Warning("event handler never invoked: %s", handler.c_str()); - } - - // Enable LeakSanitizer before zeek_init() and even before executing - // top-level statements. Even though it's not bad if a leak happens only - // once at initialization, we have to assume that script-layer code causing - // such a leak can be placed in any arbitrary event handler and potentially - // cause more severe problems. - ZEEK_LSAN_ENABLE(); - - if ( stmts ) - { - stmt_flow_type flow; - Frame f(current_scope()->Length(), nullptr, nullptr); - g_frame_stack.push_back(&f); - - try - { - stmts->Exec(&f, flow); - } - catch ( InterpreterException& ) - { - reporter->FatalError("failed to execute script statements at top-level scope"); - } - - g_frame_stack.pop_back(); - } - - if ( options.ignore_checksums ) - ignore_checksums = 1; - - if ( zeek_script_loaded ) - { - // Queue events reporting loaded scripts. - for ( std::list::iterator i = files_scanned.begin(); i != files_scanned.end(); i++ ) - { - if ( i->skipped ) - continue; - - mgr.Enqueue(zeek_script_loaded, - make_intrusive(i->name.c_str()), - IntrusivePtr{AdoptRef{}, val_mgr->GetCount(i->include_level)} - ); - } - } - - reporter->ReportViaEvents(true); - - // Drain the event queue here to support the protocols framework configuring DPM - mgr.Drain(); - - if ( reporter->Errors() > 0 && ! zeekenv("ZEEK_ALLOW_INIT_ERRORS") ) - reporter->FatalError("errors occurred while initializing"); - - broker_mgr->ZeekInitDone(); - reporter->ZeekInitDone(); - analyzer_mgr->DumpDebug(); - - have_pending_timers = ! reading_traces && timer_mgr->Size() > 0; - - if ( iosource_mgr->Size() > 0 || - have_pending_timers || - BifConst::exit_only_after_terminate ) + if ( do_net_run ) { if ( profiling_logger ) profiling_logger->Log(); @@ -952,24 +76,7 @@ int main(int argc, char** argv) (mem_net_done_total - mem_net_start_total) / 1024 / 1024, (mem_net_done_malloced - mem_net_start_malloced) / 1024 / 1024); } - - done_with_network(); - net_delete(); } - terminate_bro(); - - sqlite3_shutdown(); - - ERR_free_strings(); - EVP_cleanup(); - CRYPTO_cleanup_all_ex_data(); - - // Close files after net_delete(), because net_delete() - // might write to connection content files. - BroFile::CloseOpenFiles(); - - delete rule_matcher; - - return 0; + return zeek::cleanup(do_net_run); } diff --git a/src/util.cc b/src/util.cc index b6293307bf..1293c34804 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1094,7 +1094,8 @@ void bro_srandom(unsigned int seed) srandom(seed); } -void init_random_seed(const char* read_file, const char* write_file) +void init_random_seed(const char* read_file, const char* write_file, + bool use_empty_seeds) { static const int bufsiz = 20; uint32_t buf[bufsiz]; @@ -1111,6 +1112,8 @@ void init_random_seed(const char* read_file, const char* write_file) else seeds_done = true; } + else if ( use_empty_seeds ) + seeds_done = true; #ifdef HAVE_GETRANDOM if ( ! seeds_done ) diff --git a/src/util.h b/src/util.h index 66d65ba622..1353a36cdd 100644 --- a/src/util.h +++ b/src/util.h @@ -208,13 +208,13 @@ extern uint8_t shared_siphash_key[SIPHASH_KEYLEN]; extern void hmac_md5(size_t size, const unsigned char* bytes, unsigned char digest[16]); -// Initializes RNGs for bro_random() and MD5 usage. If seed is given, then -// it is used (to provide determinism). If load_file is given, the seeds -// (both random & MD5) are loaded from that file. This takes precedence -// over the "seed" argument. If write_file is given, the seeds are written -// to that file. -// -extern void init_random_seed(const char* load_file, const char* write_file); +// Initializes RNGs for bro_random() and MD5 usage. If load_file is given, +// the seeds (both random & MD5) are loaded from that file. This takes +// precedence over the "use_empty_seeds" argument, which just +// zero-initializes all seed values. If write_file is given, the seeds are +// written to that file. +extern void init_random_seed(const char* load_file, const char* write_file, + bool use_empty_seeds); // Retrieves the initial seed computed after the very first call to // init_random_seed(). Repeated calls to init_random_seed() will not affect diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc new file mode 100644 index 0000000000..8cc392e4fb --- /dev/null +++ b/src/zeek-setup.cc @@ -0,0 +1,922 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek-config.h" +#include "zeek-setup.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef USE_IDMEF +extern "C" { +#include +} +#endif + +#include +#include + +#include "Options.h" +#include "input.h" +#include "DNS_Mgr.h" +#include "Frame.h" +#include "Scope.h" +#include "Event.h" +#include "File.h" +#include "Reporter.h" +#include "Net.h" +#include "NetVar.h" +#include "Var.h" +#include "Timer.h" +#include "Stmt.h" +#include "Desc.h" +#include "Debug.h" +#include "DFA.h" +#include "RuleMatcher.h" +#include "Anon.h" +#include "EventRegistry.h" +#include "Stats.h" +#include "Brofiler.h" +#include "Traverse.h" +#include "Trigger.h" + +#include "supervisor/Supervisor.h" +#include "threading/Manager.h" +#include "input/Manager.h" +#include "logging/Manager.h" +#include "input/readers/raw/Raw.h" +#include "analyzer/Manager.h" +#include "analyzer/Tag.h" +#include "plugin/Manager.h" +#include "file_analysis/Manager.h" +#include "zeekygen/Manager.h" +#include "iosource/Manager.h" +#include "broker/Manager.h" + +#include "binpac_bro.h" + +#include "3rdparty/sqlite3.h" + +#define DOCTEST_CONFIG_IMPLEMENT +#include "3rdparty/doctest.h" + +Brofiler brofiler; + +#ifndef HAVE_STRSEP +extern "C" { +char* strsep(char**, const char*); +}; +#endif + +extern "C" { +#include "setsignal.h" +}; + +#ifdef USE_PERFTOOLS_DEBUG +HeapLeakChecker* heap_checker = 0; +int perftools_leaks = 0; +int perftools_profile = 0; +#endif + +DNS_Mgr* dns_mgr; +TimerMgr* timer_mgr; +ValManager* val_mgr = nullptr; +logging::Manager* log_mgr = nullptr; +threading::Manager* thread_mgr = nullptr; +input::Manager* input_mgr = nullptr; +plugin::Manager* plugin_mgr = nullptr; +analyzer::Manager* analyzer_mgr = nullptr; +file_analysis::Manager* file_mgr = nullptr; +zeekygen::Manager* zeekygen_mgr = nullptr; +iosource::Manager* iosource_mgr = nullptr; +bro_broker::Manager* broker_mgr = nullptr; +zeek::Supervisor* zeek::supervisor_mgr = nullptr; +trigger::Manager* trigger_mgr = nullptr; + +std::vector zeek_script_prefixes; +Stmt* stmts; +EventHandlerPtr net_done = nullptr; +RuleMatcher* rule_matcher = nullptr; +EventRegistry* event_registry = nullptr; +ProfileLogger* profiling_logger = nullptr; +ProfileLogger* segment_logger = nullptr; +SampleLogger* sample_logger = nullptr; +int signal_val = 0; +extern char version[]; +const char* command_line_policy = nullptr; +vector params; +set requested_plugins; +const char* proc_status_file = nullptr; + +OpaqueType* md5_type = nullptr; +OpaqueType* sha1_type = nullptr; +OpaqueType* sha256_type = nullptr; +OpaqueType* entropy_type = nullptr; +OpaqueType* cardinality_type = nullptr; +OpaqueType* topk_type = nullptr; +OpaqueType* bloomfilter_type = nullptr; +OpaqueType* x509_opaque_type = nullptr; +OpaqueType* ocsp_resp_opaque_type = nullptr; +OpaqueType* paraglob_type = nullptr; + +// Keep copy of command line +int bro_argc; +char** bro_argv; + +const char* zeek_version() + { +#ifdef DEBUG + static char* debug_version = nullptr; + + if ( ! debug_version ) + { + int n = strlen(version) + sizeof("-debug") + 1; + debug_version = new char[n]; + snprintf(debug_version, n, "%s%s", version, "-debug"); + } + + return debug_version; +#else + return version; +#endif + } + +static std::vector to_cargs(const std::vector& args) + { + std::vector rval; + rval.reserve(args.size()); + + for ( const auto& arg : args ) + rval.emplace_back(arg.data()); + + return rval; + } + +bool show_plugins(int level) + { + plugin::Manager::plugin_list plugins = plugin_mgr->ActivePlugins(); + + if ( ! plugins.size() ) + { + printf("No plugins registered, not even any built-ins. This is probably a bug.\n"); + return false; + } + + ODesc d; + + if ( level == 1 ) + d.SetShort(); + + int count = 0; + + for ( plugin::Manager::plugin_list::const_iterator i = plugins.begin(); i != plugins.end(); i++ ) + { + if ( requested_plugins.size() + && requested_plugins.find((*i)->Name()) == requested_plugins.end() ) + continue; + + (*i)->Describe(&d); + + if ( ! d.IsShort() ) + d.Add("\n"); + + ++count; + } + + printf("%s", d.Description()); + + plugin::Manager::inactive_plugin_list inactives = plugin_mgr->InactivePlugins(); + + if ( inactives.size() && ! requested_plugins.size() ) + { + printf("\nInactive dynamic plugins:\n"); + + for ( plugin::Manager::inactive_plugin_list::const_iterator i = inactives.begin(); i != inactives.end(); i++ ) + { + string name = (*i).first; + string path = (*i).second; + printf(" %s (%s)\n", name.c_str(), path.c_str()); + } + } + + return count != 0; + } + +void done_with_network() + { + set_processing_status("TERMINATING", "done_with_network"); + + // Cancel any pending alarms (watchdog, in particular). + (void) alarm(0); + + if ( net_done ) + { + mgr.Drain(); + // Don't propagate this event to remote clients. + mgr.Dispatch(new Event(net_done, + {make_intrusive(timer_mgr->Time(), TYPE_TIME)}), + true); + } + + if ( profiling_logger ) + profiling_logger->Log(); + + terminating = true; + + analyzer_mgr->Done(); + timer_mgr->Expire(); + dns_mgr->Flush(); + mgr.Drain(); + mgr.Drain(); + + net_finish(1); + +#ifdef USE_PERFTOOLS_DEBUG + + if ( perftools_profile ) + { + HeapProfilerDump("post net_run"); + HeapProfilerStop(); + } + + if ( heap_checker && ! heap_checker->NoLeaks() ) + { + fprintf(stderr, "Memory leaks - aborting.\n"); + abort(); + } +#endif + + ZEEK_LSAN_DISABLE(); + } + +void terminate_bro() + { + set_processing_status("TERMINATING", "terminate_bro"); + + terminating = true; + + iosource_mgr->Wakeup("terminate_bro"); + + // File analysis termination may produce events, so do it early on in + // the termination process. + file_mgr->Terminate(); + + brofiler.WriteStats(); + + EventHandlerPtr zeek_done = internal_handler("zeek_done"); + if ( zeek_done ) + mgr.Enqueue(zeek_done, zeek::Args{}); + + timer_mgr->Expire(); + mgr.Drain(); + + if ( profiling_logger ) + { + // FIXME: There are some occasional crashes in the memory + // allocation code when killing Bro. Disabling this for now. + if ( ! (signal_val == SIGTERM || signal_val == SIGINT) ) + profiling_logger->Log(); + + delete profiling_logger; + } + + mgr.Drain(); + + notifier::registry.Terminate(); + log_mgr->Terminate(); + input_mgr->Terminate(); + thread_mgr->Terminate(); + broker_mgr->Terminate(); + dns_mgr->Terminate(); + + mgr.Drain(); + + plugin_mgr->FinishPlugins(); + + delete zeekygen_mgr; + delete analyzer_mgr; + delete file_mgr; + // broker_mgr, timer_mgr, and supervisor are deleted via iosource_mgr + delete iosource_mgr; + delete event_registry; + delete log_mgr; + delete reporter; + delete plugin_mgr; + delete val_mgr; + + // free the global scope + pop_scope(); + + reporter = nullptr; + } + +void zeek_terminate_loop(const char* reason) + { + set_processing_status("TERMINATING", reason); + reporter->Info("%s", reason); + + net_get_final_stats(); + done_with_network(); + net_delete(); + + terminate_bro(); + + // Close files after net_delete(), because net_delete() + // might write to connection content files. + BroFile::CloseOpenFiles(); + + delete rule_matcher; + + exit(0); + } + +RETSIGTYPE sig_handler(int signo) + { + set_processing_status("TERMINATING", "sig_handler"); + signal_val = signo; + + if ( ! terminating ) + iosource_mgr->Wakeup("sig_handler"); + + return RETSIGVAL; + } + +static void atexit_handler() + { + set_processing_status("TERMINATED", "atexit"); + } + +static void bro_new_handler() + { + out_of_memory("new"); + } + +static std::vector get_script_signature_files() + { + std::vector rval; + + // Parse rule files defined on the script level. + char* script_signature_files = + copy_string(internal_val("signature_files")->AsString()->CheckString()); + + char* tmp = script_signature_files; + char* s; + while ( (s = strsep(&tmp, " \t")) ) + if ( *s ) + rval.emplace_back(s); + + delete [] script_signature_files; + return rval; + } + +static std::string get_exe_path(const std::string& invocation) + { + if ( invocation.empty() ) + return ""; + + if ( invocation[0] == '/' || invocation[0] == '~' ) + // Absolute path + return invocation; + + if ( invocation.find('/') != std::string::npos ) + { + // Relative path + char cwd[PATH_MAX]; + + if ( ! getcwd(cwd, sizeof(cwd)) ) + { + fprintf(stderr, "failed to get current directory: %s\n", + strerror(errno)); + exit(1); + } + + return std::string(cwd) + "/" + invocation; + } + + auto path = getenv("PATH"); + + if ( ! path ) + return ""; + + return find_file(invocation, path); + } + +zeek::SetupResult zeek::setup(int argc, char** argv, zeek::Options* zopts) + { + ZEEK_LSAN_DISABLE(); + std::set_new_handler(bro_new_handler); + + auto zeek_exe_path = get_exe_path(argv[0]); + + if ( zeek_exe_path.empty() ) + { + fprintf(stderr, "failed to get path to executable '%s'", argv[0]); + exit(1); + } + + bro_argc = argc; + bro_argv = new char* [argc]; + + for ( int i = 0; i < argc; i++ ) + bro_argv[i] = copy_string(argv[i]); + + auto options = zopts ? *zopts : zeek::parse_cmdline(argc, argv); + + if ( options.print_usage ) + zeek::usage(argv[0], 0); + + if ( options.print_version ) + { + fprintf(stdout, "%s version %s\n", argv[0], zeek_version()); + exit(0); + } + + if ( options.run_unit_tests ) + { + doctest::Context context; + auto dargs = to_cargs(options.doctest_args); + context.applyCommandLine(dargs.size(), dargs.data()); + ZEEK_LSAN_ENABLE(); + return {context.run(), std::move(options)}; + } + + auto stem_state = zeek::Supervisor::CreateStem(options.supervisor_mode); + + if ( zeek::Supervisor::ThisNode() ) + zeek::Supervisor::ThisNode()->Init(&options); + + brofiler.ReadStats(); + + auto dns_type = options.dns_mode; + + if ( dns_type == DNS_DEFAULT && zeek::fake_dns() ) + dns_type = DNS_FAKE; + + RETSIGTYPE (*oldhandler)(int); + + zeek_script_prefixes = options.script_prefixes; + auto zeek_prefixes = zeekenv("ZEEK_PREFIXES"); + + if ( zeek_prefixes ) + tokenize_string(zeek_prefixes, ":", &zeek_script_prefixes); + + pseudo_realtime = options.pseudo_realtime; + +#ifdef USE_PERFTOOLS_DEBUG + perftools_leaks = options.perftools_check_leaks; + perftools_profile = options.perftools_profile; +#endif + + if ( options.debug_scripts ) + { + g_policy_debug = options.debug_scripts; + fprintf(stderr, "Zeek script debugging ON.\n"); + } + + if ( options.script_code_to_exec ) + command_line_policy = options.script_code_to_exec->data(); + + if ( options.debug_script_tracing_file ) + { + g_trace_state.SetTraceFile(options.debug_script_tracing_file->data()); + g_trace_state.TraceOn(); + } + + if ( options.process_status_file ) + proc_status_file = options.process_status_file->data(); + + atexit(atexit_handler); + set_processing_status("INITIALIZING", "main"); + + bro_start_time = current_time(true); + + val_mgr = new ValManager(); + reporter = new Reporter(options.abort_on_scripting_errors); + thread_mgr = new threading::Manager(); + plugin_mgr = new plugin::Manager(); + +#ifdef DEBUG + if ( options.debug_log_streams ) + { + debug_logger.EnableStreams(options.debug_log_streams->data()); + + if ( getenv("ZEEK_DEBUG_LOG_STDERR") ) + debug_logger.OpenDebugLog(nullptr); + else + debug_logger.OpenDebugLog("debug"); + } +#endif + + if ( options.supervisor_mode ) + { + zeek::Supervisor::Config cfg = {}; + cfg.zeek_exe_path = zeek_exe_path; + options.filter_supervisor_options(); + zeek::supervisor_mgr = new zeek::Supervisor(std::move(cfg), + std::move(*stem_state)); + } + + const char* seed_load_file = zeekenv("ZEEK_SEED_FILE"); + + if ( options.random_seed_input_file ) + seed_load_file = options.random_seed_input_file->data(); + + init_random_seed((seed_load_file && *seed_load_file ? seed_load_file : nullptr), + options.random_seed_output_file ? options.random_seed_output_file->data() : nullptr, + options.deterministic_mode); + // DEBUG_MSG("HMAC key: %s\n", md5_digest_print(shared_hmac_md5_key)); + init_hash_function(); + + ERR_load_crypto_strings(); + OPENSSL_add_all_algorithms_conf(); + SSL_library_init(); + SSL_load_error_strings(); + + // FIXME: On systems that don't provide /dev/urandom, OpenSSL doesn't + // seed the PRNG. We should do this here (but at least Linux, FreeBSD + // and Solaris provide /dev/urandom). + + int r = sqlite3_initialize(); + + if ( r != SQLITE_OK ) + reporter->Error("Failed to initialize sqlite3: %s", sqlite3_errstr(r)); + +#ifdef USE_IDMEF + char* libidmef_dtd_path_cstr = new char[options.libidmef_dtd_file.size() + 1]; + safe_strncpy(libidmef_dtd_path_cstr, options.libidmef_dtd_file.data(), + options.libidmef_dtd_file.size()); + globalsInit(libidmef_dtd_path_cstr); // Init LIBIDMEF globals + createCurrentDoc("1.0"); // Set a global XML document +#endif + + timer_mgr = new PQ_TimerMgr(); + + auto zeekygen_cfg = options.zeekygen_config_file.value_or(""); + zeekygen_mgr = new zeekygen::Manager(zeekygen_cfg, bro_argv[0]); + + add_essential_input_file("base/init-bare.zeek"); + add_essential_input_file("base/init-frameworks-and-bifs.zeek"); + + if ( ! options.bare_mode ) + add_input_file("base/init-default.zeek"); + + plugin_mgr->SearchDynamicPlugins(bro_plugin_path()); + + if ( options.plugins_to_load.empty() && options.scripts_to_load.empty() && + options.script_options_to_set.empty() && + ! options.pcap_file && ! options.interface && + ! options.identifier_to_print && + ! command_line_policy && ! options.print_plugins && + ! options.supervisor_mode && ! zeek::Supervisor::ThisNode() ) + add_input_file("-"); + + for ( const auto& script_option : options.script_options_to_set ) + params.push_back(script_option); + + for ( const auto& plugin : options.plugins_to_load ) + requested_plugins.insert(plugin); + + for ( const auto& script : options.scripts_to_load ) + add_input_file(script.data()); + + push_scope(nullptr, nullptr); + + dns_mgr = new DNS_Mgr(dns_type); + + // It would nice if this were configurable. This is similar to the + // chicken and the egg problem. It would be configurable by parsing + // policy, but we can't parse policy without DNS resolution. + dns_mgr->SetDir(".state"); + + iosource_mgr = new iosource::Manager(); + event_registry = new EventRegistry(); + analyzer_mgr = new analyzer::Manager(); + log_mgr = new logging::Manager(); + input_mgr = new input::Manager(); + file_mgr = new file_analysis::Manager(); + auto broker_real_time = ! options.pcap_file && ! options.deterministic_mode; + broker_mgr = new bro_broker::Manager(broker_real_time); + trigger_mgr = new trigger::Manager(); + + plugin_mgr->InitPreScript(); + analyzer_mgr->InitPreScript(); + file_mgr->InitPreScript(); + zeekygen_mgr->InitPreScript(); + + bool missing_plugin = false; + + for ( set::const_iterator i = requested_plugins.begin(); + i != requested_plugins.end(); i++ ) + { + if ( ! plugin_mgr->ActivateDynamicPlugin(*i) ) + missing_plugin = true; + } + + if ( missing_plugin ) + reporter->FatalError("Failed to activate requested dynamic plugin(s)."); + + plugin_mgr->ActivateDynamicPlugins(! options.bare_mode); + + init_event_handlers(); + + md5_type = new OpaqueType("md5"); + sha1_type = new OpaqueType("sha1"); + sha256_type = new OpaqueType("sha256"); + entropy_type = new OpaqueType("entropy"); + cardinality_type = new OpaqueType("cardinality"); + topk_type = new OpaqueType("topk"); + bloomfilter_type = new OpaqueType("bloomfilter"); + x509_opaque_type = new OpaqueType("x509"); + ocsp_resp_opaque_type = new OpaqueType("ocsp_resp"); + paraglob_type = new OpaqueType("paraglob"); + + // The leak-checker tends to produce some false + // positives (memory which had already been + // allocated before we start the checking is + // nevertheless reported; see perftools docs), thus + // we suppress some messages here. + +#ifdef USE_PERFTOOLS_DEBUG + { + HeapLeakChecker::Disabler disabler; +#endif + + is_parsing = true; + yyparse(); + is_parsing = false; + + RecordVal::DoneParsing(); + TableVal::DoneParsing(); + + init_general_global_var(); + init_net_var(); + init_builtin_funcs_subdirs(); + + // Must come after plugin activation (and also after hash + // initialization). + binpac::FlowBuffer::Policy flowbuffer_policy; + flowbuffer_policy.max_capacity = global_scope()->Lookup( + "BinPAC::flowbuffer_capacity_max")->ID_Val()->AsCount(); + flowbuffer_policy.min_capacity = global_scope()->Lookup( + "BinPAC::flowbuffer_capacity_min")->ID_Val()->AsCount(); + flowbuffer_policy.contract_threshold = global_scope()->Lookup( + "BinPAC::flowbuffer_contract_threshold")->ID_Val()->AsCount(); + binpac::init(&flowbuffer_policy); + + plugin_mgr->InitBifs(); + + if ( reporter->Errors() > 0 ) + exit(1); + + iosource_mgr->InitPostScript(); + plugin_mgr->InitPostScript(); + zeekygen_mgr->InitPostScript(); + broker_mgr->InitPostScript(); + timer_mgr->InitPostScript(); + mgr.InitPostScript(); + + if ( zeek::supervisor_mgr ) + zeek::supervisor_mgr->InitPostScript(); + + if ( options.print_plugins ) + { + bool success = show_plugins(options.print_plugins); + exit(success ? 0 : 1); + } + + analyzer_mgr->InitPostScript(); + file_mgr->InitPostScript(); + dns_mgr->InitPostScript(); + + if ( options.parse_only ) + { + int rc = (reporter->Errors() > 0 ? 1 : 0); + exit(rc); + } + +#ifdef USE_PERFTOOLS_DEBUG + } +#endif + + if ( reporter->Errors() > 0 ) + { + delete dns_mgr; + exit(1); + } + + reporter->InitOptions(); + zeekygen_mgr->GenerateDocs(); + + if ( options.pcap_filter ) + { + ID* id = global_scope()->Lookup("cmd_line_bpf_filter"); + + if ( ! id ) + reporter->InternalError("global cmd_line_bpf_filter not defined"); + + id->SetVal(make_intrusive(*options.pcap_filter)); + } + + auto all_signature_files = options.signature_files; + + // Append signature files defined in "signature_files" script option + for ( auto&& sf : get_script_signature_files() ) + all_signature_files.emplace_back(std::move(sf)); + + // Append signature files defined in @load-sigs + for ( const auto& sf : sig_files ) + all_signature_files.emplace_back(sf); + + if ( ! all_signature_files.empty() ) + { + rule_matcher = new RuleMatcher(options.signature_re_level); + if ( ! rule_matcher->ReadFiles(all_signature_files) ) + { + delete dns_mgr; + exit(1); + } + + if ( options.print_signature_debug_info ) + rule_matcher->PrintDebug(); + + file_mgr->InitMagic(); + } + + if ( g_policy_debug ) + // ### Add support for debug command file. + dbg_init_debugger(nullptr); + + if ( ! options.pcap_file && ! options.interface ) + { + Val* interfaces_val = internal_val("interfaces"); + if ( interfaces_val ) + { + char* interfaces_str = + interfaces_val->AsString()->Render(); + + if ( interfaces_str[0] != '\0' ) + options.interface = interfaces_str; + + delete [] interfaces_str; + } + } + + if ( dns_type != DNS_PRIME ) + net_init(options.interface, options.pcap_file, options.pcap_output_file, options.use_watchdog); + + net_done = internal_handler("net_done"); + + if ( ! g_policy_debug ) + { + (void) setsignal(SIGTERM, sig_handler); + (void) setsignal(SIGINT, sig_handler); + (void) setsignal(SIGPIPE, SIG_IGN); + } + + // Cooperate with nohup(1). + if ( (oldhandler = setsignal(SIGHUP, sig_handler)) != SIG_DFL ) + (void) setsignal(SIGHUP, oldhandler); + + if ( dns_type == DNS_PRIME ) + { + dns_mgr->Verify(); + dns_mgr->Resolve(); + + if ( ! dns_mgr->Save() ) + reporter->FatalError("can't update DNS cache"); + + mgr.Drain(); + delete dns_mgr; + exit(0); + } + + // Print the ID. + if ( options.identifier_to_print ) + { + ID* id = global_scope()->Lookup(*options.identifier_to_print); + if ( ! id ) + reporter->FatalError("No such ID: %s\n", options.identifier_to_print->data()); + + ODesc desc; + desc.SetQuotes(true); + desc.SetIncludeStats(true); + id->DescribeExtended(&desc); + + fprintf(stdout, "%s\n", desc.Description()); + exit(0); + } + + if ( profiling_interval > 0 ) + { + profiling_logger = new ProfileLogger(profiling_file->AsFile(), + profiling_interval); + + if ( segment_profiling ) + segment_logger = profiling_logger; + } + + if ( ! reading_live && ! reading_traces ) + // Set up network_time to track real-time, since + // we don't have any other source for it. + net_update_time(current_time()); + + EventHandlerPtr zeek_init = internal_handler("zeek_init"); + if ( zeek_init ) //### this should be a function + mgr.Enqueue(zeek_init, zeek::Args{}); + + EventRegistry::string_list dead_handlers = + event_registry->UnusedHandlers(); + + if ( ! dead_handlers.empty() && check_for_unused_event_handlers ) + { + for ( const string& handler : dead_handlers ) + reporter->Warning("event handler never invoked: %s", handler.c_str()); + } + + // Enable LeakSanitizer before zeek_init() and even before executing + // top-level statements. Even though it's not bad if a leak happens only + // once at initialization, we have to assume that script-layer code causing + // such a leak can be placed in any arbitrary event handler and potentially + // cause more severe problems. + ZEEK_LSAN_ENABLE(); + + if ( stmts ) + { + stmt_flow_type flow; + Frame f(current_scope()->Length(), nullptr, nullptr); + g_frame_stack.push_back(&f); + + try + { + stmts->Exec(&f, flow); + } + catch ( InterpreterException& ) + { + reporter->FatalError("failed to execute script statements at top-level scope"); + } + + g_frame_stack.pop_back(); + } + + if ( options.ignore_checksums ) + ignore_checksums = 1; + + if ( zeek_script_loaded ) + { + // Queue events reporting loaded scripts. + for ( std::list::iterator i = files_scanned.begin(); i != files_scanned.end(); i++ ) + { + if ( i->skipped ) + continue; + + mgr.Enqueue(zeek_script_loaded, + make_intrusive(i->name.c_str()), + IntrusivePtr{AdoptRef{}, val_mgr->GetCount(i->include_level)} + ); + } + } + + reporter->ReportViaEvents(true); + + // Drain the event queue here to support the protocols framework configuring DPM + mgr.Drain(); + + if ( reporter->Errors() > 0 && ! zeekenv("ZEEK_ALLOW_INIT_ERRORS") ) + reporter->FatalError("errors occurred while initializing"); + + broker_mgr->ZeekInitDone(); + reporter->ZeekInitDone(); + analyzer_mgr->DumpDebug(); + + have_pending_timers = ! reading_traces && timer_mgr->Size() > 0; + + return {0, std::move(options)}; + } + +int zeek::cleanup(bool did_net_run) + { + if ( did_net_run ) + done_with_network(); + + net_delete(); + terminate_bro(); + + sqlite3_shutdown(); + + ERR_free_strings(); + EVP_cleanup(); + CRYPTO_cleanup_all_ex_data(); + + // Close files after net_delete(), because net_delete() + // might write to connection content files. + BroFile::CloseOpenFiles(); + + delete rule_matcher; + + return 0; + } diff --git a/src/zeek-setup.h b/src/zeek-setup.h new file mode 100644 index 0000000000..303ded4abd --- /dev/null +++ b/src/zeek-setup.h @@ -0,0 +1,31 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "Options.h" + +namespace zeek { + +struct SetupResult { + int code = 0; + zeek::Options options; +}; + +/** + * Initializes Zeek's global state. + * @param argc the argument count (same semantics as main function) + * @param argv the argument strings (same semantics as main function) + * @param options if provided, those options are used instead of + * deriving them by parsing the "argv" list. The "argv" list still + * needs to be provided regardless since some functionality requires + * it, particularly, several things use the value of argv[0]. + */ +SetupResult setup(int argc, char** argv, zeek::Options* options = nullptr); + +/** + * Cleans up Zeek's global state. + * @param did_net_run whether the net_run() was called. + */ +int cleanup(bool did_net_run); + +}