From 2bd4af7477c7d5401ababce579d7adaddb92fba0 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 25 Oct 2022 08:08:57 -0700 Subject: [PATCH 01/71] Add libunistd submodule, needed by future MSVC work --- .gitmodules | 3 +++ auxil/libunistd | 1 + 2 files changed, 4 insertions(+) create mode 160000 auxil/libunistd diff --git a/.gitmodules b/.gitmodules index 72c078326f..13774cbbfd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -70,3 +70,6 @@ [submodule "auxil/zeek-af_packet-plugin"] path = auxil/zeek-af_packet-plugin url = https://github.com/zeek/zeek-af_packet-plugin.git +[submodule "auxil/libunistd"] + path = auxil/libunistd + url = https://github.com/zeek/libunistd diff --git a/auxil/libunistd b/auxil/libunistd new file mode 160000 index 0000000000..cc7ac8d20a --- /dev/null +++ b/auxil/libunistd @@ -0,0 +1 @@ +Subproject commit cc7ac8d20a28905be963ae8f5684e793e198892a From 3a80b79497279a95ed10e7d51078862cb0302ca1 Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Sun, 27 Jun 2021 17:12:56 +0000 Subject: [PATCH 02/71] Compile Zeek with MSVC Allow Zeek to be embedded in another project --- .gitignore | 7 + CMakeLists.txt | 128 ++++++++++++++++-- ci/build.ps1 | 33 +++++ ci/windows/Dockerfile | 56 ++++++++ ci/windows/conanfile_windows.txt | 9 ++ ci/windows/default | 12 ++ src/CMakeLists.txt | 103 ++++++++++---- src/DebugCmdInfoConstants.in | 2 +- src/DebugLogger.h | 22 +-- src/Flare.cc | 74 +++++++++- src/Flare.h | 15 +- src/Hash.cc | 2 +- src/Hash.h | 1 + src/IP.cc | 18 +-- src/IP.h | 2 + src/Obj.cc | 4 + src/OpaqueVal.h | 4 + src/Options.cc | 6 +- src/PolicyFile.cc | 5 +- src/RunState.cc | 2 + src/SerializationFormat.cc | 13 +- src/SerializationFormat.h | 3 + src/analyzer/Manager.h | 6 +- .../analyzer/extract/functions.bif | 2 +- src/file_analysis/analyzer/x509/X509Common.cc | 2 +- .../x509-signed_certificate_timestamp.pac | 2 +- src/fuzzers/FuzzBuffer.cc | 4 + src/fuzzers/packet-fuzzer.cc | 4 + src/iosource/Packet.h | 2 + src/iosource/pcap/Source.cc | 2 + src/main.cc | 9 +- src/net_util.h | 2 + src/packet_analysis/protocol/arp/ARP.cc | 8 +- src/plugin/Manager.cc | 95 +++++++++---- src/plugin/Manager.h | 2 +- src/plugin/Plugin.cc | 4 + src/plugin/Plugin.h | 28 ++++ src/script_opt/ZAM/Stmt.cc | 4 + src/supervisor/Supervisor.cc | 20 ++- src/supervisor/Supervisor.h | 4 + src/telemetry/Counter.h | 12 +- src/telemetry/Gauge.h | 12 +- src/telemetry/Histogram.h | 12 +- src/threading/BasicThread.cc | 10 +- src/threading/BasicThread.h | 1 + src/util.cc | 54 ++++++-- src/util.h | 31 ++++- src/zeek | 1 - src/zeek-setup.cc | 9 ++ src/zeek.bif | 4 +- src/zeekygen/Target.cc | 5 + zeek-config.h.in | 3 +- zeek-path-dev.in | 2 +- 53 files changed, 724 insertions(+), 153 deletions(-) create mode 100644 ci/build.ps1 create mode 100644 ci/windows/Dockerfile create mode 100644 ci/windows/conanfile_windows.txt create mode 100644 ci/windows/default mode change 120000 => 100644 src/file_analysis/analyzer/x509/x509-signed_certificate_timestamp.pac delete mode 120000 src/zeek diff --git a/.gitignore b/.gitignore index 47d737f573..5a24d31e0f 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,10 @@ cmake-build-* # clangd .cache + +out/ + +# Visual Studio +.vs/ +.vscode/ +CMakeSettings.json diff --git a/CMakeLists.txt b/CMakeLists.txt index a2061b010c..f7c147d488 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,9 +2,86 @@ # auxil/zeek-aux/plugin-support/skeleton/CMakeLists.txt cmake_minimum_required(VERSION 3.15.0 FATAL_ERROR) +if ( WIN32 ) + # Enable usage of CMAKE_MSVC_RUNTIME_LIBRARY variable + cmake_policy(SET CMP0091 NEW) +endif() + project(Zeek C CXX) -include(GNUInstallDirs) +option(ZEEK_STANDALONE "Is Zeek compiled stand-alone or embedded in a parent project." ON) + +list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}) +list(APPEND CMAKE_PREFIX_PATH ${CMAKE_BINARY_DIR}) + +# Windows: Configure runtime and dependencies +if ( MSVC ) + cmake_policy(SET CMP0091 NEW) + if (CMAKE_BUILD_TYPE STREQUAL "Debug") + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreadedDebug") + set(CMAKE_MSVC_RUNTIME_LIBRARY_FLAG "MTd") + else () + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded") + set(CMAKE_MSVC_RUNTIME_LIBRARY_FLAG "MT") + endif () + set(OPENSSL_USE_STATIC_LIBS true) + set(OPENSSL_MSVC_STATIC_RT true) + + if ( ZEEK_STANDALONE ) + include(${CMAKE_SOURCE_DIR}/cmake/conan.cmake) + + conan_cmake_configure(REQUIRES + zlib/1.2.11 + openssl/1.1.1i + winflexbison/2.5.24 + npcap-wpcap/1.31 + GENERATORS cmake_find_package) + + # Install packages from conanfile + conan_cmake_install(PATH_OR_REFERENCE ${CMAKE_SOURCE_DIR}/ci/windows/conanfile_windows.txt + BUILD missing + SETTINGS ${settings}) + endif() + + # Set LibPCAP to point to wpcap binaries. + find_package(npcap-wpcap) + set(PCAP_ROOT_DIR "${npcap-wpcap_LIB_DIRS}/../") + set(PCAP_INCLUDE_DIR ${npcap-wpcap_INCLUDES}) + set(PCAP_LIBRARY ${npcap-wpcap_LIBS}) + set(LIBPCAP_PCAP_COMPILE_NOPCAP_HAS_ERROR_PARAMETER false) + + # Set ZLib to point at the right variable. + find_package(ZLIB) + set(ZLIB_LIBRARY ${ZLIB_LIBRARIES}) + + add_subdirectory(auxil/libunistd) + set(UNISTD_INCLUDES ${CMAKE_SOURCE_DIR}/auxil/libunistd/unistd ${CMAKE_SOURCE_DIR}/auxil/libunistd/regex) + include_directories(BEFORE ${UNISTD_INCLUDES}) + # Required for `check_include_files` to operate correctly + list(APPEND CMAKE_REQUIRED_INCLUDES ${UNISTD_INCLUDES}) + list(APPEND zeekdeps libunistd libregex) + + # Set CMAKE flags for supported windows build. + set(DISABLE_PYTHON_BINDINGS true) + set(BROKER_DISABLE_TESTS true) + set(BROKER_DISABLE_DOC_EXAMPLES true) +else () + include(GNUInstallDirs) +endif () + +set(CMAKE_FIND_LIBRARY_SUFFIXES .a ${CMAKE_FIND_LIBRARY_SUFFIXES}) +if ( CMAKE_SYSTEM_NAME STREQUAL Linux ) + set(CMAKE_DL_LIBS pthread ${CMAKE_DL_LIBS}) +endif () + +if ( NOT CMAKE_INSTALL_LIBDIR ) + # Currently, some sub-projects may use GNUInstallDirs.cmake to choose the + # library install dir, while others just default to "lib". For sake of + # consistency, this just overrides the former to always use "lib" in case + # it would have chosen something else, like "lib64", but a thing for the + # future may be to standardize all sub-projects to use GNUInstallDirs. + set(CMAKE_INSTALL_LIBDIR lib) +endif () include(cmake/CommonCMakeConfig.cmake) include(cmake/FindClangTidy.cmake) @@ -63,6 +140,21 @@ if (ZEEK_PLUGIN_DIR) else () set(BRO_PLUGIN_INSTALL_PATH ${ZEEK_LIBDIR_PATH}/plugins CACHE STRING "Installation path for plugins" FORCE) +endif() + +set(bro_plugin_install_path "${BRO_PLUGIN_INSTALL_PATH}") +set(cmake_binary_dir "${CMAKE_BINARY_DIR}") +set(cmake_current_binary_dir "${CMAKE_CURRENT_BINARY_DIR}") +set(cmake_install_prefix "${CMAKE_INSTALL_PREFIX}") +set(cmake_source_dir "${CMAKE_SOURCE_DIR}") +set(zeek_script_install_path "${ZEEK_SCRIPT_INSTALL_PATH}") +if ( MSVC ) + string(REGEX REPLACE "^([A-Za-z]):/(.*)" "/\\1/\\2" bro_plugin_install_path "${bro_plugin_install_path}") + string(REGEX REPLACE "^([A-Za-z]):/(.*)" "/\\1/\\2" cmake_binary_dir "${cmake_binary_dir}") + string(REGEX REPLACE "^([A-Za-z]):/(.*)" "/\\1/\\2" cmake_current_binary_dir "${cmake_current_binary_dir}") + string(REGEX REPLACE "^([A-Za-z]):/(.*)" "/\\1/\\2" cmake_install_prefix "${cmake_install_prefix}") + string(REGEX REPLACE "^([A-Za-z]):/(.*)" "/\\1/\\2" cmake_source_dir "${cmake_source_dir}") + string(REGEX REPLACE "^([A-Za-z]):/(.*)" "/\\1/\\2" zeek_script_install_path "${zeek_script_install_path}") endif () if ( NOT ZEEK_ETC_INSTALL_DIR ) @@ -300,7 +392,9 @@ FindRequiredPackage(FLEX) FindRequiredPackage(BISON) FindRequiredPackage(PCAP) FindRequiredPackage(OpenSSL) -FindRequiredPackage(BIND) +if ( NOT WIN32 ) + FindRequiredPackage(BIND) +endif () FindRequiredPackage(ZLIB) # Installation directory for the distribution's Python modules. An @@ -325,6 +419,9 @@ set(PY_MOD_INSTALL_DIR ${py_mod_install_dir} if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/auxil/binpac/CMakeLists.txt) set(ENABLE_STATIC_ONLY_SAVED ${ENABLE_STATIC_ONLY}) + if ( MSVC ) + set(BUILD_STATIC_BINPAC true) + endif() if ( BUILD_STATIC_BINPAC ) set(ENABLE_STATIC_ONLY true) @@ -381,6 +478,11 @@ if ( PYTHON_VERSION_STRING VERSION_LESS ${ZEEK_PYTHON_MIN} ) endif () add_subdirectory(auxil/paraglob) +if ( MSVC ) + cmake_policy(SET CMP0079 NEW) + target_link_libraries(paraglob shlwapi) + set(BROKER_DISABLE_TOOLS true) +endif () set(zeekdeps ${zeekdeps} paraglob) if ( Broker_ROOT ) @@ -397,6 +499,9 @@ else () endif () set(ENABLE_STATIC_ONLY_SAVED ${ENABLE_STATIC_ONLY}) + if ( MSVC ) + set(BUILD_STATIC_BROKER true) + endif() if ( BUILD_STATIC_BROKER ) set(ENABLE_STATIC_ONLY true) @@ -599,7 +704,12 @@ if ( ${CMAKE_SYSTEM_NAME} MATCHES Linux ) endif () endif () -set(DEFAULT_ZEEKPATH .:${ZEEK_SCRIPT_INSTALL_PATH}:${ZEEK_SCRIPT_INSTALL_PATH}/policy:${ZEEK_SCRIPT_INSTALL_PATH}/site:${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins) +set(DEFAULT_ZEEKPATH_PATHS . ${ZEEK_SCRIPT_INSTALL_PATH} ${ZEEK_SCRIPT_INSTALL_PATH}/policy ${ZEEK_SCRIPT_INSTALL_PATH}/site ${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins) +if ( WIN32 ) + list(JOIN DEFAULT_ZEEKPATH_PATHS ";" DEFAULT_ZEEKPATH) +else () + list(JOIN DEFAULT_ZEEKPATH_PATHS ":" DEFAULT_ZEEKPATH) +endif () if ( NOT BINARY_PACKAGING_MODE ) set(ZEEK_DIST ${PROJECT_SOURCE_DIR}) @@ -653,11 +763,13 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/cmake DESTINATION share/zeek USE_SOURCE_PERMISSIONS PATTERN ".git" EXCLUDE) -# Install wrapper script for Bro-to-Zeek renaming. -include(InstallShellScript) -include(InstallSymlink) -InstallShellScript("bin" "zeek-wrapper.in" "zeek-wrapper") -InstallSymlink("${CMAKE_INSTALL_PREFIX}/bin/zeek-wrapper" "${CMAKE_INSTALL_PREFIX}/bin/bro-config") +if ( NOT WIN32 ) + # Install wrapper script for Bro-to-Zeek renaming. + include(InstallShellScript) + include(InstallSymlink) + InstallShellScript("bin" "zeek-wrapper.in" "zeek-wrapper") + InstallSymlink("${CMAKE_INSTALL_PREFIX}/bin/zeek-wrapper" "${CMAKE_INSTALL_PREFIX}/bin/bro-config") +endif () ######################################################################## ## zkg configuration diff --git a/ci/build.ps1 b/ci/build.ps1 new file mode 100644 index 0000000000..9b8d397c59 --- /dev/null +++ b/ci/build.ps1 @@ -0,0 +1,33 @@ +param( + [Parameter()] + [ValidateSet("Debug", "Release")] + [string] $BuildType = "Release" +) + +$SourceDirectory = (Convert-Path "$PSScriptRoot/../").Replace("\", "/") +$WorkingDirectory = $pwd.Path + +$commands = @() +if (!(Get-Command cl)) { + $commands += '"C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Auxiliary/Build/vcvars64.bat"' +} + +$commands += @" +cmake.exe + -G "Ninja" + -S "$SourceDirectory" + -B "$WorkingDirectory" + -DCMAKE_BUILD_TYPE:STRING=$BuildType + -DCMAKE_INSTALL_PREFIX:PATH="$SourceDirectory/out/install/$BuildType" + -DDISABLE_PYTHON_BINDINGS=1 ` + 2>&1 +"@.Replace("`r`n", "") + +$commands += @" +cmake.exe --build $WorkingDirectory --config $BuildType +"@ + +$commands += @" +cmake.exe --install $WorkingDirectory +"@ +cmd /c ($commands -join " && ") \ No newline at end of file diff --git a/ci/windows/Dockerfile b/ci/windows/Dockerfile new file mode 100644 index 0000000000..c6ce894f74 --- /dev/null +++ b/ci/windows/Dockerfile @@ -0,0 +1,56 @@ +# escape=` +FROM mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-ltsc2019 + +# Restore the default Windows shell for correct batch processing. +SHELL ["cmd", "/S", "/C"] + +# Download the Build Tools bootstrapper. +ADD https://aka.ms/vs/16/release/vs_buildtools.exe C:\TEMP\vs_buildtools.exe + +# Install Build Tools. +RUN C:\TEMP\vs_buildtools.exe --quiet --wait --norestart --nocache ` + --installPath C:\BuildTools ` + --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended ` + --add Microsoft.VisualStudio.Component.VC.ATLMFC ` + --add Microsoft.VisualStudio.Component.Windows10SDK.18362 ` + --add Microsoft.VisualStudio.Component.Windows10SDK.17763 ` + --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` + --add Microsoft.VisualStudio.Component.VC.v141.x86.x64 ` + --remove Microsoft.VisualStudio.Component.Windows10SDK.10240 ` + --remove Microsoft.VisualStudio.Component.Windows10SDK.10586 ` + --remove Microsoft.VisualStudio.Component.Windows10SDK.14393 ` + --remove Microsoft.VisualStudio.Component.Windows81SDK ` + || IF "%ERRORLEVEL%"=="3010" EXIT 0 + + +SHELL [ "powershell" ] + +RUN Set-ExecutionPolicy Unrestricted -Force + +# Install Chocolatey +RUN [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; ` + iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) + +# Restore the default Windows shell for correct batch processing. +SHELL ["cmd", "/S", "/C"] + +# Install prerequisites +RUN choco install conan -y +RUN choco install winflexbison -y +RUN choco install openssl -y +RUN choco install python -y +# Can't install a proper msys2 because it caused console hanging in the container during docker build. +RUN choco install msysgit -y +RUN choco install sed -y + +# Set working environment. +RUN setx /M PATH "%PATH%;C:\\Program Files\\Git\\bin" +RUN mkdir C:\build +WORKDIR C:\build + +# Configure conan +ADD default C:\Users\ContainerAdministrator\.conan\profiles\default + +# Define the entry point for the docker container. +# This entry point starts the developer command prompt and launches the PowerShell shell. +ENTRYPOINT ["C:\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Unrestricted"] \ No newline at end of file diff --git a/ci/windows/conanfile_windows.txt b/ci/windows/conanfile_windows.txt new file mode 100644 index 0000000000..79db1eccc4 --- /dev/null +++ b/ci/windows/conanfile_windows.txt @@ -0,0 +1,9 @@ +[requires] +openssl/1.1.1i +zlib/1.2.11 +libpcap/1.10.1 +c-ares/1.18.1 + +[generators] +cmake_find_package +cmake diff --git a/ci/windows/default b/ci/windows/default new file mode 100644 index 0000000000..620a4bf3a9 --- /dev/null +++ b/ci/windows/default @@ -0,0 +1,12 @@ +[settings] +os=Windows +os_build=Windows +arch=x86_64 +arch_build=x86_64 +compiler=Visual Studio +compiler.version=16 +compiler.runtime=MT +build_type=Release +[options] +[build_requires] +[env] diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8c6ad63ba5..af63e54fd3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,5 +1,6 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/include ) @@ -31,9 +32,9 @@ configure_file(util-config.h.in ${CMAKE_CURRENT_BINARY_DIR}/util-config.h) # - deletes instances of 'extern char.*getenv' in inFile # - writes results to outFile and adds it to list TRANSFORMED_BISON_OUTPUTS macro(REPLACE_YY_PREFIX_TARGET inFile outFile yylexPrefix yyPrefix) - set(args "'/extern char.*getenv/d") + set(args "\"/extern char.*getenv/d") set(args "${args}\;s/yylex/${yylexPrefix}lex/") - set(args "${args}\;s/yy/${yyPrefix}/g'" < ${inFile} > ${outFile}) + set(args "${args}\;s/yy/${yyPrefix}/g\"" < ${inFile} > ${outFile}) add_custom_command(OUTPUT ${outFile} COMMAND ${SED_EXE} ARGS ${args} @@ -48,6 +49,12 @@ endmacro(REPLACE_YY_PREFIX_TARGET) set(BISON_FLAGS "--debug") +if ( MSVC ) + set(SIGN_COMPARE_FLAG "/wd4018") +else() + set(SIGN_COMPARE_FLAG "-Wno-sign-compare") +endif() + # BIF parser/scanner bison_target(BIFParser builtin-func.y ${CMAKE_CURRENT_BINARY_DIR}/bif_parse.cc @@ -56,7 +63,7 @@ bison_target(BIFParser builtin-func.y COMPILE_FLAGS "${BISON_FLAGS}") flex_target(BIFScanner builtin-func.l ${CMAKE_CURRENT_BINARY_DIR}/bif_lex.cc) add_flex_bison_dependency(BIFScanner BIFParser) -set_property(SOURCE bif_lex.cc APPEND_STRING PROPERTY COMPILE_FLAGS "-Wno-sign-compare") +set_property(SOURCE bif_lex.cc APPEND_STRING PROPERTY COMPILE_FLAGS "${SIGN_COMPARE_FLAG}") # Rule parser/scanner bison_target(RuleParser rule-parse.y @@ -72,7 +79,7 @@ replace_yy_prefix_target(${CMAKE_CURRENT_BINARY_DIR}/rup.h rules_ rules_) flex_target(RuleScanner rule-scan.l ${CMAKE_CURRENT_BINARY_DIR}/rule-scan.cc COMPILE_FLAGS "-Prules_") -set_property(SOURCE rule-scan.cc APPEND_STRING PROPERTY COMPILE_FLAGS "-Wno-sign-compare") +set_property(SOURCE rule-scan.cc APPEND_STRING PROPERTY COMPILE_FLAGS "${SIGN_COMPARE_FLAG}") # RE parser/scanner bison_target(REParser re-parse.y @@ -86,7 +93,7 @@ replace_yy_prefix_target(${CMAKE_CURRENT_BINARY_DIR}/rep.cc flex_target(REScanner re-scan.l ${CMAKE_CURRENT_BINARY_DIR}/re-scan.cc COMPILE_FLAGS "-Pre_") add_flex_bison_dependency(REScanner REParser) -set_property(SOURCE re-scan.cc APPEND_STRING PROPERTY COMPILE_FLAGS "-Wno-sign-compare") +set_property(SOURCE re-scan.cc APPEND_STRING PROPERTY COMPILE_FLAGS "${SIGN_COMPARE_FLAG}") # Parser/Scanner bison_target(Parser parse.y @@ -99,13 +106,15 @@ replace_yy_prefix_target(${CMAKE_CURRENT_BINARY_DIR}/p.cc zeek yy) flex_target(Scanner scan.l ${CMAKE_CURRENT_BINARY_DIR}/scan.cc COMPILE_FLAGS "-Pzeek") -set_property(SOURCE scan.cc APPEND_STRING PROPERTY COMPILE_FLAGS "-Wno-sign-compare") +set_property(SOURCE scan.cc APPEND_STRING PROPERTY COMPILE_FLAGS "${SIGN_COMPARE_FLAG}") ######################################################################## ## bifcl-dependent targets include(BifCl) +set(SUPERVISOR_SRCS supervisor/Supervisor.cc Pipe.cc) + set(BIF_SRCS zeek.bif stats.bif @@ -139,9 +148,9 @@ endforeach () include(BinPAC) set(BINPAC_AUXSRC - ${PROJECT_SOURCE_DIR}/src/binpac.pac - ${PROJECT_SOURCE_DIR}/src/zeek.pac - ${PROJECT_SOURCE_DIR}/src/binpac_zeek.h + ${CMAKE_CURRENT_SOURCE_DIR}/binpac.pac + ${CMAKE_CURRENT_SOURCE_DIR}/zeek.pac + ${CMAKE_CURRENT_SOURCE_DIR}/binpac_zeek.h ) binpac_target(binpac-lib.pac) @@ -258,6 +267,11 @@ set(_gen_zeek_script_cpp ${CMAKE_CURRENT_BINARY_DIR}/../CPP-gen.cc) add_custom_command(OUTPUT ${_gen_zeek_script_cpp} COMMAND ${CMAKE_COMMAND} -E touch ${_gen_zeek_script_cpp}) +if (!MSVC) + set_source_files_properties(legacy-netvar-init.cc PROPERTIES COMPILE_FLAGS + -Wno-deprecated-declarations) +endif() + set(MAIN_SRCS digest.cc net_util.cc @@ -311,7 +325,6 @@ set(MAIN_SRCS Options.cc Overflow.cc PacketFilter.cc - Pipe.cc PolicyFile.cc PrefixTable.cc PriorityQueue.cc @@ -346,7 +359,7 @@ set(MAIN_SRCS ZeekString.cc ZVal.cc - supervisor/Supervisor.cc + ${SUPERVISOR_SRCS} threading/BasicThread.cc threading/Formatter.cc @@ -504,23 +517,59 @@ collect_headers(zeek_HEADERS ${zeek_SRCS}) add_library(zeek_objs OBJECT ${zeek_SRCS}) -add_executable(zeek main.cc - $ - ${zeek_HEADERS} - ${bro_SUBDIR_LIBS} - ${bro_PLUGIN_LIBS} -) -target_link_libraries(zeek ${bro_PLUGIN_LINK_LIBS} ${zeekdeps} ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) -# Export symbols from zeek executable for use by plugins -set_target_properties(zeek PROPERTIES ENABLE_EXPORTS TRUE) +if (ZEEK_STANDALONE) + add_executable(zeek main.cc + $ + ${zeek_HEADERS} + ${bro_SUBDIR_LIBS} + ${bro_PLUGIN_LIBS} + ) + target_link_libraries(zeek ${bro_PLUGIN_LINK_LIBS} ${zeekdeps} ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) + # Export symbols from zeek executable for use by plugins + set_target_properties(zeek PROPERTIES ENABLE_EXPORTS TRUE) -install(TARGETS zeek DESTINATION bin) + if ( MSVC ) + set(WINDOWS_EXPORT_ALL_SYMBOLS ON) + endif () -set(BRO_EXE zeek - CACHE STRING "Zeek executable binary" FORCE) + install(TARGETS zeek RUNTIME DESTINATION bin ARCHIVE DESTINATION lib) -set(BRO_EXE_PATH ${CMAKE_CURRENT_BINARY_DIR}/zeek - CACHE STRING "Path to Zeek executable binary" FORCE) + set(BRO_EXE zeek + CACHE STRING "Zeek executable binary" FORCE) + + set(BRO_EXE_PATH ${CMAKE_CURRENT_BINARY_DIR}/zeek + CACHE STRING "Path to Zeek executable binary" FORCE) + +else() + add_library(zeek STATIC $ ${zeek_HEADERS}) + + target_link_libraries(zeek PUBLIC ${zeekdeps} + ${CMAKE_THREAD_LIBS_INIT} + ${CMAKE_DL_LIBS} + ${bro_SUBDIR_LIBS} + ${bro_PLUGIN_LIBS}) + + target_include_directories(zeek PUBLIC + ${CMAKE_SOURCE_DIR}/zeek/src + ${CMAKE_SOURCE_DIR}/zeek/src/include + ${CMAKE_BINARY_DIR} + ${CMAKE_BINARY_DIR}/zeek/src + ${CMAKE_BINARY_DIR}/zeek/src/include + ${CMAKE_SOURCE_DIR}/zeek/src/windows/usr.include) +endif() + +if ( NOT WIN32 ) + # Install wrapper script for Bro-to-Zeek renaming. + include(InstallSymlink) + InstallSymlink("${CMAKE_INSTALL_PREFIX}/bin/zeek-wrapper" "${CMAKE_INSTALL_PREFIX}/bin/bro") +endif () + +if ( NOT BINARY_PACKAGING_MODE ) + # Older plugins may still use `bro` in unit tests. + execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink + "${CMAKE_CURRENT_BINARY_DIR}/../zeek-wrapper.in" + "${CMAKE_CURRENT_BINARY_DIR}/bro") +endif () # Target to create all the autogenerated files. add_custom_target(generate_outputs_stage1) @@ -567,7 +616,7 @@ install(CODE " # Make sure to escape a bunch of special characters in the path before trying to use it as a # regular expression below. -string(REGEX REPLACE "([][+.*()^])" "\\\\\\1" escaped_path "${CMAKE_CURRENT_SOURCE_DIR}/zeek") +string(REGEX REPLACE "([][+.*()^])" "\\\\\\1" escaped_path "${CMAKE_CURRENT_SOURCE_DIR}/include/*") install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ DESTINATION include/zeek @@ -592,6 +641,8 @@ install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ PATTERN "*.bif.netvar_h" PATTERN "*.bif.h" PATTERN "CMakeFiles" EXCLUDE + # The "include/zeek -> .." symlink isn't needed in the install-tree + REGEX "${escaped_path}$" EXCLUDE ) install(FILES diff --git a/src/DebugCmdInfoConstants.in b/src/DebugCmdInfoConstants.in index ad90d7ed83..c1b3d112fd 100644 --- a/src/DebugCmdInfoConstants.in +++ b/src/DebugCmdInfoConstants.in @@ -1,6 +1,6 @@ // This invalid entry should always be first cmd: dcInvalid -names: +names: _ resume: false help: This function should not be called repeatable: false diff --git a/src/DebugLogger.h b/src/DebugLogger.h index 591bfb85c4..e60c1b8c3c 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -13,17 +13,17 @@ #include "zeek/util.h" -#define DBG_LOG(stream, args...) \ - if ( ::zeek::detail::debug_logger.IsEnabled(stream) ) \ - ::zeek::detail::debug_logger.Log(stream, args) -#define DBG_LOG_VERBOSE(stream, args...) \ - if ( ::zeek::detail::debug_logger.IsVerbose() && \ - ::zeek::detail::debug_logger.IsEnabled(stream) ) \ - ::zeek::detail::debug_logger.Log(stream, args) +#define DBG_LOG(stream, ...) \ + if ( ::zeek::detail::debug_logger.IsEnabled(stream) ) \ + ::zeek::detail::debug_logger.Log(stream, __VA_ARGS__) +#define DBG_LOG_VERBOSE(stream, ...) \ + if ( ::zeek::detail::debug_logger.IsVerbose() && \ + ::zeek::detail::debug_logger.IsEnabled(stream) ) \ + ::zeek::detail::debug_logger.Log(stream, __VA_ARGS__) #define DBG_PUSH(stream) ::zeek::detail::debug_logger.PushIndent(stream) #define DBG_POP(stream) ::zeek::detail::debug_logger.PopIndent(stream) -#define PLUGIN_DBG_LOG(plugin, args...) ::zeek::detail::debug_logger.Log(plugin, args) +#define PLUGIN_DBG_LOG(plugin, ...) ::zeek::detail::debug_logger.Log(plugin, __VA_ARGS__) namespace zeek { @@ -123,9 +123,9 @@ extern DebugLogger debug_logger; } // namespace zeek #else -#define DBG_LOG(args...) -#define DBG_LOG_VERBOSE(args...) +#define DBG_LOG(...) +#define DBG_LOG_VERBOSE(...) #define DBG_PUSH(stream) #define DBG_POP(stream) -#define PLUGIN_DBG_LOG(plugin, args...) +#define PLUGIN_DBG_LOG(plugin, ...) #endif diff --git a/src/Flare.cc b/src/Flare.cc index e2f42fa34e..cbe47ab005 100644 --- a/src/Flare.cc +++ b/src/Flare.cc @@ -8,10 +8,62 @@ #include "zeek/Reporter.h" +#if defined(_MSC_VER) + +#include + +#define fatalError(...) \ +do \ + { \ + if ( reporter ) \ + reporter->FatalError(__VA_ARGS__); \ + else \ + { \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + _exit(1); \ + } \ + } \ +while (0) + +#endif + namespace zeek::detail { -Flare::Flare() : pipe(FD_CLOEXEC, FD_CLOEXEC, O_NONBLOCK, O_NONBLOCK) { } +Flare::Flare() +#if ! defined(_MSC_VER) + : pipe(FD_CLOEXEC, FD_CLOEXEC, O_NONBLOCK, O_NONBLOCK) + { + } +#else + { + WSADATA wsaData; + if ( WSAStartup(MAKEWORD(2,2), &wsaData) != 0 ) + fatalError("WSAStartup failure: %d", WSAGetLastError()); + + recvfd = WSASocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP, nullptr, 0, + WSA_FLAG_OVERLAPPED | WSA_FLAG_NO_HANDLE_INHERIT); + if ( recvfd == (int) INVALID_SOCKET ) + fatalError("WSASocket failure: %d", WSAGetLastError()); + sendfd = WSASocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP, nullptr, 0, + WSA_FLAG_OVERLAPPED | WSA_FLAG_NO_HANDLE_INHERIT); + if ( sendfd == (int) INVALID_SOCKET ) + fatalError("WSASocket failure: %d", WSAGetLastError()); + + sockaddr_in sa; + memset(&sa, 0, sizeof(sa)); + sa.sin_family = AF_INET; + sa.sin_addr.s_addr = inet_addr("127.0.0.1"); + if ( bind(recvfd, (sockaddr*) &sa, sizeof(sa)) == SOCKET_ERROR ) + fatalError("bind failure: %d", WSAGetLastError()); + int salen = sizeof(sa); + if ( getsockname(recvfd, (sockaddr*) &sa, &salen) == SOCKET_ERROR ) + fatalError("getsockname failure: %d", WSAGetLastError()); + if ( connect(sendfd, (sockaddr*) &sa, sizeof(sa)) == SOCKET_ERROR ) + fatalError("connect failure: %d", WSAGetLastError()); + } +#endif [[noreturn]] static void bad_pipe_op(const char* which, bool signal_safe) { @@ -36,14 +88,22 @@ void Flare::Fire(bool signal_safe) for ( ;; ) { +#if ! defined(_MSC_VER) int n = write(pipe.WriteFD(), &tmp, 1); +#else + int n = send(sendfd, &tmp, 1, 0); +#endif if ( n > 0 ) // Success -- wrote a byte to pipe. break; if ( n < 0 ) { +#if defined(_MSC_VER) + errno = WSAGetLastError(); + bad_pipe_op("send", signal_safe); +#endif if ( errno == EAGAIN ) // Success: pipe is full and just need at least one byte in it. break; @@ -66,15 +126,23 @@ int Flare::Extinguish(bool signal_safe) for ( ;; ) { +#if ! defined(_MSC_VER) int n = read(pipe.ReadFD(), &tmp, sizeof(tmp)); - +#else + int n = recv(recvfd, tmp, sizeof(tmp), 0); +#endif if ( n >= 0 ) { rval += n; // Pipe may not be empty yet: try again. continue; } - +#if defined(_MSC_VER) + if ( WSAGetLastError() == WSAEWOULDBLOCK ) + break; + errno = WSAGetLastError(); + bad_pipe_op("recv", signal_safe); +#endif if ( errno == EAGAIN ) // Success: pipe is now empty. break; diff --git a/src/Flare.h b/src/Flare.h index e1bfe3b6af..a06996338f 100644 --- a/src/Flare.h +++ b/src/Flare.h @@ -2,7 +2,9 @@ #pragma once -#include "zeek/Pipe.h" +#if ! defined(_MSC_VER) +#include "Pipe.h" +#endif namespace zeek::detail { @@ -22,7 +24,12 @@ public: * @return a file descriptor that will become ready if the flare has been * Fire()'d and not yet Extinguished()'d. */ - int FD() const { return pipe.ReadFD(); } + int FD() const +#if ! defined(_MSC_VER) + { return pipe.ReadFD(); } +#else + { return recvfd; } +#endif /** * Put the object in the "ready" state. @@ -41,7 +48,11 @@ public: int Extinguish(bool signal_safe = false); private: +#if ! defined(_MSC_VER) Pipe pipe; +#else + int sendfd, recvfd; +#endif }; } // namespace zeek::detail diff --git a/src/Hash.cc b/src/Hash.cc index 9229655800..68db773de8 100644 --- a/src/Hash.cc +++ b/src/Hash.cc @@ -345,7 +345,7 @@ void HashKey::Reserve(const char* tag, size_t addl_size, size_t alignment) void HashKey::Allocate() { - if ( key != nullptr and key != reinterpret_cast(&key_u) ) + if ( key != nullptr && key != reinterpret_cast(&key_u) ) { reporter->InternalWarning("usage error in HashKey::Allocate(): already allocated"); return; diff --git a/src/Hash.h b/src/Hash.h index 4f70ce1219..06ec01ae0b 100644 --- a/src/Hash.h +++ b/src/Hash.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include "zeek/util.h" // for zeek_int_t diff --git a/src/IP.cc b/src/IP.cc index c85cdc1e96..4de4506387 100644 --- a/src/IP.cc +++ b/src/IP.cc @@ -66,8 +66,8 @@ RecordValPtr IPv6_Hdr::ToVal(VectorValPtr chain) const static auto ip6_hdr_type = id::find_type("ip6_hdr"); rv = make_intrusive(ip6_hdr_type); const struct ip6_hdr* ip6 = (const struct ip6_hdr*)data; - rv->Assign(0, (ntohl(ip6->ip6_flow) & 0x0ff00000) >> 20); - rv->Assign(1, ntohl(ip6->ip6_flow) & 0x000fffff); + rv->Assign(0, static_cast(ntohl(ip6->ip6_flow) & 0x0ff00000) >> 20); + rv->Assign(1, static_cast(ntohl(ip6->ip6_flow) & 0x000fffff)); rv->Assign(2, ntohs(ip6->ip6_plen)); rv->Assign(3, ip6->ip6_nxt); rv->Assign(4, ip6->ip6_hlim); @@ -127,7 +127,7 @@ RecordValPtr IPv6_Hdr::ToVal(VectorValPtr chain) const rv->Assign(2, (ntohs(frag->ip6f_offlg) & 0xfff8) >> 3); rv->Assign(3, (ntohs(frag->ip6f_offlg) & 0x0006) >> 1); rv->Assign(4, static_cast(ntohs(frag->ip6f_offlg) & 0x0001)); - rv->Assign(5, ntohl(frag->ip6f_ident)); + rv->Assign(5, static_cast(ntohl(frag->ip6f_ident))); } break; @@ -138,13 +138,13 @@ RecordValPtr IPv6_Hdr::ToVal(VectorValPtr chain) const rv->Assign(0, ((ip6_ext*)data)->ip6e_nxt); rv->Assign(1, ((ip6_ext*)data)->ip6e_len); rv->Assign(2, ntohs(((uint16_t*)data)[1])); - rv->Assign(3, ntohl(((uint32_t*)data)[1])); + rv->Assign(3, static_cast(ntohl(((uint32_t*)data)[1]))); if ( Length() >= 12 ) { // Sequence Number and ICV fields can only be extracted if // Payload Len was non-zero for this header. - rv->Assign(4, ntohl(((uint32_t*)data)[2])); + rv->Assign(4, static_cast(ntohl(((uint32_t*)data)[2]))); uint16_t off = 3 * sizeof(uint32_t); rv->Assign(5, new String(data + off, Length() - off, true)); } @@ -156,8 +156,8 @@ RecordValPtr IPv6_Hdr::ToVal(VectorValPtr chain) const static auto ip6_esp_type = id::find_type("ip6_esp"); rv = make_intrusive(ip6_esp_type); const uint32_t* esp = (const uint32_t*)data; - rv->Assign(0, ntohl(esp[0])); - rv->Assign(1, ntohl(esp[1])); + rv->Assign(0, static_cast(ntohl(esp[0]))); + rv->Assign(1, static_cast(ntohl(esp[1]))); } break; @@ -401,8 +401,8 @@ RecordValPtr IP_Hdr::ToPktHdrVal(RecordValPtr pkt_hdr, int sindex) const tcp_hdr->Assign(0, val_mgr->Port(ntohs(tp->th_sport), TRANSPORT_TCP)); tcp_hdr->Assign(1, val_mgr->Port(ntohs(tp->th_dport), TRANSPORT_TCP)); - tcp_hdr->Assign(2, ntohl(tp->th_seq)); - tcp_hdr->Assign(3, ntohl(tp->th_ack)); + tcp_hdr->Assign(2, static_cast(ntohl(tp->th_seq))); + tcp_hdr->Assign(3, static_cast(ntohl(tp->th_ack))); tcp_hdr->Assign(4, tcp_hdr_len); tcp_hdr->Assign(5, data_len); tcp_hdr->Assign(6, tp->th_x2); diff --git a/src/IP.h b/src/IP.h index 0488290b00..76f17f7237 100644 --- a/src/IP.h +++ b/src/IP.h @@ -12,6 +12,8 @@ #ifdef HAVE_NETINET_IP6_H #include +#else +#include "net_util.h" // for struct ip6_hdr #endif #include diff --git a/src/Obj.cc b/src/Obj.cc index fd1b9100aa..47bbf1462d 100644 --- a/src/Obj.cc +++ b/src/Obj.cc @@ -61,10 +61,14 @@ Obj::~Obj() { if ( notify_plugins ) { +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif PLUGIN_HOOK_VOID(HOOK_BRO_OBJ_DTOR, HookBroObjDtor(this)); +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif PLUGIN_HOOK_VOID(HOOK_OBJ_DTOR, HookObjDtor(this)); } diff --git a/src/OpaqueVal.h b/src/OpaqueVal.h index e18f889530..7f63a38569 100644 --- a/src/OpaqueVal.h +++ b/src/OpaqueVal.h @@ -2,6 +2,10 @@ #pragma once +#ifdef _MSC_VER +#include +#endif + #include #if ( OPENSSL_VERSION_NUMBER < 0x30000000L ) || defined(LIBRESSL_VERSION_NUMBER) #include diff --git a/src/Options.cc b/src/Options.cc index 98f251b19a..31e5b38c86 100644 --- a/src/Options.cc +++ b/src/Options.cc @@ -6,7 +6,10 @@ #include -#ifdef HAVE_GETOPT_H +#include "zeek/ScriptProfile.h" +#include "zeek/script_opt/ScriptOpt.h" + +#if defined(HAVE_GETOPT_H) && ! defined(_MSC_VER) #include #endif @@ -15,7 +18,6 @@ #include #include -#include "zeek/ScriptProfile.h" #include "zeek/logging/writers/ascii/Ascii.h" #include "zeek/script_opt/ScriptOpt.h" diff --git a/src/PolicyFile.cc b/src/PolicyFile.cc index 31e44dcdd9..db3cf77ff2 100644 --- a/src/PolicyFile.cc +++ b/src/PolicyFile.cc @@ -117,9 +117,10 @@ bool LoadPolicyFileText(const char* policy_filename, // ### This code is not necessarily Unicode safe! // (probably fine with UTF-8) pf->filedata = new char[size + 1]; - if ( fread(pf->filedata, size, 1, f) != 1 ) + size_t n = fread(pf->filedata, 1, size, f); + if ( ferror(f) ) reporter->InternalError("Failed to fread() file data"); - pf->filedata[size] = 0; + pf->filedata[n] = 0; fclose(f); } diff --git a/src/RunState.cc b/src/RunState.cc index 315ddcd24d..6a1d4c3387 100644 --- a/src/RunState.cc +++ b/src/RunState.cc @@ -40,10 +40,12 @@ extern "C" #include "zeek/plugin/Manager.h" #include "zeek/session/Manager.h" +#ifndef _MSC_VER extern "C" { extern int select(int, fd_set*, fd_set*, fd_set*, struct timeval*); } +#endif static double last_watchdog_proc_time = 0.0; // value of above during last watchdog extern int signal_val; diff --git a/src/SerializationFormat.cc b/src/SerializationFormat.cc index 42cff63cdd..b723387ceb 100644 --- a/src/SerializationFormat.cc +++ b/src/SerializationFormat.cc @@ -325,6 +325,13 @@ bool BinarySerializationFormat::Write(uint32_t v, const char* tag) return WriteData(&v, sizeof(v)); } +#if defined(_MSC_VER) +bool BinarySerializationFormat::Write(u_long v, const char* tag) + { + return Write((uint32_t) v, tag); + } +#endif + bool BinarySerializationFormat::Write(int v, const char* tag) { DBG_LOG(DBG_SERIAL, "Write int %d [%s]", v, tag); @@ -386,7 +393,7 @@ bool BinarySerializationFormat::Write(const IPAddr& addr, const char* tag) for ( int i = 0; i < n; ++i ) { - if ( ! Write(ntohl(raw[i]), "addr-part") ) + if ( ! Write(static_cast(ntohl(raw[i])), "addr-part") ) return false; } @@ -402,7 +409,7 @@ bool BinarySerializationFormat::Write(const struct in_addr& addr, const char* ta { const uint32_t* bytes = (uint32_t*)&addr.s_addr; - if ( ! Write(ntohl(bytes[0]), "addr4") ) + if ( ! Write(static_cast(ntohl(bytes[0])), "addr4") ) return false; return true; @@ -414,7 +421,7 @@ bool BinarySerializationFormat::Write(const struct in6_addr& addr, const char* t for ( int i = 0; i < 4; ++i ) { - if ( ! Write(ntohl(bytes[i]), "addr6-part") ) + if ( ! Write(static_cast(ntohl(bytes[i])), "addr6-part") ) return false; } diff --git a/src/SerializationFormat.h b/src/SerializationFormat.h index 44215792bb..060d0a996d 100644 --- a/src/SerializationFormat.h +++ b/src/SerializationFormat.h @@ -126,6 +126,9 @@ public: bool Write(int v, const char* tag) override; bool Write(uint16_t v, const char* tag) override; bool Write(uint32_t v, const char* tag) override; +#if defined(_MSC_VER) + bool Write(u_long v, const char* tag); +#endif bool Write(int64_t v, const char* tag) override; bool Write(uint64_t v, const char* tag) override; bool Write(char v, const char* tag) override; diff --git a/src/analyzer/Manager.h b/src/analyzer/Manager.h index 22167e2f03..f33241881d 100644 --- a/src/analyzer/Manager.h +++ b/src/analyzer/Manager.h @@ -402,12 +402,12 @@ extern analyzer::Manager* analyzer_mgr; DBG_LOG(zeek::DBG_ANALYZER, "%s " txt, \ fmt_conn_id(conn->OrigAddr(), ntohs(conn->OrigPort()), conn->RespAddr(), \ ntohs(conn->RespPort()))); -#define DBG_ANALYZER_ARGS(conn, fmt, args...) \ +#define DBG_ANALYZER_ARGS(conn, fmt, ...) \ DBG_LOG(zeek::DBG_ANALYZER, "%s " fmt, \ fmt_conn_id(conn->OrigAddr(), ntohs(conn->OrigPort()), conn->RespAddr(), \ ntohs(conn->RespPort())), \ - ##args); + ##__VA_ARGS__); #else #define DBG_ANALYZER(conn, txt) -#define DBG_ANALYZER_ARGS(conn, fmt, args...) +#define DBG_ANALYZER_ARGS(conn, fmt, ...) #endif diff --git a/src/file_analysis/analyzer/extract/functions.bif b/src/file_analysis/analyzer/extract/functions.bif index 96a1b07da1..25a021449d 100644 --- a/src/file_analysis/analyzer/extract/functions.bif +++ b/src/file_analysis/analyzer/extract/functions.bif @@ -3,7 +3,7 @@ module FileExtract; %%{ -#include "zeek/zeek/file_analysis/Manager.h" +#include "zeek/file_analysis/Manager.h" #include "zeek/file_analysis/file_analysis.bif.h" %%} diff --git a/src/file_analysis/analyzer/x509/X509Common.cc b/src/file_analysis/analyzer/x509/X509Common.cc index 9d5f839b0b..85bd3ae6e9 100644 --- a/src/file_analysis/analyzer/x509/X509Common.cc +++ b/src/file_analysis/analyzer/x509/X509Common.cc @@ -279,7 +279,7 @@ void X509Common::ParseExtension(X509_EXTENSION* ex, const EventHandlerPtr& h, bo auto pX509Ext = make_intrusive(BifType::Record::X509::Extension); pX509Ext->Assign(0, name); - if ( short_name and strlen(short_name) > 0 ) + if ( short_name && strlen(short_name) > 0 ) pX509Ext->Assign(1, short_name); pX509Ext->Assign(2, oid); diff --git a/src/file_analysis/analyzer/x509/x509-signed_certificate_timestamp.pac b/src/file_analysis/analyzer/x509/x509-signed_certificate_timestamp.pac deleted file mode 120000 index 88305ed8fd..0000000000 --- a/src/file_analysis/analyzer/x509/x509-signed_certificate_timestamp.pac +++ /dev/null @@ -1 +0,0 @@ -../../../analyzer/protocol/ssl/tls-handshake-signed_certificate_timestamp.pac \ No newline at end of file diff --git a/src/file_analysis/analyzer/x509/x509-signed_certificate_timestamp.pac b/src/file_analysis/analyzer/x509/x509-signed_certificate_timestamp.pac new file mode 100644 index 0000000000..5ebb1abe9a --- /dev/null +++ b/src/file_analysis/analyzer/x509/x509-signed_certificate_timestamp.pac @@ -0,0 +1 @@ +%include ../../../analyzer/protocol/ssl/tls-handshake-signed_certificate_timestamp.pac \ No newline at end of file diff --git a/src/fuzzers/FuzzBuffer.cc b/src/fuzzers/FuzzBuffer.cc index 5529e98531..81251473a4 100644 --- a/src/fuzzers/FuzzBuffer.cc +++ b/src/fuzzers/FuzzBuffer.cc @@ -2,6 +2,10 @@ #define _GNU_SOURCE #endif +#ifdef _MSC_VER +#include +#endif + #include "zeek/fuzzers/FuzzBuffer.h" #include diff --git a/src/fuzzers/packet-fuzzer.cc b/src/fuzzers/packet-fuzzer.cc index af199a1544..49b183bda4 100644 --- a/src/fuzzers/packet-fuzzer.cc +++ b/src/fuzzers/packet-fuzzer.cc @@ -1,3 +1,7 @@ +#ifdef _MSC_VER +#include +#endif + extern "C" { #include diff --git a/src/iosource/Packet.h b/src/iosource/Packet.h index cedc235493..2719047415 100644 --- a/src/iosource/Packet.h +++ b/src/iosource/Packet.h @@ -11,6 +11,8 @@ using pkt_timeval = bpf_timeval; #else using pkt_timeval = struct timeval; +#include +#include #endif #include // For DLT_ constants diff --git a/src/iosource/pcap/Source.cc b/src/iosource/pcap/Source.cc index 210263cf7c..4ebd8e77ca 100644 --- a/src/iosource/pcap/Source.cc +++ b/src/iosource/pcap/Source.cc @@ -160,7 +160,9 @@ void PcapSource::OpenLive() Info(util::fmt("pcap bufsize = %d\n", ((struct pcap*)pd)->bufsize)); #endif +#ifndef _MSC_VER props.selectable_fd = pcap_get_selectable_fd(pd); +#endif props.link_type = pcap_datalink(pd); props.is_live = true; diff --git a/src/main.cc b/src/main.cc index 254fb14c58..7436292039 100644 --- a/src/main.cc +++ b/src/main.cc @@ -7,8 +7,16 @@ #include "zeek/supervisor/Supervisor.h" #include "zeek/zeek-setup.h" +#if defined(_MSC_VER) +#include // For _O_BINARY. +#endif + int main(int argc, char** argv) { +#if defined(_MSC_VER) + _setmode(_fileno(stdout), _O_BINARY); + _setmode(_fileno(stderr), _O_BINARY); +#endif auto time_start = zeek::util::current_time(true); auto setup_result = zeek::detail::setup(argc, argv); @@ -41,7 +49,6 @@ int main(int argc, char** argv) zeek::detail::timer_mgr->Add(new zeek::detail::ParentProcessCheckTimer(1, 1)); double time_net_start = zeek::util::current_time(true); - ; uint64_t mem_net_start_total; uint64_t mem_net_start_malloced; diff --git a/src/net_util.h b/src/net_util.h index 8ddc4d99e9..7031eed81e 100644 --- a/src/net_util.h +++ b/src/net_util.h @@ -283,6 +283,7 @@ inline uint64_t htonll(uint64_t i) #else +#ifndef _MSC_VER inline double ntohd(double d) { assert(sizeof(d) == 8); @@ -328,6 +329,7 @@ inline float htonf(float f) { return ntohf(f); } +#endif #ifndef HAVE_BYTEORDER_64 inline uint64_t ntohll(uint64_t i) diff --git a/src/packet_analysis/protocol/arp/ARP.cc b/src/packet_analysis/protocol/arp/ARP.cc index 51ec3aab02..40ee22247b 100644 --- a/src/packet_analysis/protocol/arp/ARP.cc +++ b/src/packet_analysis/protocol/arp/ARP.cc @@ -94,7 +94,7 @@ bool ARPAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* packet) auto ah = (const struct arp_pkthdr*)data; // Check the size. - size_t min_length = (ar_tpa(ah) - (char*)data) + ah->ar_pln; + size_t min_length = (ar_tpa(ah) - (caddr_t) data) + ah->ar_pln; if ( min_length > len ) { Weird("truncated_ARP", packet); @@ -149,7 +149,7 @@ bool ARPAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* packet) } // Check MAC src address = ARP sender MAC address. - if ( memcmp(packet->l2_src, ar_sha(ah), ah->ar_hln) != 0 ) + if ( memcmp(packet->l2_src, (const char*)ar_sha(ah), ah->ar_hln) != 0 ) { BadARPEvent(ah, "weird-arp-sha"); return false; @@ -219,9 +219,9 @@ void ARPAnalyzer::BadARPEvent(const struct arp_pkthdr* hdr, const char* fmt, ... vsnprintf(msg, sizeof(msg), fmt, args); va_end(args); - event_mgr.Enqueue(bad_arp, ToAddrVal(ar_spa(hdr), hdr->ar_pln), + event_mgr.Enqueue(bad_arp, ToAddrVal(reinterpret_cast(ar_spa(hdr)), hdr->ar_pln), ToEthAddrStr(reinterpret_cast(ar_sha(hdr)), hdr->ar_hln), - ToAddrVal(ar_tpa(hdr), hdr->ar_pln), + ToAddrVal(reinterpret_cast(ar_tpa(hdr)), hdr->ar_pln), ToEthAddrStr(reinterpret_cast(ar_tha(hdr)), hdr->ar_hln), zeek::make_intrusive(msg)); } diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index fcf46295af..e4ee83dd83 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -3,8 +3,9 @@ #include "zeek/plugin/Manager.h" #include +#if !defined(_MSC_VER) #include -#include +#endif #include #include #include // for PATH_MAX @@ -56,13 +57,13 @@ void Manager::SearchDynamicPlugins(const std::string& dir) if ( dir.empty() ) return; - if ( dir.find(':') != string::npos ) + if ( dir.find(path_list_separator) != string::npos ) { // Split at ":". std::stringstream s(dir); std::string d; - while ( std::getline(s, d, ':') ) + while ( std::getline(s, d, path_list_separator) ) SearchDynamicPlugins(d); return; @@ -213,30 +214,51 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ } // Load shared libraries. + string dydir = dir + "/lib"; + const char *dyext = "." HOST_ARCHITECTURE DYNAMIC_PLUGIN_SUFFIX; - string dypattern = dir + "/lib/*." + HOST_ARCHITECTURE + DYNAMIC_PLUGIN_SUFFIX; + DBG_LOG(DBG_PLUGINS, " Searching for shared libraries in %s with extension %s", dydir.c_str(), dyext); - DBG_LOG(DBG_PLUGINS, " Searching for shared libraries %s", dypattern.c_str()); + DIR* d = opendir(dydir.c_str()); - glob_t gl; - - if ( glob(dypattern.c_str(), 0, 0, &gl) == 0 ) + if ( ! d ) { - for ( size_t i = 0; i < gl.gl_pathc; i++ ) - { - const char* path = gl.gl_pathv[i]; + DBG_LOG(DBG_PLUGINS, "Cannot open directory %s", dydir.c_str()); + return true; + } - current_plugin = nullptr; - current_dir = dir.c_str(); - current_sopath = path; - void* hdl = dlopen(path, RTLD_NOW | RTLD_GLOBAL); - current_dir = nullptr; - current_sopath = nullptr; + struct dirent *dp; + + while ( (dp = readdir(d)) ) + { + if ( strlen(dp->d_name) >= strlen(dyext) + && zeek::util::streq(dp->d_name + strlen(dp->d_name) - strlen(dyext), dyext) ) + { + string path = dydir + "/" + dp->d_name; + + current_plugin = nullptr; + current_dir = dydir.c_str(); + current_sopath = path.c_str(); +#if defined(_MSC_VER) + void* hdl = LoadLibraryA(path.c_str()); +#else + void* hdl = dlopen(path.c_str(), RTLD_LAZY | RTLD_GLOBAL); +#endif + current_dir = nullptr; + current_sopath = nullptr; if ( ! hdl ) { - const char* err = dlerror(); - errors->push_back(util::fmt("cannot load plugin library %s: %s", path, + const char* err = nullptr; +#if defined(_MSC_VER) + char buf[65535]; + const int flags = FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS; + if ( FormatMessageA(flags, nullptr, GetLastError(), 0, buf, sizeof(buf), nullptr ) ) + err = buf; +#else + err = dlerror(); +#endif + errors->push_back(util::fmt("cannot load plugin library %s: %s", path.c_str(), err ? err : "")); continue; } @@ -244,7 +266,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ if ( ! current_plugin ) { errors->push_back( - util::fmt("load plugin library %s did not instantiate a plugin", path)); + util::fmt("load plugin library %s did not instantiate a plugin", path.c_str())); continue; } @@ -256,10 +278,10 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ plugins_by_path.insert( std::make_pair(util::detail::normalize_path(dir), current_plugin)); - // We execute the pre-script initialization here; this in - // fact could be *during* script initialization if we got - // triggered via @load-plugin. - current_plugin->InitPreScript(); + // We execute the pre-script initialization here; this in + // fact could be *during* script initialization if we got + // triggered via @load-plugin. + current_plugin->InitPreScript(); // Make sure the name the plugin reports is consistent with // what we expect from its magic file. @@ -271,20 +293,21 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ } current_plugin = nullptr; - DBG_LOG(DBG_PLUGINS, " Loaded %s", path); + DBG_LOG(DBG_PLUGINS, " Loaded %s", path.c_str()); } - globfree(&gl); - if ( ! errors->empty() ) return false; } - else + closedir(d); + + if ( current_plugin == nullptr ) { DBG_LOG(DBG_PLUGINS, " No shared library found"); } + // Add the "scripts" and "bif" directories to ZEEKPATH. std::string scripts = dir + "scripts"; @@ -911,32 +934,48 @@ void Manager::HookBroObjDtor(void* obj) const if ( HavePluginForHook(META_HOOK_PRE) ) { args.push_back(HookArgument(obj)); +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif MetaHookPre(HOOK_BRO_OBJ_DTOR, args); +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif } +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif hook_list* l = hooks[HOOK_BRO_OBJ_DTOR]; +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif if ( l ) for ( hook_list::iterator i = l->begin(); i != l->end(); ++i ) { Plugin* p = (*i).second; +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif p->HookBroObjDtor(obj); +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif } if ( HavePluginForHook(META_HOOK_POST) ) +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif MetaHookPost(HOOK_BRO_OBJ_DTOR, args, HookArgument()); +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif } void Manager::HookObjDtor(void* obj) const diff --git a/src/plugin/Manager.h b/src/plugin/Manager.h index c3fcc287fc..e23ef65cc9 100644 --- a/src/plugin/Manager.h +++ b/src/plugin/Manager.h @@ -77,7 +77,7 @@ public: * This must be called only before InitPluginsPreScript(). * * @param dir The directory to search for plugins. Multiple directories - * can be given by splitting them with ':'. + * can be given by separating them with zeek::util::path_list_separator. */ void SearchDynamicPlugins(const std::string& dir); diff --git a/src/plugin/Plugin.cc b/src/plugin/Plugin.cc index 6b5a201564..af005a40f9 100644 --- a/src/plugin/Plugin.cc +++ b/src/plugin/Plugin.cc @@ -383,10 +383,14 @@ void Plugin::RequestEvent(EventHandlerPtr handler) void Plugin::RequestBroObjDtor(Obj* obj) { +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif plugin_mgr->RequestBroObjDtor(obj, this); +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif } void Plugin::RequestObjDtor(Obj* obj) diff --git a/src/plugin/Plugin.h b/src/plugin/Plugin.h index b3879ec9fb..e49d80b461 100644 --- a/src/plugin/Plugin.h +++ b/src/plugin/Plugin.h @@ -25,6 +25,10 @@ struct Field; namespace zeek { +#if defined(_MSC_VER) +#undef VOID +#endif + // Increase this when making incompatible changes to the plugin API. Note // that the constant is never used in C code. It's picked up on by CMake. constexpr int PLUGIN_API_VERSION = 7; @@ -116,17 +120,23 @@ public: // We force this to inline so that the API version gets hardcoded // into the external plugin. (Technically, it's not a "force", just a // strong hint.). The attribute seems generally available. +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif inline Configuration() __attribute__((always_inline)) { bro_version = ZEEK_PLUGIN_ZEEK_VERSION; zeek_version = ZEEK_PLUGIN_ZEEK_VERSION; } +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif Configuration(Configuration&& c) { bro_version = std::move(c.bro_version); @@ -136,10 +146,14 @@ public: description = std::move(c.description); version = std::move(c.version); } +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif Configuration(const Configuration& c) { bro_version = c.bro_version; @@ -149,10 +163,14 @@ public: description = c.description; version = c.version; } +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif Configuration& operator=(Configuration&& c) { bro_version = std::move(c.bro_version); @@ -164,10 +182,14 @@ public: return *this; } +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif Configuration& operator=(const Configuration& c) { bro_version = c.bro_version; @@ -179,12 +201,18 @@ public: return *this; } +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif ~Configuration() { } +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif /** * One can assign ZEEK_PLUGIN_ZEEK_VERSION to this to catch diff --git a/src/script_opt/ZAM/Stmt.cc b/src/script_opt/ZAM/Stmt.cc index 7bf47a8dc0..51e6d0e6e5 100644 --- a/src/script_opt/ZAM/Stmt.cc +++ b/src/script_opt/ZAM/Stmt.cc @@ -384,8 +384,10 @@ const ZAMStmt ZAMCompiler::GenCond(const Expr* e, int& branch_v) // from "ZAM-Conds.h". It really shouldn't worry about indentation mismatches // across included files since those are not indicative of possible // logic errors, but Oh Well. +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wmisleading-indentation" +#endif switch ( e->Tag() ) { #include "ZAM-Conds.h" @@ -393,7 +395,9 @@ const ZAMStmt ZAMCompiler::GenCond(const Expr* e, int& branch_v) default: reporter->InternalError("bad expression type in ZAMCompiler::GenCond"); } +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif // Not reached. } diff --git a/src/supervisor/Supervisor.cc b/src/supervisor/Supervisor.cc index 4dde94d4cf..2ae656b8c9 100644 --- a/src/supervisor/Supervisor.cc +++ b/src/supervisor/Supervisor.cc @@ -41,7 +41,7 @@ extern "C" #include "zeek/zeek-affinity.h" #ifdef DEBUG -#define DBG_STEM(args...) stem->LogDebug(args); +#define DBG_STEM(...) stem->LogDebug(__VA_ARGS__); #else #define DBG_STEM #endif @@ -990,6 +990,7 @@ std::optional Stem::Poll() pfds[pfd_idx++] = {pipe->InFD(), POLLIN, 0}; pfds[pfd_idx++] = {signal_flare->FD(), POLLIN, 0}; +#if !defined(_MSC_VER) for ( const auto& [name, node] : nodes ) { node_pollfd_indices[name] = pfd_idx; @@ -1004,6 +1005,7 @@ std::optional Stem::Poll() else pfds[pfd_idx++] = {-1, POLLIN, 0}; } +#endif // Note: the poll timeout here is for periodically checking if the parent // process died (see below). @@ -1277,10 +1279,14 @@ Supervisor::NodeConfig Supervisor::NodeConfig::FromRecord(const RecordVal* node) for ( auto i = 0u; i < scripts_val->Size(); ++i ) { auto script = scripts_val->StringValAt(i)->ToStdString(); +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif rval.scripts.emplace_back(std::move(script)); +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif } auto env_table_val = node->GetField("env")->AsTableVal(); @@ -1364,10 +1370,14 @@ Supervisor::NodeConfig Supervisor::NodeConfig::FromJSON(std::string_view json) auto& scripts = j["scripts"]; for ( auto it = scripts.Begin(); it != scripts.End(); ++it ) +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif rval.scripts.emplace_back(it->GetString()); +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif auto& env = j["env"]; @@ -1447,10 +1457,14 @@ RecordValPtr Supervisor::NodeConfig::ToRecord() const auto st = rt->GetFieldType("scripts"); auto scripts_val = make_intrusive(std::move(st)); +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif for ( const auto& s : scripts ) +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif scripts_val->Assign(scripts_val->Size(), make_intrusive(s)); rval->AssignField("scripts", std::move(scripts_val)); @@ -1656,10 +1670,14 @@ void SupervisedNode::Init(Options* options) const stl.insert(stl.begin(), config.addl_base_scripts.begin(), config.addl_base_scripts.end()); stl.insert(stl.end(), config.addl_user_scripts.begin(), config.addl_user_scripts.end()); +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif stl.insert(stl.end(), config.scripts.begin(), config.scripts.end()); +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif } RecordValPtr Supervisor::Status(std::string_view node_name) diff --git a/src/supervisor/Supervisor.h b/src/supervisor/Supervisor.h index 93520854c1..cee3a758f2 100644 --- a/src/supervisor/Supervisor.h +++ b/src/supervisor/Supervisor.h @@ -143,8 +143,10 @@ public: */ struct NodeConfig { +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif // This block exists because the default implementations // themselves trigger deprecation warnings for accessing the // "scripts" field. It can go when we remove that deprecation. @@ -154,7 +156,9 @@ public: NodeConfig(NodeConfig&&) = default; ~NodeConfig() = default; NodeConfig& operator=(const NodeConfig&) = default; +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif /** * Create configuration from script-layer record value. diff --git a/src/telemetry/Counter.h b/src/telemetry/Counter.h index 6f5e360a89..afa836999a 100644 --- a/src/telemetry/Counter.h +++ b/src/telemetry/Counter.h @@ -57,7 +57,7 @@ public: /** * @return Whether @c this and @p other refer to the same counter. */ - constexpr bool IsSameAs(IntCounter other) const noexcept { return hdl == other.hdl; } + constexpr bool IsSameAs(const IntCounter& other) const noexcept { return hdl == other.hdl; } private: using Handle = broker::telemetry::int_counter_hdl*; @@ -72,13 +72,13 @@ private: * @return Whether @p lhs and @p rhs refer to the same object. * @note compare their @c value instead to check for equality. */ -constexpr bool operator==(IntCounter lhs, IntCounter rhs) noexcept +constexpr bool operator==(const IntCounter& lhs, const IntCounter& rhs) noexcept { return lhs.IsSameAs(rhs); } /// @relates IntCounter -constexpr bool operator!=(IntCounter lhs, IntCounter rhs) noexcept +constexpr bool operator!=(const IntCounter& lhs, const IntCounter& rhs) noexcept { return ! (lhs == rhs); } @@ -155,7 +155,7 @@ public: /** * @return Whether @c this and @p other refer to the same counter. */ - constexpr bool IsSameAs(DblCounter other) const noexcept { return hdl == other.hdl; } + constexpr bool IsSameAs(const DblCounter& other) const noexcept { return hdl == other.hdl; } private: using Handle = broker::telemetry::dbl_counter_hdl*; @@ -170,13 +170,13 @@ private: * @return Whether @p lhs and @p rhs refer to the same object. * @note compare their @c value instead to check for equality. */ -constexpr bool operator==(DblCounter lhs, DblCounter rhs) noexcept +constexpr bool operator==(const DblCounter& lhs, const DblCounter& rhs) noexcept { return lhs.IsSameAs(rhs); } /// @relates DblCounter -constexpr bool operator!=(DblCounter lhs, DblCounter rhs) noexcept +constexpr bool operator!=(const DblCounter& lhs, const DblCounter& rhs) noexcept { return ! (lhs == rhs); } diff --git a/src/telemetry/Gauge.h b/src/telemetry/Gauge.h index 294f9b1ab1..68cc6ca6e2 100644 --- a/src/telemetry/Gauge.h +++ b/src/telemetry/Gauge.h @@ -73,7 +73,7 @@ public: /** * @return Whether @c this and @p other refer to the same counter. */ - constexpr bool IsSameAs(IntGauge other) const noexcept { return hdl == other.hdl; } + constexpr bool IsSameAs(const IntGauge& other) const noexcept { return hdl == other.hdl; } private: using Handle = broker::telemetry::int_gauge_hdl*; @@ -88,13 +88,13 @@ private: * @return Whether @p lhs and @p rhs refer to the same object. * @note compare their @c value instead to check for equality. */ -constexpr bool operator==(IntGauge lhs, IntGauge rhs) noexcept +constexpr bool operator==(const IntGauge& lhs, const IntGauge& rhs) noexcept { return lhs.IsSameAs(rhs); } /// @relates IntGauge -constexpr bool operator!=(IntGauge lhs, IntGauge rhs) noexcept +constexpr bool operator!=(const IntGauge& lhs, const IntGauge& rhs) noexcept { return ! (lhs == rhs); } @@ -180,7 +180,7 @@ public: /** * @return Whether @c this and @p other refer to the same counter. */ - constexpr bool IsSameAs(DblGauge other) const noexcept { return hdl == other.hdl; } + constexpr bool IsSameAs(const DblGauge& other) const noexcept { return hdl == other.hdl; } private: using Handle = broker::telemetry::dbl_gauge_hdl*; @@ -195,13 +195,13 @@ private: * @return Whether @p lhs and @p rhs refer to the same object. * @note compare their @c value instead to check for equality. */ -constexpr bool operator==(DblGauge lhs, DblGauge rhs) noexcept +constexpr bool operator==(const DblGauge& lhs, const DblGauge& rhs) noexcept { return lhs.IsSameAs(rhs); } /// @relates DblGauge -constexpr bool operator!=(DblGauge lhs, DblGauge rhs) noexcept +constexpr bool operator!=(const DblGauge& lhs, const DblGauge& rhs) noexcept { return ! (lhs == rhs); } diff --git a/src/telemetry/Histogram.h b/src/telemetry/Histogram.h index 069c2fc38d..0ccfc823dd 100644 --- a/src/telemetry/Histogram.h +++ b/src/telemetry/Histogram.h @@ -60,7 +60,7 @@ public: /** * @return Whether @c this and @p other refer to the same histogram. */ - constexpr bool IsSameAs(IntHistogram other) const noexcept { return hdl == other.hdl; } + constexpr bool IsSameAs(const IntHistogram& other) const noexcept { return hdl == other.hdl; } private: using Handle = broker::telemetry::int_histogram_hdl*; @@ -74,13 +74,13 @@ private: * Checks whether two @ref IntHistogram handles are identical. * @return Whether @p lhs and @p rhs refer to the same object. */ -constexpr bool operator==(IntHistogram lhs, IntHistogram rhs) noexcept +constexpr bool operator==(const IntHistogram& lhs, const IntHistogram& rhs) noexcept { return lhs.IsSameAs(rhs); } /// @relates IntHistogram -constexpr bool operator!=(IntHistogram lhs, IntHistogram rhs) noexcept +constexpr bool operator!=(const IntHistogram& lhs, const IntHistogram& rhs) noexcept { return ! (lhs == rhs); } @@ -165,7 +165,7 @@ public: /** * @return Whether @c this and @p other refer to the same histogram. */ - constexpr bool IsSameAs(DblHistogram other) const noexcept { return hdl == other.hdl; } + constexpr bool IsSameAs(const DblHistogram& other) const noexcept { return hdl == other.hdl; } private: using Handle = broker::telemetry::dbl_histogram_hdl*; @@ -179,13 +179,13 @@ private: * Checks whether two @ref DblHistogram handles are identical. * @return Whether @p lhs and @p rhs refer to the same object. */ -constexpr bool operator==(DblHistogram lhs, DblHistogram rhs) noexcept +constexpr bool operator==(const DblHistogram& lhs, const DblHistogram& rhs) noexcept { return lhs.IsSameAs(rhs); } /// @relates DblHistogram -constexpr bool operator!=(DblHistogram lhs, DblHistogram rhs) noexcept +constexpr bool operator!=(const DblHistogram& lhs, const DblHistogram& rhs) noexcept { return ! (lhs == rhs); } diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc index acb1b0087d..457906a152 100644 --- a/src/threading/BasicThread.cc +++ b/src/threading/BasicThread.cc @@ -48,9 +48,9 @@ void BasicThread::SetName(const char* arg_name) void BasicThread::SetOSName(const char* arg_name) { - static_assert(std::is_same::value, - "libstdc++ doesn't use pthread_t"); - util::detail::set_thread_name(arg_name, thread.native_handle()); + // Do it only if libc++ supports pthread_t. + if constexpr(std::is_same::value) + zeek::util::detail::set_thread_name(arg_name, reinterpret_cast(thread.native_handle())); } const char* BasicThread::Fmt(const char* format, ...) @@ -172,10 +172,9 @@ void BasicThread::Done() void* BasicThread::launcher(void* arg) { - static_assert(std::is_same::value, - "libstdc++ doesn't use pthread_t"); BasicThread* thread = (BasicThread*)arg; +#ifndef _MSC_VER // Block signals in thread. We handle signals only in the main // process. sigset_t mask_set; @@ -190,6 +189,7 @@ void* BasicThread::launcher(void* arg) sigdelset(&mask_set, SIGBUS); int res = pthread_sigmask(SIG_BLOCK, &mask_set, 0); assert(res == 0); +#endif // Run thread's main function. thread->Run(); diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h index 9d78e8dd05..2afddab509 100644 --- a/src/threading/BasicThread.h +++ b/src/threading/BasicThread.h @@ -3,6 +3,7 @@ #include "zeek/zeek-config.h" +#include #include #include #include diff --git a/src/util.cc b/src/util.cc index 5530488e19..9f61daeb80 100644 --- a/src/util.cc +++ b/src/util.cc @@ -42,6 +42,9 @@ #include #include #include +#include +#include +#include #include "zeek/3rdparty/ConvertUTF.h" #include "zeek/3rdparty/doctest.h" @@ -51,6 +54,7 @@ #include "zeek/Obj.h" #include "zeek/Reporter.h" #include "zeek/RunState.h" +#include "zeek/ScannedFile.h" #include "zeek/Val.h" #include "zeek/digest.h" #include "zeek/input.h" @@ -67,6 +71,7 @@ static bool can_read(const string& path) } static string zeek_path_value; +const string zeek_path_list_separator(1, path_list_separator); namespace zeek::util { @@ -254,6 +259,16 @@ const char* fmt_access_time(double t) time_t time = (time_t)t; struct tm ts; + if (!time) + { + // Use wall clock. + struct timeval tv = { 0 }; + if (gettimeofday(&tv, 0) < 0) + reporter->InternalError("unable to gettimeofday"); + else + time = tv.tv_sec; + } + if ( ! localtime_r(&time, &ts) ) { reporter->InternalError("unable to get time"); @@ -432,6 +447,7 @@ void init_random_seed(const char* read_file, const char* write_file, bool use_em pos += sizeof(struct timeval) / sizeof(uint32_t); // use urandom. For reasons see e.g. http://www.2uo.de/myths-about-urandom/ +#ifndef _MSC_VER #if defined(O_NONBLOCK) int fd = open("/dev/urandom", O_RDONLY | O_NONBLOCK); #elif defined(O_NDELAY) @@ -454,6 +470,12 @@ void init_random_seed(const char* read_file, const char* write_file, bool use_em errno = 0; } #endif + // C++ random device implementation in MSVC is sufficient for this purpose. + thread_local std::mt19937 gen(std::random_device{}()); + while ( pos < zeek::detail::KeyedHash::SEED_INIT_SIZE ) { + buf[pos++] = (uint32_t)gen(); + } +#endif if ( pos < zeek::detail::KeyedHash::SEED_INIT_SIZE ) reporter->FatalError("Could not read enough random data. Wanted %d, got %zu", @@ -554,7 +576,7 @@ void add_to_zeek_path(const string& dir) // Make sure path is initialized. zeek_path(); - zeek_path_value += string(":") + dir; + zeek_path_value += string(zeek_path_list_separator) + dir; } FILE* open_package(string& path, const string& mode) @@ -649,6 +671,12 @@ TEST_CASE("util normalize_path") string normalize_path(std::string_view path) { +#ifdef _MSC_VER + if (0 == path.compare(zeek::detail::ScannedFile::canonical_stdin_path)) { + return string(path); + } + return std::filesystem::canonical(path).string(); +#else if ( path.find("/.") == std::string_view::npos && path.find("//") == std::string_view::npos ) { // no need to normalize anything @@ -713,13 +741,14 @@ string normalize_path(std::string_view path) new_path.erase(new_path.size() - 1); return new_path; +#endif } string without_zeekpath_component(std::string_view path) { string rval = normalize_path(path); - const auto paths = tokenize_string(zeek_path(), ':'); + const auto paths = tokenize_string(zeek_path(), path_list_separator); for ( size_t i = 0; i < paths.size(); ++i ) { @@ -746,12 +775,13 @@ std::string get_exe_path(const std::string& invocation) { if ( invocation.empty() ) return ""; + std::filesystem::path invocation_path(invocation); - if ( invocation[0] == '/' || invocation[0] == '~' ) + if ( invocation_path.is_absolute() || invocation_path.root_directory() == "~" ) // Absolute path return invocation; - if ( invocation.find('/') != std::string::npos ) + if ( invocation_path.is_relative() ) { // Relative path char cwd[PATH_MAX]; @@ -762,7 +792,7 @@ std::string get_exe_path(const std::string& invocation) exit(1); } - return std::string(cwd) + "/" + invocation; + return (std::filesystem::path(cwd) / invocation_path).string(); } auto path = getenv("PATH"); @@ -1576,7 +1606,7 @@ const char* fmt_bytes(const char* data, int len) for ( int i = 0; i < len && p - buf < int(sizeof(buf)); ++i ) { - if ( isprint(data[i]) ) + if ( isprint((unsigned char)(data[i])) ) *p++ = data[i]; else p += snprintf(p, sizeof(buf) - (p - buf), "\\x%02x", (unsigned char)data[i]); @@ -1756,7 +1786,7 @@ string zeek_prefixes() for ( const auto& prefix : zeek::detail::zeek_script_prefixes ) { if ( ! rval.empty() ) - rval.append(":"); + rval.append(zeek_path_list_separator); rval.append(prefix); } @@ -1937,8 +1967,10 @@ static string find_file_in_path(const string& filename, const string& path, if ( filename.empty() ) return string(); + std::filesystem::path filepath(filename); + // If file name is an absolute path, searching within *path* is pointless. - if ( filename[0] == '/' ) + if ( filepath.is_absolute() ) { if ( can_read(filename) ) return filename; @@ -1946,7 +1978,7 @@ static string find_file_in_path(const string& filename, const string& path, return string(); } - string abs_path = path + '/' + filename; + auto abs_path = (std::filesystem::path(path) / filepath).string(); if ( ! opt_ext.empty() ) { @@ -1968,7 +2000,7 @@ static string find_file_in_path(const string& filename, const string& path, string find_file(const string& filename, const string& path_set, const string& opt_ext) { vector paths; - tokenize_string(path_set, ":", &paths); + tokenize_string(path_set, zeek_path_list_separator, &paths); vector ext; if ( ! opt_ext.empty() ) @@ -1988,7 +2020,7 @@ string find_file(const string& filename, const string& path_set, const string& o string find_script_file(const string& filename, const string& path_set) { vector paths; - tokenize_string(path_set, ":", &paths); + tokenize_string(path_set, zeek_path_list_separator, &paths); vector ext = {".zeek"}; diff --git a/src/util.h b/src/util.h index bdea606a8b..5acb449df7 100644 --- a/src/util.h +++ b/src/util.h @@ -15,6 +15,8 @@ #endif #include +#include + #include #include #include @@ -38,19 +40,23 @@ #endif #endif +#ifdef _MSC_VER +#include +#endif + #ifdef DEBUG #include #define ASSERT(x) assert(x) -#define DEBUG_MSG(x...) fprintf(stderr, x) +#define DEBUG_MSG(...) fprintf(stderr, __VA_ARGS__) #define DEBUG_fputs fputs #else #define ASSERT(x) -#define DEBUG_MSG(x...) -#define DEBUG_fputs(x...) +#define DEBUG_MSG(...) +#define DEBUG_fputs(...) #endif @@ -60,9 +66,10 @@ extern HeapLeakChecker* heap_checker; #endif -#include +#include #ifdef HAVE_LINUX +#include #include #endif @@ -70,12 +77,22 @@ extern HeapLeakChecker* heap_checker; #include #endif +#if defined(_MSC_VER) +const char path_list_separator = ';'; +#else +const char path_list_separator = ':'; +#endif + extern "C" { #include "zeek/3rdparty/modp_numtoa.h" } +#if defined(_MSC_VER) +#include +#else #include "zeek/3rdparty/ghc/filesystem.hpp" +#endif using zeek_int_t = int64_t; using zeek_uint_t = uint64_t; @@ -96,8 +113,12 @@ class ODesc; class RecordVal; // Expose ghc::filesystem as zeek::filesystem until we can -// switch to std::filesystem. +// switch to std::filesystem on all platforms. +#if defined(_MSC_VER) +namespace filesystem = std::filesystem; +#else namespace filesystem = ghc::filesystem; +#endif namespace util { diff --git a/src/zeek b/src/zeek deleted file mode 120000 index 945c9b46d6..0000000000 --- a/src/zeek +++ /dev/null @@ -1 +0,0 @@ -. \ No newline at end of file diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index 44bae3527d..676cc7a49d 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -49,6 +49,7 @@ #include "zeek/Traverse.h" #include "zeek/Trigger.h" #include "zeek/Var.h" + #include "zeek/analyzer/Manager.h" #include "zeek/binpac_zeek.h" #include "zeek/broker/Manager.h" @@ -63,6 +64,7 @@ #include "zeek/plugin/Manager.h" #include "zeek/script_opt/ScriptOpt.h" #include "zeek/session/Manager.h" +#include "zeek/script_opt/ScriptOpt.h" #include "zeek/supervisor/Supervisor.h" #include "zeek/telemetry/Manager.h" #include "zeek/threading/Manager.h" @@ -200,7 +202,11 @@ std::shared_ptr zeek::detail::sample_logger; zeek::detail::FragmentManager* zeek::detail::fragment_mgr = nullptr; int signal_val = 0; +#ifdef _MSC_VER +char version[] = VERSION; +#else extern char version[]; +#endif const char* zeek::detail::command_line_policy = nullptr; vector zeek::detail::params; set requested_plugins; @@ -236,6 +242,9 @@ char** zeek::detail::zeek_argv; namespace zeek { +// Define zeek version explicitly for MSVC + + const char* zeek_version() { #ifdef DEBUG diff --git a/src/zeek.bif b/src/zeek.bif index 11960435c2..e052392793 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -2226,7 +2226,7 @@ function is_local_interface%(ip: addr%) : bool %{ if ( ip->AsAddr().IsLoopback() ) return zeek::val_mgr->True(); - +#ifndef _MSC_VER list addrs; char host[MAXHOSTNAMELEN]; @@ -2259,7 +2259,7 @@ function is_local_interface%(ip: addr%) : bool if ( *it == ip->AsAddr() ) return zeek::val_mgr->True(); } - +#endif return zeek::val_mgr->False(); %} diff --git a/src/zeekygen/Target.cc b/src/zeekygen/Target.cc index 5236b8b070..eba9c81c83 100644 --- a/src/zeekygen/Target.cc +++ b/src/zeekygen/Target.cc @@ -2,7 +2,9 @@ #include "zeek/zeekygen/Target.h" +#ifndef _MSC_VER #include +#endif #include #include #include @@ -490,6 +492,8 @@ vector dir_contents_recursive(string dir) scan_path[0] = dir_copy; scan_path[1] = NULL; + // Zeekygen isn't supported in Windows, due to missing FTS library. +#ifndef _MSC_VER FTS* fts = fts_open(scan_path, FTS_NOCHDIR, 0); if ( ! fts ) @@ -516,6 +520,7 @@ vector dir_contents_recursive(string dir) delete[] scan_path; delete[] dir_copy; +#endif return rval; } diff --git a/zeek-config.h.in b/zeek-config.h.in index 5e68cff57a..d8601f7e3f 100644 --- a/zeek-config.h.in +++ b/zeek-config.h.in @@ -184,6 +184,7 @@ #define DLT_NFLOG @DLT_NFLOG@ #endif +#ifndef _MSC_VER /* IPv6 Next Header values defined by RFC 3542 */ #cmakedefine HAVE_IPPROTO_HOPOPTS #ifndef HAVE_IPPROTO_HOPOPTS @@ -225,7 +226,7 @@ #ifndef HAVE_IPPROTO_DSTOPTS #define IPPROTO_DSTOPTS 60 #endif - +#endif /* IPv6 options structure defined by RFC 3542 */ #cmakedefine HAVE_IP6_OPT diff --git a/zeek-path-dev.in b/zeek-path-dev.in index 6dcdc71c7c..0c5fa4fbd2 100755 --- a/zeek-path-dev.in +++ b/zeek-path-dev.in @@ -10,4 +10,4 @@ # ZEEKPATH=`./zeek-path-dev` ./src/zeek # -echo .:${PROJECT_SOURCE_DIR}/scripts:${PROJECT_SOURCE_DIR}/scripts/policy:${PROJECT_SOURCE_DIR}/scripts/site:${PROJECT_BINARY_DIR}/scripts:${PROJECT_BINARY_DIR}/scripts/builtin-plugins +echo .:${cmake_source_dir}/scripts:${cmake_source_dir}/scripts/policy:${cmake_source_dir}/scripts/site:${cmake_binary_dir}/scripts:${cmake_binary_dir}/scripts/builtin-plugins From 0dbfd9b04e2f0b94a342ff6b03a1e682487d5141 Mon Sep 17 00:00:00 2001 From: Tomer Lev Date: Wed, 21 Sep 2022 14:25:14 +0300 Subject: [PATCH 03/71] Building supervisor for window --- src/Val.h | 14 +++++++++----- src/supervisor/Supervisor.cc | 14 ++++++-------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/Val.h b/src/Val.h index ab1eb46036..467c1361f4 100644 --- a/src/Val.h +++ b/src/Val.h @@ -1124,15 +1124,19 @@ public: AddedField(field); } - void Assign(int field, int new_val) + // For int types, we provide both [u]int32_t and [u]int64_t versions + // for convenience, since sometimes the caller has one rather + // than the other. + void Assign(int field, int32_t new_val) + { + (*record_val)[field] = ZVal(zeek_int_t(new_val)); + AddedField(field); + } + void Assign(int field, int64_t new_val) { (*record_val)[field] = ZVal(zeek_int_t(new_val)); AddedField(field); } - - // For unsigned, we provide both uint32_t and uint64_t versions - // for convenience, since sometimes the caller has one rather - // than the other. void Assign(int field, uint32_t new_val) { (*record_val)[field] = ZVal(zeek_uint_t(new_val)); diff --git a/src/supervisor/Supervisor.cc b/src/supervisor/Supervisor.cc index 2ae656b8c9..a2dc6a66ba 100644 --- a/src/supervisor/Supervisor.cc +++ b/src/supervisor/Supervisor.cc @@ -987,25 +987,23 @@ std::optional Stem::Poll() const auto total_fd_count = fixed_fd_count + (nodes.size() * 2); auto pfds = std::make_unique(total_fd_count); int pfd_idx = 0; - pfds[pfd_idx++] = {pipe->InFD(), POLLIN, 0}; - pfds[pfd_idx++] = {signal_flare->FD(), POLLIN, 0}; + pfds[pfd_idx++] = {static_cast(pipe->InFD()), POLLIN, 0}; + pfds[pfd_idx++] = {static_cast(signal_flare->FD()), POLLIN, 0}; -#if !defined(_MSC_VER) for ( const auto& [name, node] : nodes ) { node_pollfd_indices[name] = pfd_idx; if ( node.stdout_pipe.pipe ) - pfds[pfd_idx++] = {node.stdout_pipe.pipe->ReadFD(), POLLIN, 0}; + pfds[pfd_idx++] = {static_cast(node.stdout_pipe.pipe->ReadFD()), POLLIN, 0}; else - pfds[pfd_idx++] = {-1, POLLIN, 0}; + pfds[pfd_idx++] = {static_cast(-1), POLLIN, 0}; if ( node.stderr_pipe.pipe ) - pfds[pfd_idx++] = {node.stderr_pipe.pipe->ReadFD(), POLLIN, 0}; + pfds[pfd_idx++] = {static_cast(node.stderr_pipe.pipe->ReadFD()), POLLIN, 0}; else - pfds[pfd_idx++] = {-1, POLLIN, 0}; + pfds[pfd_idx++] = {static_cast(-1), POLLIN, 0}; } -#endif // Note: the poll timeout here is for periodically checking if the parent // process died (see below). From fde3a6b73f416ecae47ff7b91046c470e118869c Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Tue, 20 Jul 2021 18:50:21 +0300 Subject: [PATCH 04/71] Added kqueue as conan package --- CMakeLists.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index f7c147d488..6594e5380a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,7 @@ if ( MSVC ) openssl/1.1.1i winflexbison/2.5.24 npcap-wpcap/1.31 + kqueue/2.3.2 GENERATORS cmake_find_package) # Install packages from conanfile @@ -54,6 +55,12 @@ if ( MSVC ) find_package(ZLIB) set(ZLIB_LIBRARY ${ZLIB_LIBRARIES}) + # Set Kqueue to point at the right variable. + find_package(kqueue) + set(LIBKQUEUE_ROOT_DIR "${kqueue_INCLUDES}/../") + set(LIBKQUEUE_INCLUDE_DIRS ${kqueue_INCLUDES}) + set(LIBKQUEUE_LIBRARIES ${kqueue_LIBS}) + add_subdirectory(auxil/libunistd) set(UNISTD_INCLUDES ${CMAKE_SOURCE_DIR}/auxil/libunistd/unistd ${CMAKE_SOURCE_DIR}/auxil/libunistd/regex) include_directories(BEFORE ${UNISTD_INCLUDES}) From 235ec2565f5ef73a9f37cf9e22d42b6bdad10321 Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Tue, 20 Jul 2021 18:51:29 +0300 Subject: [PATCH 05/71] Updated build script to keep build output --- ci/build.ps1 | 18 +++++++++++++++--- ci/windows/Dockerfile | 3 ++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/ci/build.ps1 b/ci/build.ps1 index 9b8d397c59..5b18a6a2df 100644 --- a/ci/build.ps1 +++ b/ci/build.ps1 @@ -5,7 +5,9 @@ param( ) $SourceDirectory = (Convert-Path "$PSScriptRoot/../").Replace("\", "/") -$WorkingDirectory = $pwd.Path +$WorkingDirectory = "$pwd" +$BuildOutputDirectory = "$SourceDirectory/out/build/x64/$BuildType" +$InstallDirectory = "$SourceDirectory/out/install/x64/$BuildType" $commands = @() if (!(Get-Command cl)) { @@ -18,7 +20,7 @@ cmake.exe -S "$SourceDirectory" -B "$WorkingDirectory" -DCMAKE_BUILD_TYPE:STRING=$BuildType - -DCMAKE_INSTALL_PREFIX:PATH="$SourceDirectory/out/install/$BuildType" + -DCMAKE_INSTALL_PREFIX:PATH="$InstallDirectory" -DDISABLE_PYTHON_BINDINGS=1 ` 2>&1 "@.Replace("`r`n", "") @@ -30,4 +32,14 @@ cmake.exe --build $WorkingDirectory --config $BuildType $commands += @" cmake.exe --install $WorkingDirectory "@ -cmd /c ($commands -join " && ") \ No newline at end of file +cmd /c ($commands -join " && ") + +Write-Host "Copying build output to $BuildOutputDirectory..." +mkdir $BuildOutputDirectory -Force | Out-Null +Get-ChildItem $WorkingDirectory -Recurse -Attributes !ReparsePoint | foreach { + $path = $_.FullName.Replace("$pwd", $BuildOutputDirectory) + $parent = Split-Path -Parent $path + mkdir $parent -Force | Out-Null + Copy-Item $_.FullName -Destination $path -Force +} +Write-Host "Done." \ No newline at end of file diff --git a/ci/windows/Dockerfile b/ci/windows/Dockerfile index c6ce894f74..c6bfe16fc8 100644 --- a/ci/windows/Dockerfile +++ b/ci/windows/Dockerfile @@ -42,6 +42,7 @@ RUN choco install python -y # Can't install a proper msys2 because it caused console hanging in the container during docker build. RUN choco install msysgit -y RUN choco install sed -y +RUN choco install powershell-core -y # Set working environment. RUN setx /M PATH "%PATH%;C:\\Program Files\\Git\\bin" @@ -53,4 +54,4 @@ ADD default C:\Users\ContainerAdministrator\.conan\profiles\default # Define the entry point for the docker container. # This entry point starts the developer command prompt and launches the PowerShell shell. -ENTRYPOINT ["C:\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Unrestricted"] \ No newline at end of file +ENTRYPOINT ["C:\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat", "&&", "pwsh.exe", "-NoLogo", "-ExecutionPolicy", "Unrestricted"] \ No newline at end of file From f0dac4405e95229648ef0b42835c3ed4e978bff2 Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Wed, 21 Jul 2021 13:32:47 +0300 Subject: [PATCH 06/71] Forced creation of symlink in case it gets corrupted by git --- src/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index af63e54fd3..7b06ca2021 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,6 +5,11 @@ include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}/include ) +# Force creation of includes symlink +execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink + ".." + "${CMAKE_CURRENT_SOURCE_DIR}/include/zeek") + # Allows header file inclusion via zeek/ within the build tree execute_process(COMMAND "${CMAKE_COMMAND}" -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/include") From 7b37194a483ac47920631f0da6c671e1f53fe24a Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Mon, 26 Jul 2021 11:26:33 +0000 Subject: [PATCH 07/71] Zeek Conan Package --- src/CMakeLists.txt | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7b06ca2021..4b86fcdf10 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -537,30 +537,39 @@ if (ZEEK_STANDALONE) set(WINDOWS_EXPORT_ALL_SYMBOLS ON) endif () - install(TARGETS zeek RUNTIME DESTINATION bin ARCHIVE DESTINATION lib) + install(TARGETS zeek RUNTIME DESTINATION bin) set(BRO_EXE zeek CACHE STRING "Zeek executable binary" FORCE) set(BRO_EXE_PATH ${CMAKE_CURRENT_BINARY_DIR}/zeek CACHE STRING "Path to Zeek executable binary" FORCE) +endif() -else() - add_library(zeek STATIC $ ${zeek_HEADERS}) +if (NOT ZEEK_STANDALONE OR CONAN_EXPORTED) + add_library(libzeek STATIC $ + ${zeek_HEADERS} + ${bro_SUBDIR_LIBS} + ${bro_PLUGIN_LIBS}) - target_link_libraries(zeek PUBLIC ${zeekdeps} + target_link_libraries(libzeek PUBLIC ${zeekdeps} ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS} ${bro_SUBDIR_LIBS} ${bro_PLUGIN_LIBS}) - target_include_directories(zeek PUBLIC + target_include_directories(libzeek PUBLIC ${CMAKE_SOURCE_DIR}/zeek/src ${CMAKE_SOURCE_DIR}/zeek/src/include ${CMAKE_BINARY_DIR} ${CMAKE_BINARY_DIR}/zeek/src - ${CMAKE_BINARY_DIR}/zeek/src/include - ${CMAKE_SOURCE_DIR}/zeek/src/windows/usr.include) + ${CMAKE_BINARY_DIR}/zeek/src/include) + + if ( WIN32 ) + target_include_directories(libzeek PUBLIC ${CMAKE_SOURCE_DIR}/zeek/src/windows/usr.include) + endif() + + install(TARGETS libzeek LIBRARY DESTINATION lib) endif() if ( NOT WIN32 ) @@ -621,7 +630,16 @@ install(CODE " # Make sure to escape a bunch of special characters in the path before trying to use it as a # regular expression below. -string(REGEX REPLACE "([][+.*()^])" "\\\\\\1" escaped_path "${CMAKE_CURRENT_SOURCE_DIR}/include/*") +string(REGEX REPLACE "([][+.*()^])" "\\\\\\1" escaped_include_path "${CMAKE_CURRENT_SOURCE_DIR}/include/*") +string(REGEX REPLACE "([][+.*()^])" "\\\\\\1" escaped_windows_path "${CMAKE_CURRENT_SOURCE_DIR}/windows/*") + +if (WIN32) + install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/windows/usr.include/ + DESTINATION include/ + FILES_MATCHING + PATTERN "*.h" + ) +endif() install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ DESTINATION include/zeek @@ -630,7 +648,9 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ PATTERN "*.pac" PATTERN "3rdparty/*" EXCLUDE # The "zeek -> ." symlink isn't needed in the install-tree - REGEX "^${escaped_path}$" EXCLUDE + REGEX "${escaped_include_path}$" EXCLUDE + # Windows headers are already installed if needed + REGEX "${escaped_windows_path}$" EXCLUDE # FILES_MATCHING creates empty directories: # https://gitlab.kitware.com/cmake/cmake/-/issues/17122 From fbf5b68d6f901aa5bac19725a45cd643c134969b Mon Sep 17 00:00:00 2001 From: Guy Magen Date: Thu, 9 Sep 2021 10:23:44 +0000 Subject: [PATCH 08/71] Fixed ftp bug Fixed ftp bug --- src/util.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/util.cc b/src/util.cc index 9f61daeb80..a16728370d 100644 --- a/src/util.cc +++ b/src/util.cc @@ -675,7 +675,15 @@ string normalize_path(std::string_view path) if (0 == path.compare(zeek::detail::ScannedFile::canonical_stdin_path)) { return string(path); } - return std::filesystem::canonical(path).string(); + string stringPath = string(path); + + // "//" interferes with std::weakly_canonical + if (stringPath._Starts_with("//")) + { + stringPath.erase(0, 2); + } + + return std::filesystem::weakly_canonical(path).string(); #else if ( path.find("/.") == std::string_view::npos && path.find("//") == std::string_view::npos ) { From c3b9756576ee13c2131138bc7dd78bb1acc7dda4 Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Sun, 12 Sep 2021 12:19:46 +0000 Subject: [PATCH 09/71] Zeek Statistics - Added statistics hook - Fixed rlimit usage - Removed POpen for windows implementation layer - Completed statistics plugin --- scripts/base/init-bare.zeek | 11 ++++ src/CMakeLists.txt | 1 + src/statistics/CMakeLists.txt | 9 ++++ src/statistics/Plugin.cc | 54 +++++++++++++++++++ src/statistics/Plugin.h | 29 ++++++++++ src/statistics/statistics.bif | 29 ++++++++++ .../canonified_loaded_scripts.log | 1 + .../canonified_loaded_scripts.log | 1 + 8 files changed, 135 insertions(+) create mode 100644 src/statistics/CMakeLists.txt create mode 100644 src/statistics/Plugin.cc create mode 100644 src/statistics/Plugin.h create mode 100644 src/statistics/statistics.bif diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 519db3922d..f6f87769df 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -792,6 +792,17 @@ type ReporterStats: record { weirds_by_type: table[string] of count; }; +## Statistics about how many times each event name is queued. +## +## .. zeek:see:: Statistics::get_and_reset_event_name_statistics +type EventNameCounter: record { + ## Name of the zeek event. + name: string &log; + ## Times it was queued, as captured by event hook. + times_queued: count &log; +} &log; +type EventNameStats: vector of EventNameCounter; + ## Table type used to map variable names to their memory allocation. ## ## .. todo:: We need this type definition only for declaring builtin functions diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4b86fcdf10..9fd5d2c743 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -192,6 +192,7 @@ add_subdirectory(file_analysis) add_subdirectory(input) add_subdirectory(iosource) add_subdirectory(logging) +add_subdirectory(statistics) add_subdirectory(probabilistic) add_subdirectory(session) diff --git a/src/statistics/CMakeLists.txt b/src/statistics/CMakeLists.txt new file mode 100644 index 0000000000..11538b5cf0 --- /dev/null +++ b/src/statistics/CMakeLists.txt @@ -0,0 +1,9 @@ + +include(ZeekPlugin) + +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) + +zeek_plugin_begin(Zeek Statistics) +zeek_plugin_cc(Plugin.cc) +bif_target(statistics.bif) +zeek_plugin_end() diff --git a/src/statistics/Plugin.cc b/src/statistics/Plugin.cc new file mode 100644 index 0000000000..18cb71645c --- /dev/null +++ b/src/statistics/Plugin.cc @@ -0,0 +1,54 @@ + +#include + +#include "Plugin.h" + +#include "zeek/Func.h" +#include "zeek/Event.h" +#include "zeek/Conn.h" +#include "zeek/Desc.h" +#include "zeek/threading/Formatter.h" +#include "zeek/RunState.h" + +#include "statistics.bif.h" + +namespace zeek::plugin::statistics { Plugin plugin; } + +using namespace zeek::plugin::statistics; + +zeek::plugin::Configuration Plugin::Configure() + { + zeek::plugin::Configuration config; + config.name = "Statistics"; + config.description = "Statistics module"; + config.version.major = 1; + config.version.minor = 0; + config.version.patch = 0; + return config; + } + +bool Plugin::HookQueueEvent(zeek::Event* event) + { + const char* name = event->Handler()->Name(); + + std::lock_guard scopedLock(m_lock); + if (m_eventNameCounters.find(name) == m_eventNameCounters.end()) + { + m_eventNameCounters[name] = 0; + } + m_eventNameCounters[name]++; + return false; + } + +std::unordered_map Plugin::GetAndResetEventStatistics() + { + std::lock_guard scopedLock(m_lock); + std::unordered_map result(m_eventNameCounters); + m_eventNameCounters.clear(); + return result; + } + +void Plugin::StartEventNamesStatisticsMonitor() + { + EnableHook(zeek::plugin::HOOK_QUEUE_EVENT); + } diff --git a/src/statistics/Plugin.h b/src/statistics/Plugin.h new file mode 100644 index 0000000000..0c396d8b07 --- /dev/null +++ b/src/statistics/Plugin.h @@ -0,0 +1,29 @@ + +#pragma once + +#include +#include +#include +#include "zeek/plugin/Plugin.h" + +namespace zeek::plugin::statistics { + +class Plugin : public zeek::plugin::Plugin +{ +protected: + bool HookQueueEvent(zeek::Event* event) override; + + zeek::plugin::Configuration Configure() override; + +public: + std::unordered_map GetAndResetEventStatistics(); + void StartEventNamesStatisticsMonitor(); + +private: + std::unordered_map m_eventNameCounters; + std::mutex m_lock; + +}; + +extern Plugin plugin; +} diff --git a/src/statistics/statistics.bif b/src/statistics/statistics.bif new file mode 100644 index 0000000000..7f2f14c9a2 --- /dev/null +++ b/src/statistics/statistics.bif @@ -0,0 +1,29 @@ + +%%{ +#include "zeek/statistics/Plugin.h" +%%} + +function Statistics::get_and_reset_event_name_statistics%(%): EventNameStats + %{ + auto rval = zeek::make_intrusive(zeek::id::find_type("EventNameStats")); + auto stats = zeek::plugin::statistics::plugin.GetAndResetEventStatistics(); + const auto& recordType = zeek::id::find_type("EventNameCounter"); + + auto i = 0; + for (auto& eventCounter : stats) + { + auto eventStatRecord = zeek::make_intrusive(recordType); + eventStatRecord->Assign(0, zeek::make_intrusive(eventCounter.first)); + eventStatRecord->Assign(1, zeek::val_mgr->Count(eventCounter.second)); + rval->Assign(i, std::move(eventStatRecord)); + ++i; + } + + return rval; + %} + +function Statistics::start_event_name_statistics_monitor%(%): any + %{ + zeek::plugin::statistics::plugin.StartEventNamesStatisticsMonitor(); + return nullptr; + %} \ No newline at end of file diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 429bc44f5a..cda12d3d6a 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -125,6 +125,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/telemetry.bif.zeek build/scripts/base/bif/zeekygen.bif.zeek build/scripts/base/bif/pcap.bif.zeek + build/scripts/base/bif/statistics.bif.zeek build/scripts/base/bif/bloom-filter.bif.zeek build/scripts/base/bif/cardinality-counter.bif.zeek build/scripts/base/bif/top-k.bif.zeek diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 75cc1c72d4..66c433fb47 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -125,6 +125,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/telemetry.bif.zeek build/scripts/base/bif/zeekygen.bif.zeek build/scripts/base/bif/pcap.bif.zeek + build/scripts/base/bif/statistics.bif.zeek build/scripts/base/bif/bloom-filter.bif.zeek build/scripts/base/bif/cardinality-counter.bif.zeek build/scripts/base/bif/top-k.bif.zeek From eb08b696bfdaa9dcb3ea0b3e2b2327352219b3f8 Mon Sep 17 00:00:00 2001 From: Omer Horovitz Date: Mon, 13 Dec 2021 08:43:53 +0000 Subject: [PATCH 10/71] adding StubSyslog adding StubSyslog ifdef USE_STUB_SYSLOG --- zeek-config.h.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zeek-config.h.in b/zeek-config.h.in index d8601f7e3f..032b399ba3 100644 --- a/zeek-config.h.in +++ b/zeek-config.h.in @@ -108,6 +108,9 @@ /* should we declare syslog() and openlog() */ #cmakedefine SYSLOG_INT +/* should we use stub syslog() and openlog() */ +#cmakedefine USE_STUB_SYSLOG + /* Define if you have */ #cmakedefine HAVE_SYS_TIME_H From 3f349b8a373188cdce7b6f1b984174dcb2b46f89 Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Thu, 16 Dec 2021 08:22:19 +0000 Subject: [PATCH 11/71] Optimize initial memory consumption --- CMakeLists.txt | 4 ++++ src/CMakeLists.txt | 6 ++++-- src/Val.cc | 21 ++++++++++----------- src/Val.h | 17 +++++++++++------ src/input/readers/CMakeLists.txt | 4 +++- src/logging/writers/CMakeLists.txt | 4 +++- src/supervisor/Supervisor.h | 2 ++ src/util.cc | 3 +++ src/zeek-setup.cc | 7 ++++++- zeek-config.h.in | 3 +++ 10 files changed, 49 insertions(+), 22 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6594e5380a..b107ba9118 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -180,6 +180,10 @@ if ( NOT ZEEK_LOG_DIR ) set(ZEEK_LOG_DIR ${ZEEK_ROOT_DIR}/logs) endif () +if ( NOT MSVC ) + set(HAVE_SUPERVISOR true) +endif () + install(DIRECTORY DESTINATION ${ZEEK_ETC_INSTALL_DIR}) install(DIRECTORY DESTINATION ${ZEEK_STATE_DIR}) install(DIRECTORY DESTINATION ${ZEEK_SPOOL_DIR}) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9fd5d2c743..0823b83aeb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -177,6 +177,8 @@ gen_zam_target(${GEN_ZAM_SRC}) ## Including subdirectories. ######################################################################## +option(USE_SQLITE "Should Zeek use SQLite?" ON) + set(bro_SUBDIR_LIBS CACHE INTERNAL "subdir libraries" FORCE) set(bro_SUBDIR_DEPS CACHE INTERNAL "subdir dependencies" FORCE) set(bro_PLUGIN_LIBS CACHE INTERNAL "plugin libraries" FORCE) @@ -441,7 +443,7 @@ set(THIRD_PARTY_SRCS 3rdparty/modp_numtoa.c 3rdparty/patricia.c 3rdparty/setsignal.c - 3rdparty/sqlite3.c + $<$:3rdparty/sqlite3.c> 3rdparty/strsep.c ) @@ -679,7 +681,7 @@ install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/modp_numtoa.h ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/patricia.h ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/setsignal.h - ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/sqlite3.h + $<$:${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/sqlite3.h> ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/doctest.h DESTINATION include/zeek/3rdparty ) diff --git a/src/Val.cc b/src/Val.cc index d0091d35f5..9c3db80752 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -3963,17 +3963,9 @@ ValManager::ValManager() for ( auto i = 0u; i < PREALLOCATED_INTS; ++i ) ints[i] = Val::MakeInt(PREALLOCATED_INT_LOWEST + i); - for ( auto i = 0u; i < ports.size(); ++i ) - { - auto& arr = ports[i]; - auto port_type = (TransportProto)i; - - for ( auto j = 0u; j < arr.size(); ++j ) - arr[j] = IntrusivePtr{AdoptRef{}, new PortVal(PortVal::Mask(j, port_type))}; - } } -const PortValPtr& ValManager::Port(uint32_t port_num, TransportProto port_type) const +const PortValPtr& ValManager::Port(uint32_t port_num, TransportProto port_type) { if ( port_num >= 65536 ) { @@ -3981,10 +3973,17 @@ const PortValPtr& ValManager::Port(uint32_t port_num, TransportProto port_type) port_num = 0; } - return ports[port_type][port_num]; + + std::pair key{port_num, port_type}; + if (ports.find(key) == ports.end()) + { + ports[key] = IntrusivePtr{AdoptRef{}, new PortVal(PortVal::Mask(port_num, port_type))}; } -const PortValPtr& ValManager::Port(uint32_t port_num) const + return ports[key]; + } + +const PortValPtr& ValManager::Port(uint32_t port_num) { auto mask = port_num & PORT_SPACE_MASK; port_num &= ~PORT_SPACE_MASK; diff --git a/src/Val.h b/src/Val.h index 467c1361f4..60e1aa73dd 100644 --- a/src/Val.h +++ b/src/Val.h @@ -298,9 +298,15 @@ protected: class ValManager { public: +#ifdef _MSC_VER + static constexpr zeek_uint_t PREALLOCATED_COUNTS = 1; + static constexpr zeek_uint_t PREALLOCATED_INTS = 1; + static constexpr zeek_int_t PREALLOCATED_INT_LOWEST = 0; +#else static constexpr zeek_uint_t PREALLOCATED_COUNTS = 4096; static constexpr zeek_uint_t PREALLOCATED_INTS = 512; static constexpr zeek_int_t PREALLOCATED_INT_LOWEST = -255; +#endif static constexpr zeek_int_t PREALLOCATED_INT_HIGHEST = PREALLOCATED_INT_LOWEST + PREALLOCATED_INTS - 1; @@ -327,13 +333,13 @@ public: inline const StringValPtr& EmptyString() const { return empty_string; } // Port number given in host order. - const PortValPtr& Port(uint32_t port_num, TransportProto port_type) const; + const PortValPtr& Port(uint32_t port_num, TransportProto port_type); // Host-order port number already masked with port space protocol mask. - const PortValPtr& Port(uint32_t port_num) const; + const PortValPtr& Port(uint32_t port_num); private: - std::array, NUM_PORT_SPACES> ports; + std::map, PortValPtr> ports; std::array counts; std::array ints; StringValPtr empty_string; @@ -1124,9 +1130,8 @@ public: AddedField(field); } - // For int types, we provide both [u]int32_t and [u]int64_t versions - // for convenience, since sometimes the caller has one rather - // than the other. + // For int types, we provide both [u]int32_t and [u]int64_t versions for + // convenience, since sometimes the caller has one rather than the other. void Assign(int field, int32_t new_val) { (*record_val)[field] = ZVal(zeek_int_t(new_val)); diff --git a/src/input/readers/CMakeLists.txt b/src/input/readers/CMakeLists.txt index d653789847..c82f175f08 100644 --- a/src/input/readers/CMakeLists.txt +++ b/src/input/readers/CMakeLists.txt @@ -4,4 +4,6 @@ add_subdirectory(benchmark) add_subdirectory(binary) add_subdirectory(config) add_subdirectory(raw) -add_subdirectory(sqlite) +if (USE_SQLITE) + add_subdirectory(sqlite) +endif() diff --git a/src/logging/writers/CMakeLists.txt b/src/logging/writers/CMakeLists.txt index 867ad58c47..3aa507b316 100644 --- a/src/logging/writers/CMakeLists.txt +++ b/src/logging/writers/CMakeLists.txt @@ -1,4 +1,6 @@ add_subdirectory(ascii) add_subdirectory(none) -add_subdirectory(sqlite) +if (USE_SQLITE) + add_subdirectory(sqlite) +endif() diff --git a/src/supervisor/Supervisor.h b/src/supervisor/Supervisor.h index cee3a758f2..c0812dd7d4 100644 --- a/src/supervisor/Supervisor.h +++ b/src/supervisor/Supervisor.h @@ -2,6 +2,8 @@ #pragma once +#include "zeek/zeek-config.h" + #include #include #include diff --git a/src/util.cc b/src/util.cc index a16728370d..6b90c1cd8a 100644 --- a/src/util.cc +++ b/src/util.cc @@ -2434,6 +2434,9 @@ void get_memory_usage(uint64_t* total, uint64_t* malloced) // In KB. ret_total = r.ru_maxrss * 1024; + + if ( malloced ) + *malloced = r.ru_ixrss * 1024; #endif if ( total ) diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index 676cc7a49d..c39b9a8833 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -16,7 +16,9 @@ #include #include +#ifdef USE_SQLITE #include "zeek/3rdparty/sqlite3.h" +#endif #define DOCTEST_CONFIG_IMPLEMENT @@ -645,11 +647,12 @@ SetupResult setup(int argc, char** argv, Options* zopts) // FIXME: On systems that don't provide /dev/urandom, OpenSSL doesn't // seed the PRNG. We should do this here (but at least Linux, FreeBSD // and Solaris provide /dev/urandom). - +#ifdef USE_SQLITE int r = sqlite3_initialize(); if ( r != SQLITE_OK ) reporter->Error("Failed to initialize sqlite3: %s", sqlite3_errstr(r)); +#endif timer_mgr = new TimerMgr(); @@ -1121,7 +1124,9 @@ int cleanup(bool did_run_loop) run_state::detail::delete_run(); terminate_zeek(); +#ifdef USE_SQLITE sqlite3_shutdown(); +#endif do_ssl_deinit(); diff --git a/zeek-config.h.in b/zeek-config.h.in index 032b399ba3..a6492ddd53 100644 --- a/zeek-config.h.in +++ b/zeek-config.h.in @@ -135,6 +135,9 @@ /* Use the ElasticSearch writer. */ #cmakedefine USE_ELASTICSEARCH +/* Use the sqlite reader/writer. */ +#cmakedefine USE_SQLITE + /* Version number of package */ #define VERSION "@VERSION@" From d2f625c115c7a33aa472756ca97fda1722486036 Mon Sep 17 00:00:00 2001 From: Tomer Lev Date: Mon, 20 Dec 2021 09:49:17 +0000 Subject: [PATCH 12/71] Replaced weakly_canonical with lexically_normal in normalize_path func. Replaced weakly_canonical with lexically_normal in normalize_path func. Normalizing the path now does not access the filesystem. --- src/util.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/util.cc b/src/util.cc index 6b90c1cd8a..9682cd7a87 100644 --- a/src/util.cc +++ b/src/util.cc @@ -675,15 +675,13 @@ string normalize_path(std::string_view path) if (0 == path.compare(zeek::detail::ScannedFile::canonical_stdin_path)) { return string(path); } - string stringPath = string(path); - // "//" interferes with std::weakly_canonical + string stringPath = string(path); if (stringPath._Starts_with("//")) { stringPath.erase(0, 2); } - - return std::filesystem::weakly_canonical(path).string(); + return std::filesystem::path(stringPath).lexically_normal().string(); #else if ( path.find("/.") == std::string_view::npos && path.find("//") == std::string_view::npos ) { From d352ace4d2acae10e023b776e5bac02384896612 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 21 Oct 2022 12:02:28 -0700 Subject: [PATCH 13/71] Add conanfile for windows builds --- CMakeLists.txt | 42 +++++++++++++++++++++------------- ci/conan/conanfile_windows.txt | 9 ++++++++ 2 files changed, 35 insertions(+), 16 deletions(-) create mode 100644 ci/conan/conanfile_windows.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index b107ba9118..62da640bf0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,27 +17,37 @@ list(APPEND CMAKE_PREFIX_PATH ${CMAKE_BINARY_DIR}) # Windows: Configure runtime and dependencies if ( MSVC ) cmake_policy(SET CMP0091 NEW) - if (CMAKE_BUILD_TYPE STREQUAL "Debug") - set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreadedDebug") - set(CMAKE_MSVC_RUNTIME_LIBRARY_FLAG "MTd") - else () - set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded") - set(CMAKE_MSVC_RUNTIME_LIBRARY_FLAG "MT") - endif () + + # Remove existing runtime flags + set(CompilerFlags + CMAKE_CXX_FLAGS + CMAKE_CXX_FLAGS_DEBUG + CMAKE_CXX_FLAGS_RELEASE + CMAKE_CXX_FLAGS_RELWITHDEBINFO + CMAKE_C_FLAGS + CMAKE_C_FLAGS_DEBUG + CMAKE_C_FLAGS_RELEASE + CMAKE_C_FLAGS_RELWITHDEBINFO + ) + foreach(CompilerFlag ${CompilerFlags}) + string(REGEX REPLACE "[/|-]MDd" "" ${CompilerFlag} "${${CompilerFlag}}") + string(REGEX REPLACE "[/|-]MD" "" ${CompilerFlag} "${${CompilerFlag}}") + string(REGEX REPLACE "[/|-]MTd" "" ${CompilerFlag} "${${CompilerFlag}}") + string(REGEX REPLACE "[/|-]MT" "" ${CompilerFlag} "${${CompilerFlag}}") + string(REGEX REPLACE "[/|-]Zi" "" ${CompilerFlag} "${${CompilerFlag}}") + string(REGEX REPLACE "[/|-]W3" "" ${CompilerFlag} "${${CompilerFlag}}") + string(REGEX REPLACE "[/|-]W4" "" ${CompilerFlag} "${${CompilerFlag}}") + endforeach() + + # Set always to static runtime + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd") set(OPENSSL_USE_STATIC_LIBS true) set(OPENSSL_MSVC_STATIC_RT true) if ( ZEEK_STANDALONE ) include(${CMAKE_SOURCE_DIR}/cmake/conan.cmake) - - conan_cmake_configure(REQUIRES - zlib/1.2.11 - openssl/1.1.1i - winflexbison/2.5.24 - npcap-wpcap/1.31 - kqueue/2.3.2 - GENERATORS cmake_find_package) - + conan_cmake_autodetect(settings) # Install packages from conanfile conan_cmake_install(PATH_OR_REFERENCE ${CMAKE_SOURCE_DIR}/ci/windows/conanfile_windows.txt BUILD missing diff --git a/ci/conan/conanfile_windows.txt b/ci/conan/conanfile_windows.txt new file mode 100644 index 0000000000..cefa5503b2 --- /dev/null +++ b/ci/conan/conanfile_windows.txt @@ -0,0 +1,9 @@ +[requires] +zlib/1.2.11 +openssl/1.1.1i +winflexbison/2.5.24 +npcap-wpcap/1.31 +kqueue/2.3.2 + +[generators] +cmake_find_package \ No newline at end of file From b712f00b7e5fb20b0c6b95419d7ad7bfd7745df0 Mon Sep 17 00:00:00 2001 From: PIE Easy Start Date: Sun, 22 Aug 2021 16:04:59 +0000 Subject: [PATCH 14/71] Adding CI files --- ci/build.ps1 | 45 ----------------- ci/windows/CMakePresets.json | 67 +++++++++++++++++++++++++ ci/windows/Initialize-MSVC.ps1 | 73 ++++++++++++++++++++++++++++ ci/windows/Install-Prerequisites.ps1 | 12 +++++ 4 files changed, 152 insertions(+), 45 deletions(-) delete mode 100644 ci/build.ps1 create mode 100644 ci/windows/CMakePresets.json create mode 100644 ci/windows/Initialize-MSVC.ps1 create mode 100644 ci/windows/Install-Prerequisites.ps1 diff --git a/ci/build.ps1 b/ci/build.ps1 deleted file mode 100644 index 5b18a6a2df..0000000000 --- a/ci/build.ps1 +++ /dev/null @@ -1,45 +0,0 @@ -param( - [Parameter()] - [ValidateSet("Debug", "Release")] - [string] $BuildType = "Release" -) - -$SourceDirectory = (Convert-Path "$PSScriptRoot/../").Replace("\", "/") -$WorkingDirectory = "$pwd" -$BuildOutputDirectory = "$SourceDirectory/out/build/x64/$BuildType" -$InstallDirectory = "$SourceDirectory/out/install/x64/$BuildType" - -$commands = @() -if (!(Get-Command cl)) { - $commands += '"C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Auxiliary/Build/vcvars64.bat"' -} - -$commands += @" -cmake.exe - -G "Ninja" - -S "$SourceDirectory" - -B "$WorkingDirectory" - -DCMAKE_BUILD_TYPE:STRING=$BuildType - -DCMAKE_INSTALL_PREFIX:PATH="$InstallDirectory" - -DDISABLE_PYTHON_BINDINGS=1 ` - 2>&1 -"@.Replace("`r`n", "") - -$commands += @" -cmake.exe --build $WorkingDirectory --config $BuildType -"@ - -$commands += @" -cmake.exe --install $WorkingDirectory -"@ -cmd /c ($commands -join " && ") - -Write-Host "Copying build output to $BuildOutputDirectory..." -mkdir $BuildOutputDirectory -Force | Out-Null -Get-ChildItem $WorkingDirectory -Recurse -Attributes !ReparsePoint | foreach { - $path = $_.FullName.Replace("$pwd", $BuildOutputDirectory) - $parent = Split-Path -Parent $path - mkdir $parent -Force | Out-Null - Copy-Item $_.FullName -Destination $path -Force -} -Write-Host "Done." \ No newline at end of file diff --git a/ci/windows/CMakePresets.json b/ci/windows/CMakePresets.json new file mode 100644 index 0000000000..515f9df339 --- /dev/null +++ b/ci/windows/CMakePresets.json @@ -0,0 +1,67 @@ +{ + "version": 2, + "configurePresets": [ + { + "name": "base", + "hidden": true, + "description": "Base preset", + "generator": "Ninja", + "binaryDir": "${sourceDir}/out/build/${presetName}", + "cacheVariables": { + "CMAKE_C_COMPILER": "cl.exe", + "CMAKE_CXX_COMPILER": "cl.exe" + }, + "architecture": { + "value": "x64", + "strategy": "external" + }, + "toolset": { + "value": "host=x64,version=16.11", + "strategy": "external" + } + }, + { + "name": "x64-Debug", + "inherits": "base", + "displayName": "x64 Debug", + "description": "Sets debug build type and x64 arch", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } + }, + { + "name": "x64-Release", + "displayName": "x64 Release", + "description": "Sets release build type", + "inherits": "x64-Debug", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } + } + ], + "buildPresets": [ + { + "name": "default-debug-build", + "displayName": "Debug Build", + "configurePreset": "x64-Debug" + }, + { + "name": "verbose-debug-build", + "displayName": "Verbose Debug Build", + "configurePreset": "x64-Debug", + "nativeToolOptions": [ "-v" ] + }, + { + "name": "default-release-build", + "displayName": "Release Build", + "configurePreset": "x64-Release" + }, + { + "name": "verbose-release-build", + "displayName": "Verbose Release Build", + "configurePreset": "x64-Release", + "nativeToolOptions": [ "-v" ] + } + ], + "testPresets": [] +} \ No newline at end of file diff --git a/ci/windows/Initialize-MSVC.ps1 b/ci/windows/Initialize-MSVC.ps1 new file mode 100644 index 0000000000..28b4a9611e --- /dev/null +++ b/ci/windows/Initialize-MSVC.ps1 @@ -0,0 +1,73 @@ +param ( + [string] $Architecture='x64' +) + +if (!(Get-Module VSSetup)) +{ + Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force + Install-Module VSSetup -Force +} + +function Get-VSInstallPath { + [CmdletBinding()] + param ( + [Parameter(Mandatory=$true)] + [int] $MaxVersion + ) + $latest = Get-VSSetupInstance ` + | where { $_.InstallationVersion.Major -le $MaxVersion } ` + | Select-VSSetupInstance -Require Microsoft.VisualStudio.VC.CMake -Latest + + if (!$latest) + { + throw [System.IO.FileNotFoundException]::new("No Visual Studio installation found that matches max version: $MaxVersion!") + } + return $latest.InstallationPath +} + +function Where-Program { + [CmdletBinding()] + param ( + [Parameter(Mandatory=$true,ValueFromPipeline=$true)] + [string] $Program + ) + process + { + return Get-Command $Program | select Source -ExpandProperty source | Split-Path -Parent + } +} + +function Persist-EnvironmentVariable { + [CmdletBinding()] + param ( + [Parameter(Mandatory=$true,ValueFromPipeline=$true)] + [string] $Name + ) + process + { + $value=[System.Environment]::GetEnvironmentVariable($Name) + [System.Environment]::SetEnvironmentVariable($Name, $value, [System.EnvironmentVariableTarget]::Machine) + } +} + +$VsInstallationPath = Get-VSInstallPath -MaxVersion 16 +Write-Host "Found VS installation: $VsInstallationPath" + +[array] $originalEnv = [System.Environment]::GetEnvironmentVariables().Keys + +Import-Module "$VsInstallationPath\Common7\Tools\Microsoft.VisualStudio.DevShell.dll" +Enter-VsDevShell -VsInstallPath $VsInstallationPath -SkipAutomaticLocation -DevCmdArguments "-arch=$Architecture" + +$Programs = 'cmake', 'Ninja', 'cl' +$programsPath = $Programs | Where-Program +Write-Host "Found paths for $($Programs -join ', '): $($programsPath -join ', ')" + +$newPath = "$($programsPath -join ';');${env:Path}" +Write-Host "Persisting new PATH: $newPath" +[System.Environment]::SetEnvironmentVariable('PATH', $newPath, [System.EnvironmentVariableTarget]::Machine) + +[array] $vsEnv = [System.Environment]::GetEnvironmentVariables().Keys +[array] $newEnv = $vsEnv | where { $_ -notin $originalEnv -and $_ -ne 'PATH' } + +Write-Host "Persisting new environment variables: $($newEnv -join ', ')" +$newEnv | Persist-EnvironmentVariable diff --git a/ci/windows/Install-Prerequisites.ps1 b/ci/windows/Install-Prerequisites.ps1 new file mode 100644 index 0000000000..2d15cb9b35 --- /dev/null +++ b/ci/windows/Install-Prerequisites.ps1 @@ -0,0 +1,12 @@ +[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072 +iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) + +choco install conan -y +choco install sed -y +choco install winflexbison -y +if (!(Get-Command python)) +{ + choco install python -y +} + +[System.Environment]::SetEnvironmentVariable('PATH', "C:\Program Files\Git\bin;${env:PATH}", [System.EnvironmentVariableTarget]::Machine) \ No newline at end of file From 216c00d7e9c4b561284dcbdcd7a01f51e2d89b3d Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 21 Oct 2022 12:05:09 -0700 Subject: [PATCH 15/71] Use multithreaded runtime library under MSVC --- CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 62da640bf0..8ed8bd3cd8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,14 @@ if ( MSVC ) # Set always to static runtime set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd") + if (CMAKE_BUILD_TYPE STREQUAL "Debug") + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreadedDebug") + set(CMAKE_MSVC_RUNTIME_LIBRARY_FLAG "MTd") + else () + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded") + set(CMAKE_MSVC_RUNTIME_LIBRARY_FLAG "MT") + endif () + set(OPENSSL_USE_STATIC_LIBS true) set(OPENSSL_MSVC_STATIC_RT true) From 62e2f85d2f69276786f9024401631bfc1b51393c Mon Sep 17 00:00:00 2001 From: Tomer Lev Date: Wed, 19 Jan 2022 09:32:51 +0000 Subject: [PATCH 16/71] Manually patched 2 commits from zeek github In case transport layer has zero length and we are ignoring checksums, calculate the length ourselvs. This happens due to protocol fragmantation --- ci/conan/conanfile_windows.txt | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 ci/conan/conanfile_windows.txt diff --git a/ci/conan/conanfile_windows.txt b/ci/conan/conanfile_windows.txt deleted file mode 100644 index cefa5503b2..0000000000 --- a/ci/conan/conanfile_windows.txt +++ /dev/null @@ -1,9 +0,0 @@ -[requires] -zlib/1.2.11 -openssl/1.1.1i -winflexbison/2.5.24 -npcap-wpcap/1.31 -kqueue/2.3.2 - -[generators] -cmake_find_package \ No newline at end of file From f500c4faf4a769869bbee09cdf9679f67aa2880a Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Sun, 27 Mar 2022 14:16:07 +0300 Subject: [PATCH 17/71] Added Compilation flags for vpack --- CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8ed8bd3cd8..4f20b230d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,6 +38,15 @@ if ( MSVC ) string(REGEX REPLACE "[/|-]W3" "" ${CompilerFlag} "${${CompilerFlag}}") string(REGEX REPLACE "[/|-]W4" "" ${CompilerFlag} "${${CompilerFlag}}") endforeach() + + # Set compilation flags for Windows + add_compile_options( + /guard:cf # required by CheckCFlags + /Z7) # required by CheckCFlags + + add_link_options( + /debug:full # required by CheckCFlags + ) # Set always to static runtime set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT") From 1b3719cf29ce8486630144b7f6c655ff4bd8b148 Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Wed, 30 Mar 2022 14:47:18 +0300 Subject: [PATCH 18/71] Fix libpcap usage --- CMakeLists.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4f20b230d9..ec72cc59d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,12 +38,12 @@ if ( MSVC ) string(REGEX REPLACE "[/|-]W3" "" ${CompilerFlag} "${${CompilerFlag}}") string(REGEX REPLACE "[/|-]W4" "" ${CompilerFlag} "${${CompilerFlag}}") endforeach() - + # Set compilation flags for Windows add_compile_options( /guard:cf # required by CheckCFlags /Z7) # required by CheckCFlags - + add_link_options( /debug:full # required by CheckCFlags ) @@ -71,11 +71,11 @@ if ( MSVC ) SETTINGS ${settings}) endif() - # Set LibPCAP to point to wpcap binaries. - find_package(npcap-wpcap) - set(PCAP_ROOT_DIR "${npcap-wpcap_LIB_DIRS}/../") - set(PCAP_INCLUDE_DIR ${npcap-wpcap_INCLUDES}) - set(PCAP_LIBRARY ${npcap-wpcap_LIBS}) + # Set LibPCAP to point to libpcap binaries. + find_package(libpcap) + set(PCAP_ROOT_DIR "${libpcap_LIB_DIRS}/../") + set(PCAP_INCLUDE_DIR ${libpcap_INCLUDES}) + set(PCAP_LIBRARY ${libpcap_LIBS}) set(LIBPCAP_PCAP_COMPILE_NOPCAP_HAS_ERROR_PARAMETER false) # Set ZLib to point at the right variable. From 032f0587f47715a700d3278d397b04c17e663214 Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Wed, 30 Mar 2022 22:14:21 +0300 Subject: [PATCH 19/71] Updated windows dockerfile --- ci/windows/Dockerfile | 9 ++------- ci/windows/default | 12 ------------ 2 files changed, 2 insertions(+), 19 deletions(-) delete mode 100644 ci/windows/default diff --git a/ci/windows/Dockerfile b/ci/windows/Dockerfile index c6bfe16fc8..2f00bc856f 100644 --- a/ci/windows/Dockerfile +++ b/ci/windows/Dockerfile @@ -35,23 +35,18 @@ RUN [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePoin SHELL ["cmd", "/S", "/C"] # Install prerequisites +RUN choco install powershell-core -y RUN choco install conan -y +RUN choco install sed -y RUN choco install winflexbison -y -RUN choco install openssl -y -RUN choco install python -y # Can't install a proper msys2 because it caused console hanging in the container during docker build. RUN choco install msysgit -y -RUN choco install sed -y -RUN choco install powershell-core -y # Set working environment. RUN setx /M PATH "%PATH%;C:\\Program Files\\Git\\bin" RUN mkdir C:\build WORKDIR C:\build -# Configure conan -ADD default C:\Users\ContainerAdministrator\.conan\profiles\default - # Define the entry point for the docker container. # This entry point starts the developer command prompt and launches the PowerShell shell. ENTRYPOINT ["C:\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat", "&&", "pwsh.exe", "-NoLogo", "-ExecutionPolicy", "Unrestricted"] \ No newline at end of file diff --git a/ci/windows/default b/ci/windows/default deleted file mode 100644 index 620a4bf3a9..0000000000 --- a/ci/windows/default +++ /dev/null @@ -1,12 +0,0 @@ -[settings] -os=Windows -os_build=Windows -arch=x86_64 -arch_build=x86_64 -compiler=Visual Studio -compiler.version=16 -compiler.runtime=MT -build_type=Release -[options] -[build_requires] -[env] From d7734ca783e900d8b3bb3bb6ada895dddc49fab9 Mon Sep 17 00:00:00 2001 From: Tomer Lev Date: Sun, 22 May 2022 07:43:36 +0000 Subject: [PATCH 20/71] Return the actual error code instead of a message when compiling to windows --- src/util.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/util.cc b/src/util.cc index 9682cd7a87..5d0b9a8341 100644 --- a/src/util.cc +++ b/src/util.cc @@ -2547,7 +2547,12 @@ static void strerror_r_helper(int result, char* buf, size_t buflen) void zeek_strerror_r(int zeek_errno, char* buf, size_t buflen) { +#if defined(_MSC_VER) + auto str = "Error number: " + std::to_string(zeek_errno); + auto res = str.data(); +#else auto res = strerror_r(zeek_errno, buf, buflen); +#endif // GNU vs. XSI flavors make it harder to use strerror_r. strerror_r_helper(res, buf, buflen); } From 27d0bbde0f47c3f68bcbf1f0d64690b406de3767 Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Sun, 22 May 2022 10:41:17 +0000 Subject: [PATCH 21/71] Updated dockerfile for Windows --- ci/windows/Dockerfile | 33 ++++++--------------------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/ci/windows/Dockerfile b/ci/windows/Dockerfile index 2f00bc856f..8e93fdd8af 100644 --- a/ci/windows/Dockerfile +++ b/ci/windows/Dockerfile @@ -1,28 +1,6 @@ # escape=` FROM mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-ltsc2019 -# Restore the default Windows shell for correct batch processing. -SHELL ["cmd", "/S", "/C"] - -# Download the Build Tools bootstrapper. -ADD https://aka.ms/vs/16/release/vs_buildtools.exe C:\TEMP\vs_buildtools.exe - -# Install Build Tools. -RUN C:\TEMP\vs_buildtools.exe --quiet --wait --norestart --nocache ` - --installPath C:\BuildTools ` - --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended ` - --add Microsoft.VisualStudio.Component.VC.ATLMFC ` - --add Microsoft.VisualStudio.Component.Windows10SDK.18362 ` - --add Microsoft.VisualStudio.Component.Windows10SDK.17763 ` - --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` - --add Microsoft.VisualStudio.Component.VC.v141.x86.x64 ` - --remove Microsoft.VisualStudio.Component.Windows10SDK.10240 ` - --remove Microsoft.VisualStudio.Component.Windows10SDK.10586 ` - --remove Microsoft.VisualStudio.Component.Windows10SDK.14393 ` - --remove Microsoft.VisualStudio.Component.Windows81SDK ` - || IF "%ERRORLEVEL%"=="3010" EXIT 0 - - SHELL [ "powershell" ] RUN Set-ExecutionPolicy Unrestricted -Force @@ -31,22 +9,23 @@ RUN Set-ExecutionPolicy Unrestricted -Force RUN [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; ` iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) -# Restore the default Windows shell for correct batch processing. -SHELL ["cmd", "/S", "/C"] - # Install prerequisites +RUN choco install -y --no-progress visualstudio2019buildtools --version=16.11.11.0 +RUN choco install -y --no-progress visualstudio2019-workload-vctools --version=1.0.0 --package-parameters '--add Microsoft.VisualStudio.Component.VC.ATLMFC' RUN choco install powershell-core -y RUN choco install conan -y RUN choco install sed -y RUN choco install winflexbison -y # Can't install a proper msys2 because it caused console hanging in the container during docker build. RUN choco install msysgit -y +RUN choco install python -y # Set working environment. +SHELL [ "cmd", "/c" ] RUN setx /M PATH "%PATH%;C:\\Program Files\\Git\\bin" +RUN setx /M CONAN_SKIP_BROKEN_SYMLINKS_CHECK 1 RUN mkdir C:\build WORKDIR C:\build -# Define the entry point for the docker container. # This entry point starts the developer command prompt and launches the PowerShell shell. -ENTRYPOINT ["C:\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat", "&&", "pwsh.exe", "-NoLogo", "-ExecutionPolicy", "Unrestricted"] \ No newline at end of file +ENTRYPOINT ["C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "-arch=x64", "&&", "pwsh.exe", "-NoLogo", "-ExecutionPolicy", "Unrestricted"] \ No newline at end of file From b7cbd946d6a73d69f8b4ce6dfa97942a4dc77b5b Mon Sep 17 00:00:00 2001 From: Tomer Lev Date: Thu, 7 Jul 2022 09:21:55 +0000 Subject: [PATCH 22/71] Updating submodule to support new doctest version. Compiling without allowing doctest's multithreading which causes crashes on runtime --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index ec72cc59d4..33e9a931d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,6 +99,7 @@ if ( MSVC ) set(DISABLE_PYTHON_BINDINGS true) set(BROKER_DISABLE_TESTS true) set(BROKER_DISABLE_DOC_EXAMPLES true) + add_definitions(-DDOCTEST_CONFIG_NO_MULTITHREADING) else () include(GNUInstallDirs) endif () From af656a235bac3d9c0e1766346925d951d5bfd573 Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Mon, 22 Aug 2022 17:15:10 +0300 Subject: [PATCH 23/71] Fixed cmake scripts --- CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 33e9a931d9..458eedaecc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -88,6 +88,9 @@ if ( MSVC ) set(LIBKQUEUE_INCLUDE_DIRS ${kqueue_INCLUDES}) set(LIBKQUEUE_LIBRARIES ${kqueue_LIBS}) + # Set CAres + find_package(c-ares) + add_subdirectory(auxil/libunistd) set(UNISTD_INCLUDES ${CMAKE_SOURCE_DIR}/auxil/libunistd/unistd ${CMAKE_SOURCE_DIR}/auxil/libunistd/regex) include_directories(BEFORE ${UNISTD_INCLUDES}) @@ -100,6 +103,9 @@ if ( MSVC ) set(BROKER_DISABLE_TESTS true) set(BROKER_DISABLE_DOC_EXAMPLES true) add_definitions(-DDOCTEST_CONFIG_NO_MULTITHREADING) + + # Disable Spicy as it is not yet supported in Windows. + set(DISABLE_SPICY true) else () include(GNUInstallDirs) endif () From f39a29435c10fdf8e8eb1155aaad5bf9a7412a85 Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Mon, 22 Aug 2022 17:50:06 +0300 Subject: [PATCH 24/71] Fixed CARES and SUPERVISOR cmake scripts --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 458eedaecc..e344607820 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,6 +90,9 @@ if ( MSVC ) # Set CAres find_package(c-ares) + set(HAVE_CARES true) + set(zeekdeps ${zeekdeps} ${CARES_LIBRARIES}) + include_directories(BEFORE ${CARES_INCLUDE_DIRS}) add_subdirectory(auxil/libunistd) set(UNISTD_INCLUDES ${CMAKE_SOURCE_DIR}/auxil/libunistd/unistd ${CMAKE_SOURCE_DIR}/auxil/libunistd/regex) From dbaf6696ef1491996bd7afa3e211201224619c33 Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Thu, 8 Sep 2022 19:55:49 +0300 Subject: [PATCH 25/71] Fixed arec package linkage --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e344607820..bf978698db 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,8 +91,8 @@ if ( MSVC ) # Set CAres find_package(c-ares) set(HAVE_CARES true) - set(zeekdeps ${zeekdeps} ${CARES_LIBRARIES}) - include_directories(BEFORE ${CARES_INCLUDE_DIRS}) + set(zeekdeps ${zeekdeps} ${c-ares_LIBRARIES}) + include_directories(BEFORE ${c-ares_INCLUDE_DIRS}) add_subdirectory(auxil/libunistd) set(UNISTD_INCLUDES ${CMAKE_SOURCE_DIR}/auxil/libunistd/unistd ${CMAKE_SOURCE_DIR}/auxil/libunistd/regex) From 4043b67e0b954780ac29bccb9b5b3134b3c2bf50 Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Thu, 8 Sep 2022 19:56:46 +0300 Subject: [PATCH 26/71] Fixed zeek::filesystem::path conversion to string --- src/logging/writers/ascii/Ascii.cc | 10 +++++----- src/logging/writers/sqlite/SQLite.cc | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/logging/writers/ascii/Ascii.cc b/src/logging/writers/ascii/Ascii.cc index 8a703331e2..13b9711b86 100644 --- a/src/logging/writers/ascii/Ascii.cc +++ b/src/logging/writers/ascii/Ascii.cc @@ -88,7 +88,7 @@ struct LeftoverLog * Return the "path" (logging framework parlance) of the log without the * directory or file extension. E.g. the "path" of "logs/conn.log" is just "conn". */ - std::string Path() const { return zeek::filesystem::path(filename).stem(); } + std::string Path() const { return zeek::filesystem::path(filename).stem().string(); } /** * Deletes the shadow file and returns whether it succeeded. @@ -493,7 +493,7 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const threading::Fiel } if ( fname.front() != '/' && ! logdir.empty() ) - fname = zeek::filesystem::path(logdir) / fname; + fname = (zeek::filesystem::path(logdir) / fname).string(); fname += ext; @@ -773,7 +773,7 @@ static std::vector find_leftover_logs() if ( BifConst::LogAscii::logdir->Len() > 0 ) logdir = zeek::filesystem::absolute(BifConst::LogAscii::logdir->ToStdString()); - auto d = opendir(logdir.c_str()); + auto d = opendir(logdir.string().c_str()); struct dirent* dp; if ( ! d ) @@ -788,8 +788,8 @@ static std::vector find_leftover_logs() if ( strncmp(dp->d_name, shadow_file_prefix, prefix_len) != 0 ) continue; - std::string shadow_fname = logdir / dp->d_name; - std::string log_fname = logdir / (dp->d_name + prefix_len); + std::string shadow_fname = (logdir / dp->d_name).string(); + std::string log_fname = (logdir / (dp->d_name + prefix_len)).string(); if ( util::is_file(log_fname) ) { diff --git a/src/logging/writers/sqlite/SQLite.cc b/src/logging/writers/sqlite/SQLite.cc index f92c3f6e74..9eee5596e9 100644 --- a/src/logging/writers/sqlite/SQLite.cc +++ b/src/logging/writers/sqlite/SQLite.cc @@ -147,7 +147,7 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields, const Field* con tablename = it->second; if ( checkError(sqlite3_open_v2( - fullpath.c_str(), &db, + fullpath.string().c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX, NULL)) ) return false; From dfd6cbe6b46225b2bb0103c8f59d2e0c40066de4 Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Thu, 8 Sep 2022 19:57:08 +0300 Subject: [PATCH 27/71] Fixed usages of path_list_separator --- src/plugin/Manager.cc | 2 +- src/util.cc | 6 +++--- src/util.h | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index e4ee83dd83..8830cc9f65 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -63,7 +63,7 @@ void Manager::SearchDynamicPlugins(const std::string& dir) std::stringstream s(dir); std::string d; - while ( std::getline(s, d, path_list_separator) ) + while ( std::getline(s, d, path_list_separator[0]) ) SearchDynamicPlugins(d); return; diff --git a/src/util.cc b/src/util.cc index 5d0b9a8341..d62b05e304 100644 --- a/src/util.cc +++ b/src/util.cc @@ -71,7 +71,7 @@ static bool can_read(const string& path) } static string zeek_path_value; -const string zeek_path_list_separator(1, path_list_separator); +const string zeek_path_list_separator(path_list_separator.begin(), path_list_separator.end()); namespace zeek::util { @@ -576,7 +576,7 @@ void add_to_zeek_path(const string& dir) // Make sure path is initialized. zeek_path(); - zeek_path_value += string(zeek_path_list_separator) + dir; + zeek_path_value += zeek_path_list_separator + dir; } FILE* open_package(string& path, const string& mode) @@ -754,7 +754,7 @@ string without_zeekpath_component(std::string_view path) { string rval = normalize_path(path); - const auto paths = tokenize_string(zeek_path(), path_list_separator); + const auto paths = tokenize_string(zeek_path(), path_list_separator[0]); for ( size_t i = 0; i < paths.size(); ++i ) { diff --git a/src/util.h b/src/util.h index 5acb449df7..c7fa0ceab5 100644 --- a/src/util.h +++ b/src/util.h @@ -78,9 +78,9 @@ extern HeapLeakChecker* heap_checker; #endif #if defined(_MSC_VER) -const char path_list_separator = ';'; +inline constexpr std::string_view path_list_separator = ";"; #else -const char path_list_separator = ':'; +inline constexpr std::string_view path_list_separator = ":"; #endif extern "C" From 44df3087d60a78f133eb9806e0b89bd0fe6fe1e2 Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Thu, 8 Sep 2022 19:57:52 +0300 Subject: [PATCH 28/71] Fixed intrusive ptr conversion to Obj* --- src/IntrusivePtr.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/IntrusivePtr.h b/src/IntrusivePtr.h index 93efcc0a71..2d8767cb0d 100644 --- a/src/IntrusivePtr.h +++ b/src/IntrusivePtr.h @@ -6,6 +6,8 @@ #include #include +#include "Obj.h" + namespace zeek { @@ -111,7 +113,7 @@ public: ~IntrusivePtr() { if ( ptr_ ) - Unref(ptr_); + Unref((zeek::Obj*)ptr_); } void swap(IntrusivePtr& other) noexcept { std::swap(ptr_, other.ptr_); } From 79fbfd0af7cd9dae04331ce6563e8b29da37fe49 Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Thu, 8 Sep 2022 19:59:51 +0300 Subject: [PATCH 29/71] Fixed include order of unistd, repositioned it at the top --- src/DNS_Mgr.cc | 3 ++- src/Options.cc | 4 ++-- src/iosource/BPF_Program.cc | 3 +++ src/main.cc | 2 ++ src/zeek-setup.cc | 3 ++- 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 46739eeddf..8440d15de5 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -1,5 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include + #include "zeek/DNS_Mgr.h" #include "zeek/zeek-config.h" @@ -10,7 +12,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Options.cc b/src/Options.cc index 31e5b38c86..458a52f93b 100644 --- a/src/Options.cc +++ b/src/Options.cc @@ -1,11 +1,11 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include + #include "zeek/Options.h" #include "zeek/zeek-config.h" -#include - #include "zeek/ScriptProfile.h" #include "zeek/script_opt/ScriptOpt.h" diff --git a/src/iosource/BPF_Program.cc b/src/iosource/BPF_Program.cc index 71c9f7ee27..1ee1952143 100644 --- a/src/iosource/BPF_Program.cc +++ b/src/iosource/BPF_Program.cc @@ -1,5 +1,8 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include +#include + #include "zeek/iosource/BPF_Program.h" #include "zeek/zeek-config.h" diff --git a/src/main.cc b/src/main.cc index 7436292039..aeb15627f5 100644 --- a/src/main.cc +++ b/src/main.cc @@ -1,5 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include + #include "zeek/zeek-config.h" #include "zeek/RunState.h" diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index c39b9a8833..b718988b82 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -1,5 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include + #include "zeek/zeek-setup.h" #include "zeek/zeek-config.h" @@ -8,7 +10,6 @@ #include #include #include -#include #include #include #include From b5c243b9babd7729f3c8ea651045a757ee52cc4e Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Thu, 8 Sep 2022 20:01:28 +0300 Subject: [PATCH 30/71] Fixed type of ntohl return value --- src/packet_analysis/protocol/tcp/TCPSessionAdapter.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/packet_analysis/protocol/tcp/TCPSessionAdapter.cc b/src/packet_analysis/protocol/tcp/TCPSessionAdapter.cc index 3bd418ea86..58cd2fca9f 100644 --- a/src/packet_analysis/protocol/tcp/TCPSessionAdapter.cc +++ b/src/packet_analysis/protocol/tcp/TCPSessionAdapter.cc @@ -1791,8 +1791,8 @@ int TCPSessionAdapter::ParseTCPOptions(const struct tcphdr* tcp, bool is_orig) // timestamps if ( length == 10 ) { - auto send = ntohl(*reinterpret_cast(o + 2)); - auto echo = ntohl(*reinterpret_cast(o + 6)); + uint32_t send = ntohl(*reinterpret_cast(o + 2)); + uint32_t echo = ntohl(*reinterpret_cast(o + 6)); option_record->Assign(6, send); option_record->Assign(7, echo); } @@ -1809,7 +1809,7 @@ int TCPSessionAdapter::ParseTCPOptions(const struct tcphdr* tcp, bool is_orig) { auto rate = o[2]; auto ttl_diff = o[3]; - auto qs_nonce = ntohl(*reinterpret_cast(o + 4)); + uint32_t qs_nonce = ntohl(*reinterpret_cast(o + 4)); option_record->Assign(8, rate); option_record->Assign(9, ttl_diff); option_record->Assign(10, qs_nonce); From 3d29f03d6dde2f51efffc32833e31aca017e78ee Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Thu, 8 Sep 2022 20:02:54 +0300 Subject: [PATCH 31/71] Fixed ares usage --- src/DNS_Mgr.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 8440d15de5..c256b5304d 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -541,10 +541,10 @@ static void query_cb(void* arg, int status, int timeouts, unsigned char* buf, in * some source of data (either a host file or a DNS server). This indicates that we're * able to do lookups against c-ares now and should activate the IOSource. */ -static void sock_cb(void* data, int s, int read, int write) +static void sock_cb(void* data, ares_socket_t s, int read, int write) { auto mgr = reinterpret_cast(data); - mgr->RegisterSocket(s, read == 1, write == 1); + mgr->RegisterSocket((int)s, read == 1, write == 1); } DNS_Mgr::DNS_Mgr(DNS_MgrMode arg_mode) : IOSource(true), mode(arg_mode) From 98a0e242ad6c322de6997590f9e97ca660fbedae Mon Sep 17 00:00:00 2001 From: Elad Solomon Date: Thu, 8 Sep 2022 21:12:38 +0300 Subject: [PATCH 32/71] Fixed invokation of make_enum__CPP --- src/script_opt/CPP/RuntimeInits.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/script_opt/CPP/RuntimeInits.cc b/src/script_opt/CPP/RuntimeInits.cc index b2126b4dcc..1e5678d8b0 100644 --- a/src/script_opt/CPP/RuntimeInits.cc +++ b/src/script_opt/CPP/RuntimeInits.cc @@ -28,7 +28,7 @@ void CPP_IndexedInits::Generate(InitsManager* im, std::vector& iv { auto& e_type = im->Types(init_vals[0]); int val = init_vals[1]; - ivec[offset] = make_enum__CPP(e_type, val); + ivec[offset] = zeek::detail::make_enum__CPP(e_type, val); } template From 0c61bcbfcdf94ede563b10d16e9a57e80b7cc165 Mon Sep 17 00:00:00 2001 From: Tomer Lev Date: Sun, 18 Sep 2022 15:04:06 +0300 Subject: [PATCH 33/71] Fixed extern C issue with windows headers, fixed a GEN-ZAM regex issue, fixd compilation with c-ares --- CMakeLists.txt | 5 +++-- src/CMakeLists.txt | 1 + src/telemetry/CMakeLists.txt | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bf978698db..e52092ca61 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,9 +90,10 @@ if ( MSVC ) # Set CAres find_package(c-ares) - set(HAVE_CARES true) - set(zeekdeps ${zeekdeps} ${c-ares_LIBRARIES}) + set(HAVE_CARES true) # Disable FindCAres cmake file include_directories(BEFORE ${c-ares_INCLUDE_DIRS}) + set(zeekdeps ${zeekdeps} ${c-ares_LIBRARIES}) + add_definitions(-DCARES_STATICLIB) add_subdirectory(auxil/libunistd) set(UNISTD_INCLUDES ${CMAKE_SOURCE_DIR}/auxil/libunistd/unistd ${CMAKE_SOURCE_DIR}/auxil/libunistd/regex) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0823b83aeb..d76a47d337 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -533,6 +533,7 @@ if (ZEEK_STANDALONE) ${bro_PLUGIN_LIBS} ) target_link_libraries(zeek ${bro_PLUGIN_LINK_LIBS} ${zeekdeps} ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) + # Export symbols from zeek executable for use by plugins set_target_properties(zeek PROPERTIES ENABLE_EXPORTS TRUE) diff --git a/src/telemetry/CMakeLists.txt b/src/telemetry/CMakeLists.txt index c4b8359a48..129f49effb 100644 --- a/src/telemetry/CMakeLists.txt +++ b/src/telemetry/CMakeLists.txt @@ -13,3 +13,4 @@ bif_target(telemetry.bif) bro_add_subdir_library(telemetry ${telemetry_SRCS}) add_dependencies(bro_telemetry generate_outputs) + From 45fa4c0dc4f519d015f76c451a150beb6d1284bd Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 21 Oct 2022 15:51:21 -0700 Subject: [PATCH 34/71] Disable Manager::ActivateDynamicPluginInternal if building on Windows --- src/plugin/Manager.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index 8830cc9f65..ec0ef1530f 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -161,6 +161,10 @@ void Manager::SearchDynamicPlugins(const std::string& dir) bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_not_found, std::vector* errors) { +#if defined(_MSC_VER) + return false; +#endif + errors->clear(); // caller should pass it in empty, but just to be sure dynamic_plugin_map::iterator m = dynamic_plugins.find(util::strtolower(name)); From 77c555a3a878d6af0a7729646b9aa229917e7c4d Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 24 Oct 2022 15:20:54 -0700 Subject: [PATCH 35/71] Fixing some issues from rebasing --- CMakeLists.txt | 8 ---- src/CMakeLists.txt | 24 ++-------- src/DNS_Mgr.cc | 3 +- src/DebugLogger.h | 10 ++-- src/Hash.h | 2 +- src/IntrusivePtr.h | 15 +++++- src/Options.cc | 8 ++-- src/SerializationFormat.cc | 7 --- src/SerializationFormat.h | 3 -- src/Val.cc | 7 ++- src/analyzer/Manager.h | 2 +- src/analyzer/protocol/dns/DNS.cc | 10 ++-- src/input/readers/CMakeLists.txt | 2 +- src/iosource/pcap/Dumper.cc | 2 +- src/main.cc | 4 +- src/plugin/Manager.cc | 80 +++++++++++++------------------- src/supervisor/Supervisor.h | 4 ++ src/util.cc | 37 ++++----------- src/util.h | 4 -- src/zeek-setup.cc | 6 +-- src/zeekygen/Target.cc | 5 -- 21 files changed, 85 insertions(+), 158 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e52092ca61..9af117bddd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,8 +16,6 @@ list(APPEND CMAKE_PREFIX_PATH ${CMAKE_BINARY_DIR}) # Windows: Configure runtime and dependencies if ( MSVC ) - cmake_policy(SET CMP0091 NEW) - # Remove existing runtime flags set(CompilerFlags CMAKE_CXX_FLAGS @@ -82,12 +80,6 @@ if ( MSVC ) find_package(ZLIB) set(ZLIB_LIBRARY ${ZLIB_LIBRARIES}) - # Set Kqueue to point at the right variable. - find_package(kqueue) - set(LIBKQUEUE_ROOT_DIR "${kqueue_INCLUDES}/../") - set(LIBKQUEUE_INCLUDE_DIRS ${kqueue_INCLUDES}) - set(LIBKQUEUE_LIBRARIES ${kqueue_LIBS}) - # Set CAres find_package(c-ares) set(HAVE_CARES true) # Disable FindCAres cmake file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d76a47d337..2a62e009c9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -527,10 +527,10 @@ add_library(zeek_objs OBJECT ${zeek_SRCS}) if (ZEEK_STANDALONE) add_executable(zeek main.cc - $ - ${zeek_HEADERS} - ${bro_SUBDIR_LIBS} - ${bro_PLUGIN_LIBS} + $ + ${zeek_HEADERS} + ${bro_SUBDIR_LIBS} + ${bro_PLUGIN_LIBS} ) target_link_libraries(zeek ${bro_PLUGIN_LINK_LIBS} ${zeekdeps} ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) @@ -569,10 +569,6 @@ if (NOT ZEEK_STANDALONE OR CONAN_EXPORTED) ${CMAKE_BINARY_DIR}/zeek/src ${CMAKE_BINARY_DIR}/zeek/src/include) - if ( WIN32 ) - target_include_directories(libzeek PUBLIC ${CMAKE_SOURCE_DIR}/zeek/src/windows/usr.include) - endif() - install(TARGETS libzeek LIBRARY DESTINATION lib) endif() @@ -582,13 +578,6 @@ if ( NOT WIN32 ) InstallSymlink("${CMAKE_INSTALL_PREFIX}/bin/zeek-wrapper" "${CMAKE_INSTALL_PREFIX}/bin/bro") endif () -if ( NOT BINARY_PACKAGING_MODE ) - # Older plugins may still use `bro` in unit tests. - execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink - "${CMAKE_CURRENT_BINARY_DIR}/../zeek-wrapper.in" - "${CMAKE_CURRENT_BINARY_DIR}/bro") -endif () - # Target to create all the autogenerated files. add_custom_target(generate_outputs_stage1) add_dependencies(generate_outputs_stage1 ${bro_ALL_GENERATED_OUTPUTS}) @@ -635,7 +624,6 @@ install(CODE " # Make sure to escape a bunch of special characters in the path before trying to use it as a # regular expression below. string(REGEX REPLACE "([][+.*()^])" "\\\\\\1" escaped_include_path "${CMAKE_CURRENT_SOURCE_DIR}/include/*") -string(REGEX REPLACE "([][+.*()^])" "\\\\\\1" escaped_windows_path "${CMAKE_CURRENT_SOURCE_DIR}/windows/*") if (WIN32) install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/windows/usr.include/ @@ -653,8 +641,6 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ PATTERN "3rdparty/*" EXCLUDE # The "zeek -> ." symlink isn't needed in the install-tree REGEX "${escaped_include_path}$" EXCLUDE - # Windows headers are already installed if needed - REGEX "${escaped_windows_path}$" EXCLUDE # FILES_MATCHING creates empty directories: # https://gitlab.kitware.com/cmake/cmake/-/issues/17122 @@ -671,7 +657,7 @@ install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ PATTERN "*.bif.h" PATTERN "CMakeFiles" EXCLUDE # The "include/zeek -> .." symlink isn't needed in the install-tree - REGEX "${escaped_path}$" EXCLUDE + REGEX "${escaped_include_path}$" EXCLUDE ) install(FILES diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index c256b5304d..daaf1d17e2 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -1,7 +1,5 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include - #include "zeek/DNS_Mgr.h" #include "zeek/zeek-config.h" @@ -12,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/src/DebugLogger.h b/src/DebugLogger.h index e60c1b8c3c..6b4daf73c2 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -13,12 +13,12 @@ #include "zeek/util.h" -#define DBG_LOG(stream, ...) \ - if ( ::zeek::detail::debug_logger.IsEnabled(stream) ) \ +#define DBG_LOG(stream, ...) \ + if ( ::zeek::detail::debug_logger.IsEnabled(stream) ) \ ::zeek::detail::debug_logger.Log(stream, __VA_ARGS__) -#define DBG_LOG_VERBOSE(stream, ...) \ - if ( ::zeek::detail::debug_logger.IsVerbose() && \ - ::zeek::detail::debug_logger.IsEnabled(stream) ) \ +#define DBG_LOG_VERBOSE(stream, ...) \ + if ( ::zeek::detail::debug_logger.IsVerbose() && \ + ::zeek::detail::debug_logger.IsEnabled(stream) ) \ ::zeek::detail::debug_logger.Log(stream, __VA_ARGS__) #define DBG_PUSH(stream) ::zeek::detail::debug_logger.PushIndent(stream) #define DBG_POP(stream) ::zeek::detail::debug_logger.PopIndent(stream) diff --git a/src/Hash.h b/src/Hash.h index 06ec01ae0b..796763d509 100644 --- a/src/Hash.h +++ b/src/Hash.h @@ -19,8 +19,8 @@ #pragma once -#include #include +#include #include "zeek/util.h" // for zeek_int_t diff --git a/src/IntrusivePtr.h b/src/IntrusivePtr.h index 2d8767cb0d..bdeb501e85 100644 --- a/src/IntrusivePtr.h +++ b/src/IntrusivePtr.h @@ -27,6 +27,12 @@ struct NewRef { }; +/** + * This has to be forward decalred and known here in order for us to be able + * cast this in the `Unref` function. + */ +class OpaqueVal; + /** * An intrusive, reference counting smart pointer implementation. Much like * @c std::shared_ptr, this smart pointer models shared ownership of an object @@ -113,7 +119,14 @@ public: ~IntrusivePtr() { if ( ptr_ ) - Unref((zeek::Obj*)ptr_); + { + // Specializing `OpaqueVal` as MSVC compiler does not detect it + // inheriting from `zeek::Obj` so we have to do that manually. + if constexpr ( std::is_same_v ) + Unref(reinterpret_cast(ptr_)); + else + Unref(ptr_); + } } void swap(IntrusivePtr& other) noexcept { std::swap(ptr_, other.ptr_); } diff --git a/src/Options.cc b/src/Options.cc index 458a52f93b..16c5dd8d1b 100644 --- a/src/Options.cc +++ b/src/Options.cc @@ -1,23 +1,21 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include - #include "zeek/Options.h" #include "zeek/zeek-config.h" -#include "zeek/ScriptProfile.h" -#include "zeek/script_opt/ScriptOpt.h" - #if defined(HAVE_GETOPT_H) && ! defined(_MSC_VER) #include #endif +#include #include #include #include #include +#include "zeek/3rdparty/bsd-getopt-long.h" +#include "zeek/ScriptProfile.h" #include "zeek/logging/writers/ascii/Ascii.h" #include "zeek/script_opt/ScriptOpt.h" diff --git a/src/SerializationFormat.cc b/src/SerializationFormat.cc index b723387ceb..f8c67a3c52 100644 --- a/src/SerializationFormat.cc +++ b/src/SerializationFormat.cc @@ -325,13 +325,6 @@ bool BinarySerializationFormat::Write(uint32_t v, const char* tag) return WriteData(&v, sizeof(v)); } -#if defined(_MSC_VER) -bool BinarySerializationFormat::Write(u_long v, const char* tag) - { - return Write((uint32_t) v, tag); - } -#endif - bool BinarySerializationFormat::Write(int v, const char* tag) { DBG_LOG(DBG_SERIAL, "Write int %d [%s]", v, tag); diff --git a/src/SerializationFormat.h b/src/SerializationFormat.h index 060d0a996d..44215792bb 100644 --- a/src/SerializationFormat.h +++ b/src/SerializationFormat.h @@ -126,9 +126,6 @@ public: bool Write(int v, const char* tag) override; bool Write(uint16_t v, const char* tag) override; bool Write(uint32_t v, const char* tag) override; -#if defined(_MSC_VER) - bool Write(u_long v, const char* tag); -#endif bool Write(int64_t v, const char* tag) override; bool Write(uint64_t v, const char* tag) override; bool Write(char v, const char* tag) override; diff --git a/src/Val.cc b/src/Val.cc index 9c3db80752..fc70ef3bdb 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -3973,12 +3973,11 @@ const PortValPtr& ValManager::Port(uint32_t port_num, TransportProto port_type) port_num = 0; } - std::pair key{port_num, port_type}; - if (ports.find(key) == ports.end()) - { + if ( ports.find(key) == ports.end() ) + { ports[key] = IntrusivePtr{AdoptRef{}, new PortVal(PortVal::Mask(port_num, port_type))}; - } + } return ports[key]; } diff --git a/src/analyzer/Manager.h b/src/analyzer/Manager.h index f33241881d..3cedb549be 100644 --- a/src/analyzer/Manager.h +++ b/src/analyzer/Manager.h @@ -402,7 +402,7 @@ extern analyzer::Manager* analyzer_mgr; DBG_LOG(zeek::DBG_ANALYZER, "%s " txt, \ fmt_conn_id(conn->OrigAddr(), ntohs(conn->OrigPort()), conn->RespAddr(), \ ntohs(conn->RespPort()))); -#define DBG_ANALYZER_ARGS(conn, fmt, ...) \ +#define DBG_ANALYZER_ARGS(conn, fmt, ...) \ DBG_LOG(zeek::DBG_ANALYZER, "%s " fmt, \ fmt_conn_id(conn->OrigAddr(), ntohs(conn->OrigPort()), conn->RespAddr(), \ ntohs(conn->RespPort())), \ diff --git a/src/analyzer/protocol/dns/DNS.cc b/src/analyzer/protocol/dns/DNS.cc index fa22dabbb2..8bb0e76fd0 100644 --- a/src/analyzer/protocol/dns/DNS.cc +++ b/src/analyzer/protocol/dns/DNS.cc @@ -823,8 +823,7 @@ bool DNS_Interpreter::ParseRR_EDNS(detail::DNS_MsgInfo* msg, const u_char*& data case TYPE_TCP_KA: { - EDNS_TCP_KEEPALIVE edns_tcp_keepalive{.keepalive_timeout_omitted = true, - .keepalive_timeout = 0}; + EDNS_TCP_KEEPALIVE edns_tcp_keepalive{true, 0}; if ( option_len == 0 || option_len == 2 ) { // 0 bytes is permitted by RFC 7828, showing that the timeout value is @@ -1736,11 +1735,8 @@ bool DNS_Interpreter::ParseRR_SVCB(detail::DNS_MsgInfo* msg, const u_char*& data name_end = target_name + 1; } - SVCB_DATA svcb_data = { - .svc_priority = svc_priority, - .target_name = make_intrusive( - new String(target_name, name_end - target_name, true)), - }; + SVCB_DATA svcb_data = {svc_priority, make_intrusive(new String( + target_name, name_end - target_name, true))}; // TODO: parse svcparams // we consume all the remaining raw data (svc params) but do nothing. diff --git a/src/input/readers/CMakeLists.txt b/src/input/readers/CMakeLists.txt index c82f175f08..032a286052 100644 --- a/src/input/readers/CMakeLists.txt +++ b/src/input/readers/CMakeLists.txt @@ -5,5 +5,5 @@ add_subdirectory(binary) add_subdirectory(config) add_subdirectory(raw) if (USE_SQLITE) - add_subdirectory(sqlite) + add_subdirectory(sqlite) endif() diff --git a/src/iosource/pcap/Dumper.cc b/src/iosource/pcap/Dumper.cc index 3e7b5e32bb..24a5628fec 100644 --- a/src/iosource/pcap/Dumper.cc +++ b/src/iosource/pcap/Dumper.cc @@ -107,7 +107,7 @@ bool PcapDumper::Dump(const Packet* pkt) return false; // Reconstitute the pcap_pkthdr. - const struct pcap_pkthdr phdr = {.ts = pkt->ts, .caplen = pkt->cap_len, .len = pkt->len}; + const struct pcap_pkthdr phdr = {pkt->ts, pkt->cap_len, pkt->len}; pcap_dump((u_char*)dumper, &phdr, pkt->data); pcap_dump_flush(dumper); diff --git a/src/main.cc b/src/main.cc index aeb15627f5..2796ebf1ad 100644 --- a/src/main.cc +++ b/src/main.cc @@ -1,9 +1,9 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include - #include "zeek/zeek-config.h" +#include + #include "zeek/RunState.h" #include "zeek/iosource/Manager.h" #include "zeek/supervisor/Supervisor.h" diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index ec0ef1530f..43eb1daee0 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -5,6 +5,7 @@ #include #if !defined(_MSC_VER) #include +#include #endif #include #include @@ -163,8 +164,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ { #if defined(_MSC_VER) return false; -#endif - +#else errors->clear(); // caller should pass it in empty, but just to be sure dynamic_plugin_map::iterator m = dynamic_plugins.find(util::strtolower(name)); @@ -218,51 +218,30 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ } // Load shared libraries. - string dydir = dir + "/lib"; - const char *dyext = "." HOST_ARCHITECTURE DYNAMIC_PLUGIN_SUFFIX; - DBG_LOG(DBG_PLUGINS, " Searching for shared libraries in %s with extension %s", dydir.c_str(), dyext); + string dypattern = dir + "/lib/*." + HOST_ARCHITECTURE + DYNAMIC_PLUGIN_SUFFIX; - DIR* d = opendir(dydir.c_str()); + DBG_LOG(DBG_PLUGINS, " Searching for shared libraries %s", dypattern.c_str()); - if ( ! d ) + glob_t gl; + + if ( glob(dypattern.c_str(), 0, 0, &gl) == 0 ) { - DBG_LOG(DBG_PLUGINS, "Cannot open directory %s", dydir.c_str()); - return true; - } - - struct dirent *dp; - - while ( (dp = readdir(d)) ) - { - if ( strlen(dp->d_name) >= strlen(dyext) - && zeek::util::streq(dp->d_name + strlen(dp->d_name) - strlen(dyext), dyext) ) + for ( size_t i = 0; i < gl.gl_pathc; i++ ) { - string path = dydir + "/" + dp->d_name; + const char* path = gl.gl_pathv[i]; - current_plugin = nullptr; - current_dir = dydir.c_str(); - current_sopath = path.c_str(); -#if defined(_MSC_VER) - void* hdl = LoadLibraryA(path.c_str()); -#else - void* hdl = dlopen(path.c_str(), RTLD_LAZY | RTLD_GLOBAL); -#endif - current_dir = nullptr; - current_sopath = nullptr; + current_plugin = nullptr; + current_dir = dir.c_str(); + current_sopath = path; + void* hdl = dlopen(path, RTLD_NOW | RTLD_GLOBAL); + current_dir = nullptr; + current_sopath = nullptr; if ( ! hdl ) { - const char* err = nullptr; -#if defined(_MSC_VER) - char buf[65535]; - const int flags = FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS; - if ( FormatMessageA(flags, nullptr, GetLastError(), 0, buf, sizeof(buf), nullptr ) ) - err = buf; -#else - err = dlerror(); -#endif - errors->push_back(util::fmt("cannot load plugin library %s: %s", path.c_str(), + const char* err = dlerror(); + errors->push_back(util::fmt("cannot load plugin library %s: %s", path, err ? err : "")); continue; } @@ -270,7 +249,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ if ( ! current_plugin ) { errors->push_back( - util::fmt("load plugin library %s did not instantiate a plugin", path.c_str())); + util::fmt("load plugin library %s did not instantiate a plugin", path)); continue; } @@ -282,10 +261,10 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ plugins_by_path.insert( std::make_pair(util::detail::normalize_path(dir), current_plugin)); - // We execute the pre-script initialization here; this in - // fact could be *during* script initialization if we got - // triggered via @load-plugin. - current_plugin->InitPreScript(); + // We execute the pre-script initialization here; this in + // fact could be *during* script initialization if we got + // triggered via @load-plugin. + current_plugin->InitPreScript(); // Make sure the name the plugin reports is consistent with // what we expect from its magic file. @@ -297,21 +276,20 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ } current_plugin = nullptr; - DBG_LOG(DBG_PLUGINS, " Loaded %s", path.c_str()); + DBG_LOG(DBG_PLUGINS, " Loaded %s", path); } + globfree(&gl); + if ( ! errors->empty() ) return false; } - closedir(d); - - if ( current_plugin == nullptr ) + else { DBG_LOG(DBG_PLUGINS, " No shared library found"); } - // Add the "scripts" and "bif" directories to ZEEKPATH. std::string scripts = dir + "scripts"; @@ -353,6 +331,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ m->second.clear(); return true; +#endif } void Manager::ActivateDynamicPlugin(const std::string& name) @@ -373,7 +352,10 @@ void Manager::ActivateDynamicPlugins(bool all) // Activate plugins that were specifically requested. for ( const auto& x : requested_plugins ) - plugins_to_activate.emplace(x, false); + { + if ( ! x.empty() ) + plugins_to_activate.emplace(x, false); + } // Activate plugins that our environment tells us to. vector p; diff --git a/src/supervisor/Supervisor.h b/src/supervisor/Supervisor.h index c0812dd7d4..d44afe51d5 100644 --- a/src/supervisor/Supervisor.h +++ b/src/supervisor/Supervisor.h @@ -153,7 +153,11 @@ public: // themselves trigger deprecation warnings for accessing the // "scripts" field. It can go when we remove that deprecation. NodeConfig() = default; +#ifndef _MSC_VER + // MSVC throws this error when specifing this constructor: + // error C2580: multiple versions of a defaulted special member functions are not allowed NodeConfig(NodeConfig&) = default; +#endif NodeConfig(const NodeConfig&) = default; NodeConfig(NodeConfig&&) = default; ~NodeConfig() = default; diff --git a/src/util.cc b/src/util.cc index d62b05e304..5929c59f28 100644 --- a/src/util.cc +++ b/src/util.cc @@ -43,7 +43,6 @@ #include #include #include -#include #include #include "zeek/3rdparty/ConvertUTF.h" @@ -259,16 +258,6 @@ const char* fmt_access_time(double t) time_t time = (time_t)t; struct tm ts; - if (!time) - { - // Use wall clock. - struct timeval tv = { 0 }; - if (gettimeofday(&tv, 0) < 0) - reporter->InternalError("unable to gettimeofday"); - else - time = tv.tv_sec; - } - if ( ! localtime_r(&time, &ts) ) { reporter->InternalError("unable to get time"); @@ -447,7 +436,6 @@ void init_random_seed(const char* read_file, const char* write_file, bool use_em pos += sizeof(struct timeval) / sizeof(uint32_t); // use urandom. For reasons see e.g. http://www.2uo.de/myths-about-urandom/ -#ifndef _MSC_VER #if defined(O_NONBLOCK) int fd = open("/dev/urandom", O_RDONLY | O_NONBLOCK); #elif defined(O_NDELAY) @@ -470,12 +458,6 @@ void init_random_seed(const char* read_file, const char* write_file, bool use_em errno = 0; } #endif - // C++ random device implementation in MSVC is sufficient for this purpose. - thread_local std::mt19937 gen(std::random_device{}()); - while ( pos < zeek::detail::KeyedHash::SEED_INIT_SIZE ) { - buf[pos++] = (uint32_t)gen(); - } -#endif if ( pos < zeek::detail::KeyedHash::SEED_INIT_SIZE ) reporter->FatalError("Could not read enough random data. Wanted %d, got %zu", @@ -1638,17 +1620,16 @@ const char* vfmt(const char* format, va_list al) va_copy(alc, al); int n = vsnprintf(buf, buf_len, format, al); - if ( (unsigned int)n >= buf_len ) + if ( n < 0 && buf_len < 1024 * 1024 ) { // Not enough room, grow the buffer. - buf_len = n + 32; + buf_len += 32; buf = (char*)safe_realloc(buf, buf_len); - - n = vsnprintf(buf, buf_len, format, alc); - - if ( (unsigned int)n >= buf_len ) - reporter->InternalError("confusion reformatting in fmt()"); + n = vsnprintf(buf, buf_len, format, al); } + if ( n < 0 ) + reporter->InternalError("confusion reformatting in fmt()"); + va_end(alc); return buf; } @@ -1792,7 +1773,7 @@ string zeek_prefixes() for ( const auto& prefix : zeek::detail::zeek_script_prefixes ) { if ( ! rval.empty() ) - rval.append(zeek_path_list_separator); + rval.append(path_list_separator); rval.append(prefix); } @@ -2006,7 +1987,7 @@ static string find_file_in_path(const string& filename, const string& path, string find_file(const string& filename, const string& path_set, const string& opt_ext) { vector paths; - tokenize_string(path_set, zeek_path_list_separator, &paths); + tokenize_string(path_set, path_list_separator, &paths); vector ext; if ( ! opt_ext.empty() ) @@ -2026,7 +2007,7 @@ string find_file(const string& filename, const string& path_set, const string& o string find_script_file(const string& filename, const string& path_set) { vector paths; - tokenize_string(path_set, zeek_path_list_separator, &paths); + tokenize_string(path_set, path_list_separator, &paths); vector ext = {".zeek"}; diff --git a/src/util.h b/src/util.h index c7fa0ceab5..25537f2d18 100644 --- a/src/util.h +++ b/src/util.h @@ -40,10 +40,6 @@ #endif #endif -#ifdef _MSC_VER -#include -#endif - #ifdef DEBUG #include diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index b718988b82..fbef570fca 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -52,7 +52,6 @@ #include "zeek/Traverse.h" #include "zeek/Trigger.h" #include "zeek/Var.h" - #include "zeek/analyzer/Manager.h" #include "zeek/binpac_zeek.h" #include "zeek/broker/Manager.h" @@ -67,7 +66,6 @@ #include "zeek/plugin/Manager.h" #include "zeek/script_opt/ScriptOpt.h" #include "zeek/session/Manager.h" -#include "zeek/script_opt/ScriptOpt.h" #include "zeek/supervisor/Supervisor.h" #include "zeek/telemetry/Manager.h" #include "zeek/threading/Manager.h" @@ -210,6 +208,7 @@ char version[] = VERSION; #else extern char version[]; #endif + const char* zeek::detail::command_line_policy = nullptr; vector zeek::detail::params; set requested_plugins; @@ -245,9 +244,6 @@ char** zeek::detail::zeek_argv; namespace zeek { -// Define zeek version explicitly for MSVC - - const char* zeek_version() { #ifdef DEBUG diff --git a/src/zeekygen/Target.cc b/src/zeekygen/Target.cc index eba9c81c83..5236b8b070 100644 --- a/src/zeekygen/Target.cc +++ b/src/zeekygen/Target.cc @@ -2,9 +2,7 @@ #include "zeek/zeekygen/Target.h" -#ifndef _MSC_VER #include -#endif #include #include #include @@ -492,8 +490,6 @@ vector dir_contents_recursive(string dir) scan_path[0] = dir_copy; scan_path[1] = NULL; - // Zeekygen isn't supported in Windows, due to missing FTS library. -#ifndef _MSC_VER FTS* fts = fts_open(scan_path, FTS_NOCHDIR, 0); if ( ! fts ) @@ -520,7 +516,6 @@ vector dir_contents_recursive(string dir) delete[] scan_path; delete[] dir_copy; -#endif return rval; } From 78fb845f63dd9c7d6e1c40ecb3298fcbb0824b75 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 24 Oct 2022 17:01:45 -0700 Subject: [PATCH 36/71] Rework change to util::vfmt to fix strings getting truncated --- src/util.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/util.cc b/src/util.cc index 5929c59f28..41b6410d23 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1620,11 +1620,11 @@ const char* vfmt(const char* format, va_list al) va_copy(alc, al); int n = vsnprintf(buf, buf_len, format, al); - if ( n < 0 && buf_len < 1024 * 1024 ) + if ( n > 0 && buf_len < n ) { // Not enough room, grow the buffer. - buf_len += 32; + buf_len = n + 32; buf = (char*)safe_realloc(buf, buf_len); - n = vsnprintf(buf, buf_len, format, al); + n = vsnprintf(buf, buf_len, format, alc); } if ( n < 0 ) From 71540c5eff97da049876cc7f550f9efa37f37cc5 Mon Sep 17 00:00:00 2001 From: Tomer Lev Date: Wed, 26 Oct 2022 15:00:08 +0300 Subject: [PATCH 37/71] introduced pthread.h include in util.h for MSVC. Squashed warnning originated from this file and also merged a few ifdefs into one place --- src/util.h | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/src/util.h b/src/util.h index 25537f2d18..6471b8019a 100644 --- a/src/util.h +++ b/src/util.h @@ -32,13 +32,11 @@ #ifdef TIME_WITH_SYS_TIME #include #include -#else -#ifdef HAVE_SYS_TIME_H +#elif defined(HAVE_SYS_TIME_H) #include #else #include #endif -#endif #ifdef DEBUG @@ -49,6 +47,9 @@ #define DEBUG_fputs fputs #else +#ifdef MSTCPIP_ASSERT_UNDEFINED +#undef ASSERT +#endif #define ASSERT(x) #define DEBUG_MSG(...) @@ -64,6 +65,11 @@ extern HeapLeakChecker* heap_checker; #include +extern "C" + { +#include "zeek/3rdparty/modp_numtoa.h" + } + #ifdef HAVE_LINUX #include #include @@ -73,21 +79,17 @@ extern HeapLeakChecker* heap_checker; #include #endif -#if defined(_MSC_VER) +#ifdef _MSC_VER +#include +#include +namespace zeek { namespace filesystem = std::filesystem; } inline constexpr std::string_view path_list_separator = ";"; #else -inline constexpr std::string_view path_list_separator = ":"; -#endif - -extern "C" - { -#include "zeek/3rdparty/modp_numtoa.h" - } - -#if defined(_MSC_VER) -#include -#else +// Expose ghc::filesystem as zeek::filesystem until we can +// switch to std::filesystem on all platforms. #include "zeek/3rdparty/ghc/filesystem.hpp" +namespace zeek { namespace filesystem = ghc::filesystem; } +inline constexpr std::string_view path_list_separator = ":"; #endif using zeek_int_t = int64_t; @@ -108,13 +110,7 @@ namespace zeek class ODesc; class RecordVal; -// Expose ghc::filesystem as zeek::filesystem until we can -// switch to std::filesystem on all platforms. -#if defined(_MSC_VER) -namespace filesystem = std::filesystem; -#else -namespace filesystem = ghc::filesystem; -#endif + namespace util { From 9535729949c05f816164316efc63871898d537db Mon Sep 17 00:00:00 2001 From: Tomer Lev Date: Wed, 26 Oct 2022 16:44:37 +0300 Subject: [PATCH 38/71] Creating a symlink src/include/zeek -> src/ --- .gitignore | 2 ++ src/include/.gitkeep | 1 + 2 files changed, 3 insertions(+) create mode 100644 src/include/.gitkeep diff --git a/.gitignore b/.gitignore index 5a24d31e0f..9a7412b64b 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ out/ .vs/ .vscode/ CMakeSettings.json + +src/include \ No newline at end of file diff --git a/src/include/.gitkeep b/src/include/.gitkeep new file mode 100644 index 0000000000..d3f5a12faa --- /dev/null +++ b/src/include/.gitkeep @@ -0,0 +1 @@ + From f35c66ebc7a2ef3f7b9c3a3fb9ad9e00ce8e50a9 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 26 Oct 2022 09:55:53 -0700 Subject: [PATCH 39/71] Updating submodules [nomail] --- auxil/libunistd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auxil/libunistd b/auxil/libunistd index cc7ac8d20a..7555c4708e 160000 --- a/auxil/libunistd +++ b/auxil/libunistd @@ -1 +1 @@ -Subproject commit cc7ac8d20a28905be963ae8f5684e793e198892a +Subproject commit 7555c4708e7e7bc408cd129215682acdd839c840 From e39e1df0a2c817cd24b5a8a662b4d3e6d5e09046 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 26 Oct 2022 15:57:03 -0700 Subject: [PATCH 40/71] Ensure src/include exists and add a comment --- src/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2a62e009c9..b10064fcb2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,7 +5,10 @@ include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}/include ) -# Force creation of includes symlink +# Force creation of includes symlink. This can't just be in the src directory +# because MSVC will end up with an include loop. +execute_process(COMMAND "${CMAKE_COMMAND}" -E make_directory + "${CMAKE_CURRENT_SOURCE_DIR}/include") execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink ".." "${CMAKE_CURRENT_SOURCE_DIR}/include/zeek") From 90dc9509f71fd0d54fad92951ab403a7efe285a6 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 26 Oct 2022 16:33:15 -0700 Subject: [PATCH 41/71] Changes needed to make plugin btests pass on UNIX platforms --- testing/btest/Baseline/plugins.hooks/output | 6 ++++++ .../btest/Baseline/plugins.plugin-load-dependency/output | 1 + 2 files changed, 7 insertions(+) diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index f6098e74b3..dfa36e5ae6 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -935,6 +935,7 @@ 0.000000 MetaHookPost LoadFile(0, ./site, <...>/site.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./smb1-main, <...>/smb1-main.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./smb2-main, <...>/smb2-main.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./statistics.bif.zeek, <...>/statistics.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./std-dev, <...>/std-dev.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./store, <...>/store.zeek) -> -1 @@ -1320,6 +1321,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./site, <...>/site.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./smb1-main, <...>/smb1-main.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./smb2-main, <...>/smb2-main.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./statistics.bif.zeek, <...>/statistics.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./std-dev, <...>/std-dev.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./store, <...>/store.zeek) -> (-1, ) @@ -2451,6 +2453,7 @@ 0.000000 MetaHookPre LoadFile(0, ./site, <...>/site.zeek) 0.000000 MetaHookPre LoadFile(0, ./smb1-main, <...>/smb1-main.zeek) 0.000000 MetaHookPre LoadFile(0, ./smb2-main, <...>/smb2-main.zeek) +0.000000 MetaHookPre LoadFile(0, ./statistics.bif.zeek, <...>/statistics.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./stats.bif.zeek, <...>/stats.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./std-dev, <...>/std-dev.zeek) 0.000000 MetaHookPre LoadFile(0, ./store, <...>/store.zeek) @@ -2836,6 +2839,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./site, <...>/site.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./smb1-main, <...>/smb1-main.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./smb2-main, <...>/smb2-main.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./statistics.bif.zeek, <...>/statistics.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./stats.bif.zeek, <...>/stats.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./std-dev, <...>/std-dev.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./store, <...>/store.zeek) @@ -3977,6 +3981,7 @@ 0.000000 | HookLoadFile ./site <...>/site.zeek 0.000000 | HookLoadFile ./smb1-main <...>/smb1-main.zeek 0.000000 | HookLoadFile ./smb2-main <...>/smb2-main.zeek +0.000000 | HookLoadFile ./statistics.bif.zeek <...>/statistics.bif.zeek 0.000000 | HookLoadFile ./stats.bif.zeek <...>/stats.bif.zeek 0.000000 | HookLoadFile ./std-dev <...>/std-dev.zeek 0.000000 | HookLoadFile ./store <...>/store.zeek @@ -4362,6 +4367,7 @@ 0.000000 | HookLoadFileExtended ./site <...>/site.zeek 0.000000 | HookLoadFileExtended ./smb1-main <...>/smb1-main.zeek 0.000000 | HookLoadFileExtended ./smb2-main <...>/smb2-main.zeek +0.000000 | HookLoadFileExtended ./statistics.bif.zeek <...>/statistics.bif.zeek 0.000000 | HookLoadFileExtended ./stats.bif.zeek <...>/stats.bif.zeek 0.000000 | HookLoadFileExtended ./std-dev <...>/std-dev.zeek 0.000000 | HookLoadFileExtended ./store <...>/store.zeek diff --git a/testing/btest/Baseline/plugins.plugin-load-dependency/output b/testing/btest/Baseline/plugins.plugin-load-dependency/output index e788232bd8..5d6ae02103 100644 --- a/testing/btest/Baseline/plugins.plugin-load-dependency/output +++ b/testing/btest/Baseline/plugins.plugin-load-dependency/output @@ -9,6 +9,7 @@ Testing::Plugin2 - Plugin2 provides a load dependency for Plugin1 and Plugin3 (d in Plugin1 in Plugin2 +Statistics - Statistics module (built-in) Testing::Plugin1 - Plugin1 has a load dependency on Plugin2 (dynamic, version 1.0.0) Testing::Plugin2 - Plugin2 provides a load dependency for Plugin1 and Plugin3 (dynamic, version 1.0.0) Testing::Plugin3 - Plugin3 has a load dependency on Plugin2 (dynamic, version 1.0.0) From 6307158b0436b01ba93f9e194ac40e5dd37cfe4d Mon Sep 17 00:00:00 2001 From: Tomer Lev Date: Thu, 27 Oct 2022 11:21:16 +0300 Subject: [PATCH 42/71] Updating dependencies in the dockerfile and removing redundant from conanfile.txt --- ci/windows/CMakePresets.json | 67 ------------------------- ci/windows/Dockerfile | 7 ++- ci/windows/Initialize-MSVC.ps1 | 73 ---------------------------- ci/windows/Install-Prerequisites.ps1 | 12 ----- ci/windows/conanfile_windows.txt | 1 - 5 files changed, 3 insertions(+), 157 deletions(-) delete mode 100644 ci/windows/CMakePresets.json delete mode 100644 ci/windows/Initialize-MSVC.ps1 delete mode 100644 ci/windows/Install-Prerequisites.ps1 diff --git a/ci/windows/CMakePresets.json b/ci/windows/CMakePresets.json deleted file mode 100644 index 515f9df339..0000000000 --- a/ci/windows/CMakePresets.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "version": 2, - "configurePresets": [ - { - "name": "base", - "hidden": true, - "description": "Base preset", - "generator": "Ninja", - "binaryDir": "${sourceDir}/out/build/${presetName}", - "cacheVariables": { - "CMAKE_C_COMPILER": "cl.exe", - "CMAKE_CXX_COMPILER": "cl.exe" - }, - "architecture": { - "value": "x64", - "strategy": "external" - }, - "toolset": { - "value": "host=x64,version=16.11", - "strategy": "external" - } - }, - { - "name": "x64-Debug", - "inherits": "base", - "displayName": "x64 Debug", - "description": "Sets debug build type and x64 arch", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug" - } - }, - { - "name": "x64-Release", - "displayName": "x64 Release", - "description": "Sets release build type", - "inherits": "x64-Debug", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release" - } - } - ], - "buildPresets": [ - { - "name": "default-debug-build", - "displayName": "Debug Build", - "configurePreset": "x64-Debug" - }, - { - "name": "verbose-debug-build", - "displayName": "Verbose Debug Build", - "configurePreset": "x64-Debug", - "nativeToolOptions": [ "-v" ] - }, - { - "name": "default-release-build", - "displayName": "Release Build", - "configurePreset": "x64-Release" - }, - { - "name": "verbose-release-build", - "displayName": "Verbose Release Build", - "configurePreset": "x64-Release", - "nativeToolOptions": [ "-v" ] - } - ], - "testPresets": [] -} \ No newline at end of file diff --git a/ci/windows/Dockerfile b/ci/windows/Dockerfile index 8e93fdd8af..86d204ca50 100644 --- a/ci/windows/Dockerfile +++ b/ci/windows/Dockerfile @@ -12,13 +12,12 @@ RUN [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePoin # Install prerequisites RUN choco install -y --no-progress visualstudio2019buildtools --version=16.11.11.0 RUN choco install -y --no-progress visualstudio2019-workload-vctools --version=1.0.0 --package-parameters '--add Microsoft.VisualStudio.Component.VC.ATLMFC' -RUN choco install powershell-core -y RUN choco install conan -y RUN choco install sed -y -RUN choco install winflexbison -y -# Can't install a proper msys2 because it caused console hanging in the container during docker build. +RUN choco install winflexbison3 -y RUN choco install msysgit -y RUN choco install python -y +RUN choco install openssl -y # Set working environment. SHELL [ "cmd", "/c" ] @@ -28,4 +27,4 @@ RUN mkdir C:\build WORKDIR C:\build # This entry point starts the developer command prompt and launches the PowerShell shell. -ENTRYPOINT ["C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "-arch=x64", "&&", "pwsh.exe", "-NoLogo", "-ExecutionPolicy", "Unrestricted"] \ No newline at end of file +ENTRYPOINT ["C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "-arch=x64", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Unrestricted"] \ No newline at end of file diff --git a/ci/windows/Initialize-MSVC.ps1 b/ci/windows/Initialize-MSVC.ps1 deleted file mode 100644 index 28b4a9611e..0000000000 --- a/ci/windows/Initialize-MSVC.ps1 +++ /dev/null @@ -1,73 +0,0 @@ -param ( - [string] $Architecture='x64' -) - -if (!(Get-Module VSSetup)) -{ - Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force - Install-Module VSSetup -Force -} - -function Get-VSInstallPath { - [CmdletBinding()] - param ( - [Parameter(Mandatory=$true)] - [int] $MaxVersion - ) - $latest = Get-VSSetupInstance ` - | where { $_.InstallationVersion.Major -le $MaxVersion } ` - | Select-VSSetupInstance -Require Microsoft.VisualStudio.VC.CMake -Latest - - if (!$latest) - { - throw [System.IO.FileNotFoundException]::new("No Visual Studio installation found that matches max version: $MaxVersion!") - } - return $latest.InstallationPath -} - -function Where-Program { - [CmdletBinding()] - param ( - [Parameter(Mandatory=$true,ValueFromPipeline=$true)] - [string] $Program - ) - process - { - return Get-Command $Program | select Source -ExpandProperty source | Split-Path -Parent - } -} - -function Persist-EnvironmentVariable { - [CmdletBinding()] - param ( - [Parameter(Mandatory=$true,ValueFromPipeline=$true)] - [string] $Name - ) - process - { - $value=[System.Environment]::GetEnvironmentVariable($Name) - [System.Environment]::SetEnvironmentVariable($Name, $value, [System.EnvironmentVariableTarget]::Machine) - } -} - -$VsInstallationPath = Get-VSInstallPath -MaxVersion 16 -Write-Host "Found VS installation: $VsInstallationPath" - -[array] $originalEnv = [System.Environment]::GetEnvironmentVariables().Keys - -Import-Module "$VsInstallationPath\Common7\Tools\Microsoft.VisualStudio.DevShell.dll" -Enter-VsDevShell -VsInstallPath $VsInstallationPath -SkipAutomaticLocation -DevCmdArguments "-arch=$Architecture" - -$Programs = 'cmake', 'Ninja', 'cl' -$programsPath = $Programs | Where-Program -Write-Host "Found paths for $($Programs -join ', '): $($programsPath -join ', ')" - -$newPath = "$($programsPath -join ';');${env:Path}" -Write-Host "Persisting new PATH: $newPath" -[System.Environment]::SetEnvironmentVariable('PATH', $newPath, [System.EnvironmentVariableTarget]::Machine) - -[array] $vsEnv = [System.Environment]::GetEnvironmentVariables().Keys -[array] $newEnv = $vsEnv | where { $_ -notin $originalEnv -and $_ -ne 'PATH' } - -Write-Host "Persisting new environment variables: $($newEnv -join ', ')" -$newEnv | Persist-EnvironmentVariable diff --git a/ci/windows/Install-Prerequisites.ps1 b/ci/windows/Install-Prerequisites.ps1 deleted file mode 100644 index 2d15cb9b35..0000000000 --- a/ci/windows/Install-Prerequisites.ps1 +++ /dev/null @@ -1,12 +0,0 @@ -[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072 -iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) - -choco install conan -y -choco install sed -y -choco install winflexbison -y -if (!(Get-Command python)) -{ - choco install python -y -} - -[System.Environment]::SetEnvironmentVariable('PATH', "C:\Program Files\Git\bin;${env:PATH}", [System.EnvironmentVariableTarget]::Machine) \ No newline at end of file diff --git a/ci/windows/conanfile_windows.txt b/ci/windows/conanfile_windows.txt index 79db1eccc4..cb4bc35ada 100644 --- a/ci/windows/conanfile_windows.txt +++ b/ci/windows/conanfile_windows.txt @@ -1,5 +1,4 @@ [requires] -openssl/1.1.1i zlib/1.2.11 libpcap/1.10.1 c-ares/1.18.1 From 88b3a449a777ab23a845131ed4b62d65e828c1e0 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 27 Oct 2022 14:52:21 -0700 Subject: [PATCH 43/71] Use the right avx/sse flags when building highwayhash --- src/CMakeLists.txt | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b10064fcb2..dafb4662f4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -489,10 +489,18 @@ elseif (${COMPILER_ARCHITECTURE} STREQUAL "power") ../auxil/highwayhash/highwayhash/hh_vsx.cc ) elseif(${COMPILER_ARCHITECTURE} STREQUAL "x86_64") + if (MSVC) + set(_avx_flag /arch:AVX2) + set(_sse_flag /d2archSSE42) + else() + set(_avx_flag -mavx2) + set(_sse_flag -msse4.1) + endif() + set_source_files_properties(../auxil/highwayhash/highwayhash/hh_avx2.cc PROPERTIES COMPILE_FLAGS - -mavx2) + ${_avx_flag}) set_source_files_properties(../auxil/highwayhash/highwayhash/hh_sse41.cc PROPERTIES COMPILE_FLAGS - -msse4.1) + ${_sse_flag}) list(APPEND HH_SRCS ../auxil/highwayhash/highwayhash/hh_avx2.cc From bf06cc9c2f9a17e9c2a434b8fe0201f9b691cfb4 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 27 Oct 2022 16:36:06 -0700 Subject: [PATCH 44/71] Turn on unit tests by default, and ifdef out a few of them --- CMakeLists.txt | 1 + src/DNS_Mapping.cc | 4 ++++ src/DNS_Mgr.cc | 3 +++ src/logging/writers/ascii/Ascii.cc | 7 ++++--- src/util.cc | 19 ++++++++++++++++++- 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9af117bddd..ac780c6841 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,7 @@ endif() project(Zeek C CXX) option(ZEEK_STANDALONE "Is Zeek compiled stand-alone or embedded in a parent project." ON) +option(ENABLE_ZEEK_UNIT_TESTS "Should the doctest unit tests be built?" ON) list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}) list(APPEND CMAKE_PREFIX_PATH ${CMAKE_BINARY_DIR}) diff --git a/src/DNS_Mapping.cc b/src/DNS_Mapping.cc index b83983aaaa..f5264f5b03 100644 --- a/src/DNS_Mapping.cc +++ b/src/DNS_Mapping.cc @@ -296,6 +296,9 @@ TEST_CASE("dns_mapping init addr") TEST_CASE("dns_mapping save reload") { + // TODO: this test uses fmemopen and mkdtemp, both of which aren't available on + // Windows. We'll have to figure out another way to do this test there. +#ifndef _MSC_VER IPAddr addr("1.2.3.4"); in4_addr in4; addr.CopyIPv4(&in4); @@ -356,6 +359,7 @@ TEST_CASE("dns_mapping save reload") CHECK(svh->ToStdString() == "testing.home"); delete[] he.h_name; +#endif } TEST_CASE("dns_mapping multiple addresses") diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index daaf1d17e2..70dff59df0 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -1613,6 +1613,8 @@ void TestDNS_Mgr::Process() TEST_CASE("dns_mgr priming" * doctest::skip(true)) { + // TODO: This test uses mkdtemp, which isn't available on Windows. +#ifndef _MSC_VER char prefix[] = "/tmp/zeek-unit-test-XXXXXX"; auto tmpdir = mkdtemp(prefix); @@ -1658,6 +1660,7 @@ TEST_CASE("dns_mgr priming" * doctest::skip(true)) // Clean up cache file and the temp directory unlink(mgr2.CacheFile().c_str()); rmdir(tmpdir); +#endif } TEST_CASE("dns_mgr alternate server" * doctest::skip(true)) diff --git a/src/logging/writers/ascii/Ascii.cc b/src/logging/writers/ascii/Ascii.cc index 13b9711b86..1e4ee3e8cf 100644 --- a/src/logging/writers/ascii/Ascii.cc +++ b/src/logging/writers/ascii/Ascii.cc @@ -110,12 +110,13 @@ static std::string prefix_basename_with(const std::string& path, const std::stri TEST_CASE("writers.ascii prefix_basename_with") { - CHECK(prefix_basename_with("", ".shadow.") == ".shadow."); - CHECK(prefix_basename_with("conn.log", ".shadow.") == ".shadow.conn.log"); - CHECK(prefix_basename_with("/conn.log", ".shadow.") == "/.shadow.conn.log"); +#ifdef _MSC_VER + // TODO: adapt this test to Windows paths +#else CHECK(prefix_basename_with("a/conn.log", ".shadow.") == "a/.shadow.conn.log"); CHECK(prefix_basename_with("/a/conn.log", ".shadow.") == "/a/.shadow.conn.log"); CHECK(prefix_basename_with("a/b/conn.log", ".shadow.") == "a/b/.shadow.conn.log"); +#endif } static std::optional parse_shadow_log(const std::string& fname) diff --git a/src/util.cc b/src/util.cc index 41b6410d23..7203deeef0 100644 --- a/src/util.cc +++ b/src/util.cc @@ -600,7 +600,12 @@ void SafePathOp::CheckValid(const char* op_result, const char* path, bool error_ TEST_CASE("util flatten_script_name") { CHECK(flatten_script_name("script", "some/path") == "some.path.script"); +#ifndef _MSC_VER + // TODO: this test fails on Windows because the implementation of dirname() in libunistd + // returns a trailing slash on paths, even tho the POSIX implementation doesn't. Commenting + // this out until we can fix that. CHECK(flatten_script_name("other/path/__load__.zeek", "some/path") == "some.path.other.path"); +#endif CHECK(flatten_script_name("path/to/script", "") == "path.to.script"); } @@ -626,6 +631,9 @@ string flatten_script_name(const string& name, const string& prefix) TEST_CASE("util normalize_path") { +#ifdef _MSC_VER + // TODO: adapt these tests to Windows +#else CHECK(normalize_path("/1/2/3") == "/1/2/3"); CHECK(normalize_path("/1/./2/3") == "/1/2/3"); CHECK(normalize_path("/1/2/../3") == "/1/3"); @@ -649,6 +657,7 @@ TEST_CASE("util normalize_path") CHECK(normalize_path("~/../..") == "~/../.."); CHECK(normalize_path("zeek/..") == ""); CHECK(normalize_path("zeek/../..") == ".."); +#endif } string normalize_path(std::string_view path) @@ -1334,7 +1343,7 @@ const char* strpbrk_n(size_t len, const char* s, const char* charset) return nullptr; } -#ifndef HAVE_STRCASESTR +#if ! defined(HAVE_STRCASESTR) && ! defined(_MSC_VER) TEST_CASE("util strcasestr") { @@ -1799,6 +1808,9 @@ FILE* open_file(const string& path, const string& mode) TEST_CASE("util path ops") { +#ifdef _MSC_VER + // TODO: adapt these tests to Windows paths +#else SUBCASE("SafeDirname") { SafeDirname d("/this/is/a/path", false); @@ -1821,6 +1833,7 @@ TEST_CASE("util path ops") CHECK(b2.result == "justafile"); CHECK(! b2.error); } +#endif } SafeDirname::SafeDirname(const char* path, bool error_aborts) : SafePathOp() @@ -2725,6 +2738,9 @@ string json_escape_utf8(const char* val, size_t val_size, bool escape_printable_ TEST_CASE("util filesystem") { +#ifdef _MSC_VER + // TODO: adapt these tests to Windows paths +#else zeek::filesystem::path path1("/a/b"); CHECK(path1.is_absolute()); CHECK(! path1.is_relative()); @@ -2739,6 +2755,7 @@ TEST_CASE("util filesystem") auto info = zeek::filesystem::space("."); CHECK(info.capacity > 0); +#endif } TEST_CASE("util split") From 2bdc8dd3a4002b86601c98a857f4fc88f9e032c9 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 27 Oct 2022 12:25:14 -0700 Subject: [PATCH 45/71] Cirrus configuration for Windows builds --- .cirrus.yml | 22 +++++++++++++++++++++- .gitignore | 1 + ci/windows/Dockerfile | 12 ++++++------ ci/windows/build.cmd | 11 +++++++++++ ci/windows/prepare.cmd | 7 +++++++ ci/windows/test.cmd | 7 +++++++ 6 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 ci/windows/build.cmd create mode 100644 ci/windows/prepare.cmd create mode 100644 ci/windows/test.cmd diff --git a/.cirrus.yml b/.cirrus.yml index 2c5648a0fd..faf44239ec 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -59,7 +59,7 @@ ci_template: &CI_TEMPLATE only_if: > ( $CIRRUS_REPO_NAME != 'zeek-security' || $CIRRUS_OS != "darwin" ) && ( ( $CIRRUS_PR != '' && $CIRRUS_BRANCH !=~ 'dependabot/.*' ) || - ( $CIRRUS_REPO_NAME == 'zeek' && + ( $CIRRUS_REPO_NAME == 'ms-zeek' && ( $CIRRUS_BRANCH == 'master' || $CIRRUS_BRANCH =~ 'release/.*' @@ -357,3 +357,23 @@ ubsan_sanitizer_task: # CXXFLAGS: -DZEEK_DICT_DEBUG # ZEEK_CI_CONFIGURE_FLAGS: *TSAN_SANITIZER_CONFIG # ZEEK_CI_DISABLE_SCRIPT_PROFILING: 1 + +windows_task: + # 2 hour timeout just for potential of building Docker image taking a while + timeout_in: 120m + windows_container: + # image: cirrusci/windowsservercore:cmake + # image: zeekurity/broker-ci-windows:latest + dockerfile: ci/windows/Dockerfile + os_version: 2019 + cpu: 8 + # Not allowed to request less than 8GB for an 8 CPU Windows VM. + memory: 8GB + sync_submodules_script: git submodule update --recursive --init + prepare_script: ci/windows/prepare.cmd + build_script: ci/windows/build.cmd + test_script: ci/windows/test.cmd + env: + ZEEK_CI_CPUS: 8 + # Give verbose error output on a test failure. + CTEST_OUTPUT_ON_FAILURE: 1 diff --git a/.gitignore b/.gitignore index 9a7412b64b..6a31b11bd7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Ignore anything prefixed with build since people # tend to name all of their build directories prefixed that way. build* +!ci/windows/build.cmd tmp *.gcov diff --git a/ci/windows/Dockerfile b/ci/windows/Dockerfile index 86d204ca50..af632901db 100644 --- a/ci/windows/Dockerfile +++ b/ci/windows/Dockerfile @@ -12,12 +12,12 @@ RUN [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePoin # Install prerequisites RUN choco install -y --no-progress visualstudio2019buildtools --version=16.11.11.0 RUN choco install -y --no-progress visualstudio2019-workload-vctools --version=1.0.0 --package-parameters '--add Microsoft.VisualStudio.Component.VC.ATLMFC' -RUN choco install conan -y -RUN choco install sed -y -RUN choco install winflexbison3 -y -RUN choco install msysgit -y -RUN choco install python -y -RUN choco install openssl -y +RUN choco install -y --no-progress conan +RUN choco install -y --no-progress sed +RUN choco install -y --no-progress winflexbison3 +RUN choco install -y --no-progress msysgit +RUN choco install -y --no-progress python +RUN choco install -y --no-progress openssl # Set working environment. SHELL [ "cmd", "/c" ] diff --git a/ci/windows/build.cmd b/ci/windows/build.cmd new file mode 100644 index 0000000000..b64396a417 --- /dev/null +++ b/ci/windows/build.cmd @@ -0,0 +1,11 @@ +:: Import the visual studio compiler environment into the one running in the +:: cmd current shell. This path is hard coded to the one on the CI image, but +:: can be adjusted if running builds locally. Unfortunately, the initial path +:: isn't in the environment so we have to hardcode the whole path. +call "c:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 + +mkdir build +cd build + +cmake.exe .. -DCMAKE_BUILD_TYPE=release -DENABLE_ZEEK_UNIT_TESTS=yes -G Ninja +cmake.exe --build . diff --git a/ci/windows/prepare.cmd b/ci/windows/prepare.cmd new file mode 100644 index 0000000000..162381367e --- /dev/null +++ b/ci/windows/prepare.cmd @@ -0,0 +1,7 @@ +@echo on + +echo %ZEEK_CI_CPUS% +wmic cpu get NumberOfCores, NumberOfLogicalProcessors/Format:List +systeminfo +dir C: +choco list --localonly diff --git a/ci/windows/test.cmd b/ci/windows/test.cmd new file mode 100644 index 0000000000..aae40887d9 --- /dev/null +++ b/ci/windows/test.cmd @@ -0,0 +1,7 @@ +:: See build.cmd for documentation on this call. +call "c:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 + +:: We currently don't have any tests to run on Windows, so this is just commented out. +:: We'll expand on this later. +:: cd build +:: ctest -C release || exit \b 1 From d3baa0d1f70b450dd42298f777458689b9da8d12 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 28 Oct 2022 10:52:25 -0700 Subject: [PATCH 46/71] Temporarily move cmake directive into MSVC section --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ac780c6841..f629292c0e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -103,11 +103,12 @@ if ( MSVC ) # Disable Spicy as it is not yet supported in Windows. set(DISABLE_SPICY true) + + set(CMAKE_FIND_LIBRARY_SUFFIXES .a ${CMAKE_FIND_LIBRARY_SUFFIXES}) else () include(GNUInstallDirs) endif () -set(CMAKE_FIND_LIBRARY_SUFFIXES .a ${CMAKE_FIND_LIBRARY_SUFFIXES}) if ( CMAKE_SYSTEM_NAME STREQUAL Linux ) set(CMAKE_DL_LIBS pthread ${CMAKE_DL_LIBS}) endif () From 5494e4e6fd5c8f20fcc9902bd1e02935332c3aa6 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 28 Oct 2022 13:25:25 -0700 Subject: [PATCH 47/71] Fix relative path checking in get_exe_path, plus a signed comparison --- src/util.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util.cc b/src/util.cc index 7203deeef0..7183eefcfc 100644 --- a/src/util.cc +++ b/src/util.cc @@ -778,7 +778,7 @@ std::string get_exe_path(const std::string& invocation) // Absolute path return invocation; - if ( invocation_path.is_relative() ) + if ( invocation_path.is_relative() && invocation_path.has_parent_path() ) { // Relative path char cwd[PATH_MAX]; @@ -1620,7 +1620,7 @@ const char* fmt_bytes(const char* data, int len) const char* vfmt(const char* format, va_list al) { static char* buf = nullptr; - static unsigned int buf_len = 1024; + static int buf_len = 1024; if ( ! buf ) buf = (char*)safe_malloc(buf_len); From 42575a63db6a7e2564ad03d61fd7c4a8bf202f59 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 28 Oct 2022 13:33:42 -0700 Subject: [PATCH 48/71] Add src/include to hilti include paths --- auxil/spicy/hilti-cxx-include-dirs.in | 1 + 1 file changed, 1 insertion(+) diff --git a/auxil/spicy/hilti-cxx-include-dirs.in b/auxil/spicy/hilti-cxx-include-dirs.in index 06ae06a806..6c4a140be9 100755 --- a/auxil/spicy/hilti-cxx-include-dirs.in +++ b/auxil/spicy/hilti-cxx-include-dirs.in @@ -14,6 +14,7 @@ PATHS=$PATHS:@CMAKE_BINARY_DIR@ PATHS=$PATHS:@CMAKE_BINARY_DIR@/src PATHS=$PATHS:@CMAKE_BINARY_DIR@/src/include PATHS=$PATHS:@CMAKE_SOURCE_DIR@/src +PATHS=$PATHS:@CMAKE_SOURCE_DIR@/src/include PATHS=$PATHS:@CMAKE_SOURCE_DIR@/auxil/broker/include/ echo $PATHS From baee0d8026f10535632bdc537251477aabfc290a Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 28 Oct 2022 21:53:22 +0000 Subject: [PATCH 49/71] Replace uses of std::filesystem with zeek::filesystem --- src/util.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/util.cc b/src/util.cc index 7183eefcfc..71b8ce914c 100644 --- a/src/util.cc +++ b/src/util.cc @@ -672,7 +672,7 @@ string normalize_path(std::string_view path) { stringPath.erase(0, 2); } - return std::filesystem::path(stringPath).lexically_normal().string(); + return zeek::filesystem::path(stringPath).lexically_normal().string(); #else if ( path.find("/.") == std::string_view::npos && path.find("//") == std::string_view::npos ) { @@ -772,7 +772,7 @@ std::string get_exe_path(const std::string& invocation) { if ( invocation.empty() ) return ""; - std::filesystem::path invocation_path(invocation); + zeek::filesystem::path invocation_path(invocation); if ( invocation_path.is_absolute() || invocation_path.root_directory() == "~" ) // Absolute path @@ -789,7 +789,7 @@ std::string get_exe_path(const std::string& invocation) exit(1); } - return (std::filesystem::path(cwd) / invocation_path).string(); + return (zeek::filesystem::path(cwd) / invocation_path).string(); } auto path = getenv("PATH"); @@ -1967,7 +1967,7 @@ static string find_file_in_path(const string& filename, const string& path, if ( filename.empty() ) return string(); - std::filesystem::path filepath(filename); + zeek::filesystem::path filepath(filename); // If file name is an absolute path, searching within *path* is pointless. if ( filepath.is_absolute() ) @@ -1978,7 +1978,7 @@ static string find_file_in_path(const string& filename, const string& path, return string(); } - auto abs_path = (std::filesystem::path(path) / filepath).string(); + auto abs_path = (zeek::filesystem::path(path) / filepath).string(); if ( ! opt_ext.empty() ) { From 6bf469b7a834902243901aabf1a407eafa8bd67b Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 28 Oct 2022 22:02:32 +0000 Subject: [PATCH 50/71] Remove extra include from util.cc --- src/util.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/util.cc b/src/util.cc index 71b8ce914c..bdf4cc452a 100644 --- a/src/util.cc +++ b/src/util.cc @@ -42,7 +42,6 @@ #include #include #include -#include #include #include "zeek/3rdparty/ConvertUTF.h" From 2e457eb3eae4aad1f0f4d77e84a3bc2b5bdb2c52 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 28 Oct 2022 15:50:43 -0700 Subject: [PATCH 51/71] Fix a few compiler warnings from MSVC --- src/ScriptProfile.cc | 2 +- src/packet_analysis/protocol/ip/IP.cc | 2 +- src/supervisor/Supervisor.cc | 2 +- src/util.cc | 10 +++++----- src/util.h | 2 +- zeek-config.h.in | 23 +++++++++++------------ 6 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/ScriptProfile.cc b/src/ScriptProfile.cc index 862b4c5ebe..22f09dfb63 100644 --- a/src/ScriptProfile.cc +++ b/src/ScriptProfile.cc @@ -102,7 +102,7 @@ ScriptProfileMgr::~ScriptProfileMgr() auto& fp = fs.second; auto n = func->GetBodies().size(); if ( n > 1 ) - fprintf(f, "%s\t%lu-locations\t%s\t%d\t%.06f\t%0.6f\t%" PRIu64 "\t%lld\n", + fprintf(f, "%s\t%zu-locations\t%s\t%d\t%.06f\t%0.6f\t%" PRIu64 "\t%lld\n", fp.Name().c_str(), n, func->GetType()->FlavorString().c_str(), fp.NumCalls(), fp.CPUTime(), 0.0, fp.Memory(), 0LL); } diff --git a/src/packet_analysis/protocol/ip/IP.cc b/src/packet_analysis/protocol/ip/IP.cc index 1263317317..19d5908d67 100644 --- a/src/packet_analysis/protocol/ip/IP.cc +++ b/src/packet_analysis/protocol/ip/IP.cc @@ -62,7 +62,7 @@ bool IPAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* packet) return false; } - ip_hdr = std::make_shared((const struct ip6_hdr*)data, false, len); + ip_hdr = std::make_shared((const struct ip6_hdr*)data, false, static_cast(len)); packet->l3_proto = L3_IPV6; } else diff --git a/src/supervisor/Supervisor.cc b/src/supervisor/Supervisor.cc index a2dc6a66ba..8d7fc4f3ab 100644 --- a/src/supervisor/Supervisor.cc +++ b/src/supervisor/Supervisor.cc @@ -1249,7 +1249,7 @@ Supervisor::NodeConfig Supervisor::NodeConfig::FromRecord(const RecordVal* node) const auto& affinity_val = node->GetField("cpu_affinity"); if ( affinity_val ) - rval.cpu_affinity = affinity_val->AsInt(); + rval.cpu_affinity = static_cast(affinity_val->AsInt()); const auto& bare_mode_val = node->GetField("bare_mode"); diff --git a/src/util.cc b/src/util.cc index bdf4cc452a..79da74bf9f 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1128,9 +1128,9 @@ TEST_CASE("util streq") CHECK(streq("abcd", "efgh") == false); } -int streq(const char* s1, const char* s2) +bool streq(const char* s1, const char* s2) { - return ! strcmp(s1, s2); + return strcmp(s1, s2) == 0; } bool starts_with(std::string_view s, std::string_view beginning) @@ -2282,8 +2282,8 @@ const void* memory_align(const void* ptr, size_t size) const char* buf = reinterpret_cast(ptr); size_t mask = size - 1; // Assume size is a power of 2. - unsigned long l_ptr = reinterpret_cast(ptr); - unsigned long offset = l_ptr & mask; + intptr_t l_ptr = reinterpret_cast(ptr); + ptrdiff_t offset = l_ptr & mask; if ( offset > 0 ) return reinterpret_cast(buf - offset + size); @@ -2322,7 +2322,7 @@ void* memory_align_and_pad(void* ptr, size_t size) char* buf = reinterpret_cast(ptr); size_t mask = size - 1; - while ( (reinterpret_cast(buf) & mask) != 0 ) + while ( (reinterpret_cast(buf) & mask) != 0 ) // Not aligned - zero pad. *buf++ = '\0'; diff --git a/src/util.h b/src/util.h index 6471b8019a..8919b8276a 100644 --- a/src/util.h +++ b/src/util.h @@ -331,7 +331,7 @@ std::vector* tokenize_string(std::string_view input, std::string_vi std::vector tokenize_string(std::string_view input, const char delim) noexcept; extern char* copy_string(const char* s); -extern int streq(const char* s1, const char* s2); +extern bool streq(const char* s1, const char* s2); extern bool starts_with(std::string_view s, std::string_view beginning); extern bool ends_with(std::string_view s, std::string_view ending); diff --git a/zeek-config.h.in b/zeek-config.h.in index a6492ddd53..f2aa463686 100644 --- a/zeek-config.h.in +++ b/zeek-config.h.in @@ -271,20 +271,19 @@ extern const char* ZEEK_VERSION_FUNCTION(); #endif #endif -// FreeBSD doesn't support LeakSanitizer #if defined(ZEEK_ASAN) && !defined(__FreeBSD__) - #include - #define ZEEK_LSAN_CHECK(x) __lsan_do_leak_check(x) - #define ZEEK_LSAN_ENABLE(x) __lsan_enable(x) - #define ZEEK_LSAN_IGNORE(x) __lsan_ignore_object(x) - #define ZEEK_LSAN_DISABLE(x) __lsan_disable(x) - #define ZEEK_LSAN_DISABLE_SCOPE(x) __lsan::ScopedDisabler x + #include + #define ZEEK_LSAN_CHECK(...) __lsan_do_leak_check(__VA_ARGS__) + #define ZEEK_LSAN_ENABLE(...) __lsan_enable(__VA_ARGS__) + #define ZEEK_LSAN_IGNORE(...) __lsan_ignore_object(__VA_ARGS__) + #define ZEEK_LSAN_DISABLE(...) __lsan_disable(__VA_ARGS__) + #define ZEEK_LSAN_DISABLE_SCOPE(...) __lsan::ScopedDisabler __VA_ARGS__ #else - #define ZEEK_LSAN_CHECK(x) - #define ZEEK_LSAN_ENABLE(x) - #define ZEEK_LSAN_IGNORE(x) - #define ZEEK_LSAN_DISABLE(x) - #define ZEEK_LSAN_DISABLE_SCOPE(x) + #define ZEEK_LSAN_CHECK(...) + #define ZEEK_LSAN_ENABLE(...) + #define ZEEK_LSAN_IGNORE(...) + #define ZEEK_LSAN_DISABLE(...) + #define ZEEK_LSAN_DISABLE_SCOPE(...) #endif // This part is dependent on calling configure with '--sanitizers=thread' From 359b5547cd002fc8ab879d086a3c16eba2583374 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 28 Oct 2022 15:50:51 -0700 Subject: [PATCH 52/71] Don't call tokenize_string if the input string is empty --- src/plugin/Manager.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index 43eb1daee0..fa127419c3 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -359,10 +359,14 @@ void Manager::ActivateDynamicPlugins(bool all) // Activate plugins that our environment tells us to. vector p; - util::tokenize_string(util::zeek_plugin_activate(), ",", &p); + std::string plugin_activate = util::zeek_plugin_activate(); + if ( ! plugin_activate.empty() ) + { + util::tokenize_string(util::zeek_plugin_activate(), ",", &p); - for ( const auto& x : p ) - plugins_to_activate.emplace(x, true); + for ( const auto& x : p ) + plugins_to_activate.emplace(x, true); + } if ( all ) { From ef096eedf322d0f23f09ab13ff371d8e81019286 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 28 Oct 2022 16:24:55 -0700 Subject: [PATCH 53/71] Update libkqueue submodule [nomail] --- auxil/libunistd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auxil/libunistd b/auxil/libunistd index 7555c4708e..2c290dc3c7 160000 --- a/auxil/libunistd +++ b/auxil/libunistd @@ -1 +1 @@ -Subproject commit 7555c4708e7e7bc408cd129215682acdd839c840 +Subproject commit 2c290dc3c7c9c706b5c0abbe06158bfb8a2721fe From fbcb7bd2ec54a05b5082a141692c0b3fedee7991 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Sat, 29 Oct 2022 18:31:11 -0700 Subject: [PATCH 54/71] Use std::chrono instead of gettimeofday --- src/logging/writers/ascii/Ascii.cc | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/logging/writers/ascii/Ascii.cc b/src/logging/writers/ascii/Ascii.cc index 1e4ee3e8cf..9247ec6a0b 100644 --- a/src/logging/writers/ascii/Ascii.cc +++ b/src/logging/writers/ascii/Ascii.cc @@ -910,17 +910,12 @@ string Ascii::Timestamp(double t) time_t teatime = time_t(t); if ( ! teatime ) - { - // Use wall clock. - struct timeval tv; - if ( gettimeofday(&tv, 0) < 0 ) - Error("gettimeofday failed"); - else - teatime = tv.tv_sec; - } + teatime = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); struct tm tmbuf; struct tm* tm = localtime_r(&teatime, &tmbuf); + if ( tm == nullptr ) + Error(util::fmt("localtime_r failed: %s", strerror(errno))); char tmp[128]; const char* const date_fmt = "%Y-%m-%d-%H-%M-%S"; From af947ae000d90c1d35b768d57827d4af9745ff84 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Sat, 29 Oct 2022 22:36:10 -0700 Subject: [PATCH 55/71] Add a couple of extra unistd.h includes to fix a build failure --- src/iosource/BPF_Program.cc | 5 ++--- src/iosource/pcap/Dumper.h | 2 ++ src/iosource/pcap/Source.h | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/iosource/BPF_Program.cc b/src/iosource/BPF_Program.cc index 1ee1952143..3717ce0d97 100644 --- a/src/iosource/BPF_Program.cc +++ b/src/iosource/BPF_Program.cc @@ -1,12 +1,11 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include -#include - #include "zeek/iosource/BPF_Program.h" #include "zeek/zeek-config.h" +#include +#include #include #include "zeek/util.h" diff --git a/src/iosource/pcap/Dumper.h b/src/iosource/pcap/Dumper.h index 1c78f4e575..58bd169030 100644 --- a/src/iosource/pcap/Dumper.h +++ b/src/iosource/pcap/Dumper.h @@ -2,6 +2,8 @@ #pragma once +#include + extern "C" { #include diff --git a/src/iosource/pcap/Source.h b/src/iosource/pcap/Source.h index 1f100273ea..b02828ae1c 100644 --- a/src/iosource/pcap/Source.h +++ b/src/iosource/pcap/Source.h @@ -3,6 +3,7 @@ #pragma once #include // for u_char +#include extern "C" { From 7d55057d88e382336b39122ea8c7585fee18d19e Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 31 Oct 2022 08:30:03 -0700 Subject: [PATCH 56/71] Add NEWS entry about Windows port --- NEWS | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/NEWS b/NEWS index d8ffd1bf9a..0360e1f136 100644 --- a/NEWS +++ b/NEWS @@ -23,6 +23,17 @@ Breaking Changes New Functionality ----------------- +- Experimental support added for building and running Zeek on Microsoft Windows + environments. This is considered experimental due to the fact that our + standard testing setup (btest) doesn't run properly on Windows. This will be + fixed in the future. In the meantime we have done some basic testing against + builds done with Visual Studio 2019. Information on how to build on Windows is + available in the Zeek documentation. Note also that Spicy is currently + unsupported and will be fixed in the future. + + The Zeek team wants to give a huge thank you to the team at Microsoft for all + of their effort in completing this port. + - New ``analyzer_confirmation_info`` and ``analyzer_violation_info`` events with accompanying record types ``AnalyzerConfirmationInfo`` and ``AnalyzerViolationInfo`` have been added. These supersede From 12494aac45d00e1440c9f75654ad025445e6cd6d Mon Sep 17 00:00:00 2001 From: Tomer Lev Date: Tue, 1 Nov 2022 18:26:08 +0200 Subject: [PATCH 57/71] Fixed initial CR comments --- .cirrus.yml | 2 +- CMakeLists.txt | 15 --------------- COPYING.3rdparty | 28 ++++++++++++++++++++++++++++ src/CMakeLists.txt | 1 + src/IntrusivePtr.h | 2 +- src/fuzzers/FuzzBuffer.cc | 4 ++-- src/plugin/Manager.cc | 1 + src/util.h | 2 -- 8 files changed, 34 insertions(+), 21 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index faf44239ec..85902c4d81 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -59,7 +59,7 @@ ci_template: &CI_TEMPLATE only_if: > ( $CIRRUS_REPO_NAME != 'zeek-security' || $CIRRUS_OS != "darwin" ) && ( ( $CIRRUS_PR != '' && $CIRRUS_BRANCH !=~ 'dependabot/.*' ) || - ( $CIRRUS_REPO_NAME == 'ms-zeek' && + ( $CIRRUS_REPO_NAME == 'zeek' && ( $CIRRUS_BRANCH == 'master' || $CIRRUS_BRANCH =~ 'release/.*' diff --git a/CMakeLists.txt b/CMakeLists.txt index f629292c0e..e811f2948f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -103,25 +103,10 @@ if ( MSVC ) # Disable Spicy as it is not yet supported in Windows. set(DISABLE_SPICY true) - - set(CMAKE_FIND_LIBRARY_SUFFIXES .a ${CMAKE_FIND_LIBRARY_SUFFIXES}) else () include(GNUInstallDirs) endif () -if ( CMAKE_SYSTEM_NAME STREQUAL Linux ) - set(CMAKE_DL_LIBS pthread ${CMAKE_DL_LIBS}) -endif () - -if ( NOT CMAKE_INSTALL_LIBDIR ) - # Currently, some sub-projects may use GNUInstallDirs.cmake to choose the - # library install dir, while others just default to "lib". For sake of - # consistency, this just overrides the former to always use "lib" in case - # it would have chosen something else, like "lib64", but a thing for the - # future may be to standardize all sub-projects to use GNUInstallDirs. - set(CMAKE_INSTALL_LIBDIR lib) -endif () - include(cmake/CommonCMakeConfig.cmake) include(cmake/FindClangTidy.cmake) include(cmake/CheckCompilerArch.cmake) diff --git a/COPYING.3rdparty b/COPYING.3rdparty index e984d6eb40..a023dc9efc 100644 --- a/COPYING.3rdparty +++ b/COPYING.3rdparty @@ -754,3 +754,31 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +============================================================================== + +%%% auxil/libunistd + +============================================================================== + +The MIT License (MIT) + +Copyright (c) 2015 Robin Rowe + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index dafb4662f4..2fe0a31c1e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -491,6 +491,7 @@ elseif (${COMPILER_ARCHITECTURE} STREQUAL "power") elseif(${COMPILER_ARCHITECTURE} STREQUAL "x86_64") if (MSVC) set(_avx_flag /arch:AVX2) + # Using an undocumentd compiler flag: https://stackoverflow.com/questions/64053597/how-do-i-enable-sse4-1-and-sse3-but-not-avx-in-msvc/69328426#69328426 set(_sse_flag /d2archSSE42) else() set(_avx_flag -mavx2) diff --git a/src/IntrusivePtr.h b/src/IntrusivePtr.h index bdeb501e85..7d9578d29f 100644 --- a/src/IntrusivePtr.h +++ b/src/IntrusivePtr.h @@ -28,7 +28,7 @@ struct NewRef }; /** - * This has to be forward decalred and known here in order for us to be able + * This has to be forward declared and known here in order for us to be able * cast this in the `Unref` function. */ class OpaqueVal; diff --git a/src/fuzzers/FuzzBuffer.cc b/src/fuzzers/FuzzBuffer.cc index 81251473a4..5ea074f5c3 100644 --- a/src/fuzzers/FuzzBuffer.cc +++ b/src/fuzzers/FuzzBuffer.cc @@ -2,12 +2,12 @@ #define _GNU_SOURCE #endif +#include "zeek/fuzzers/FuzzBuffer.h" + #ifdef _MSC_VER #include #endif -#include "zeek/fuzzers/FuzzBuffer.h" - #include namespace zeek::detail diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index fa127419c3..e52185bd33 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -162,6 +162,7 @@ void Manager::SearchDynamicPlugins(const std::string& dir) bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_not_found, std::vector* errors) { +// Loading dynamic plugins is not currently supported on Windows platform. #if defined(_MSC_VER) return false; #else diff --git a/src/util.h b/src/util.h index 8919b8276a..7019560058 100644 --- a/src/util.h +++ b/src/util.h @@ -110,8 +110,6 @@ namespace zeek class ODesc; class RecordVal; - - namespace util { namespace detail From 5cdc6e150e9e4d233c54471e2c111af62937fe84 Mon Sep 17 00:00:00 2001 From: Tomer Lev Date: Tue, 1 Nov 2022 19:31:15 +0200 Subject: [PATCH 58/71] Clang format it all --- src/Flare.cc | 41 ++- src/Flare.h | 8 +- src/Hash.h | 2 +- src/IP.cc | 2 +- src/Val.cc | 3 - src/logging/writers/sqlite/SQLite.cc | 2 +- src/net_util.h | 4 +- src/packet_analysis/protocol/arp/ARP.cc | 2 +- src/packet_analysis/protocol/ip/IP.cc | 3 +- src/plugin/Manager.cc | 4 +- src/plugin/Plugin.h | 2 +- src/statistics/Plugin.cc | 71 +++-- src/statistics/Plugin.h | 15 +- src/supervisor/Supervisor.cc | 6 +- src/threading/BasicThread.cc | 5 +- src/util.cc | 61 ++-- src/util.h | 11 +- src/zeek-setup.cc | 394 ++++++++++++------------ 18 files changed, 323 insertions(+), 313 deletions(-) diff --git a/src/Flare.cc b/src/Flare.cc index cbe47ab005..0ab45b7452 100644 --- a/src/Flare.cc +++ b/src/Flare.cc @@ -12,19 +12,18 @@ #include -#define fatalError(...) \ -do \ - { \ - if ( reporter ) \ - reporter->FatalError(__VA_ARGS__); \ - else \ - { \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, "\n"); \ - _exit(1); \ - } \ - } \ -while (0) +#define fatalError(...) \ + do \ + { \ + if ( reporter ) \ + reporter->FatalError(__VA_ARGS__); \ + else \ + { \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + _exit(1); \ + } \ + } while ( 0 ) #endif @@ -39,28 +38,28 @@ Flare::Flare() #else { WSADATA wsaData; - if ( WSAStartup(MAKEWORD(2,2), &wsaData) != 0 ) + if ( WSAStartup(MAKEWORD(2, 2), &wsaData) != 0 ) fatalError("WSAStartup failure: %d", WSAGetLastError()); recvfd = WSASocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP, nullptr, 0, - WSA_FLAG_OVERLAPPED | WSA_FLAG_NO_HANDLE_INHERIT); - if ( recvfd == (int) INVALID_SOCKET ) + WSA_FLAG_OVERLAPPED | WSA_FLAG_NO_HANDLE_INHERIT); + if ( recvfd == (int)INVALID_SOCKET ) fatalError("WSASocket failure: %d", WSAGetLastError()); sendfd = WSASocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP, nullptr, 0, - WSA_FLAG_OVERLAPPED | WSA_FLAG_NO_HANDLE_INHERIT); - if ( sendfd == (int) INVALID_SOCKET ) + WSA_FLAG_OVERLAPPED | WSA_FLAG_NO_HANDLE_INHERIT); + if ( sendfd == (int)INVALID_SOCKET ) fatalError("WSASocket failure: %d", WSAGetLastError()); sockaddr_in sa; memset(&sa, 0, sizeof(sa)); sa.sin_family = AF_INET; sa.sin_addr.s_addr = inet_addr("127.0.0.1"); - if ( bind(recvfd, (sockaddr*) &sa, sizeof(sa)) == SOCKET_ERROR ) + if ( bind(recvfd, (sockaddr*)&sa, sizeof(sa)) == SOCKET_ERROR ) fatalError("bind failure: %d", WSAGetLastError()); int salen = sizeof(sa); - if ( getsockname(recvfd, (sockaddr*) &sa, &salen) == SOCKET_ERROR ) + if ( getsockname(recvfd, (sockaddr*)&sa, &salen) == SOCKET_ERROR ) fatalError("getsockname failure: %d", WSAGetLastError()); - if ( connect(sendfd, (sockaddr*) &sa, sizeof(sa)) == SOCKET_ERROR ) + if ( connect(sendfd, (sockaddr*)&sa, sizeof(sa)) == SOCKET_ERROR ) fatalError("connect failure: %d", WSAGetLastError()); } #endif diff --git a/src/Flare.h b/src/Flare.h index a06996338f..6ded8af25e 100644 --- a/src/Flare.h +++ b/src/Flare.h @@ -26,9 +26,13 @@ public: */ int FD() const #if ! defined(_MSC_VER) - { return pipe.ReadFD(); } + { + return pipe.ReadFD(); + } #else - { return recvfd; } + { + return recvfd; + } #endif /** diff --git a/src/Hash.h b/src/Hash.h index 796763d509..b727fc7243 100644 --- a/src/Hash.h +++ b/src/Hash.h @@ -372,7 +372,7 @@ protected: void Set(double d); void Set(const void* p); - union { + union { bool b; int i; zeek_int_t bi; diff --git a/src/IP.cc b/src/IP.cc index 4de4506387..d7ec2dbeb1 100644 --- a/src/IP.cc +++ b/src/IP.cc @@ -546,7 +546,7 @@ void IPv6_Hdr_Chain::Init(const struct ip6_hdr* ip6, int total_len, bool set_nex total_len -= cur_len; } while ( current_type != IPPROTO_FRAGMENT && current_type != IPPROTO_ESP && - current_type != IPPROTO_MOBILITY && isIPv6ExtHeader(next_type) ); + current_type != IPPROTO_MOBILITY && isIPv6ExtHeader(next_type) ); } bool IPv6_Hdr_Chain::IsFragment() const diff --git a/src/Val.cc b/src/Val.cc index fc70ef3bdb..a20e00570d 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -3962,7 +3962,6 @@ ValManager::ValManager() for ( auto i = 0u; i < PREALLOCATED_INTS; ++i ) ints[i] = Val::MakeInt(PREALLOCATED_INT_LOWEST + i); - } const PortValPtr& ValManager::Port(uint32_t port_num, TransportProto port_type) @@ -3975,9 +3974,7 @@ const PortValPtr& ValManager::Port(uint32_t port_num, TransportProto port_type) std::pair key{port_num, port_type}; if ( ports.find(key) == ports.end() ) - { ports[key] = IntrusivePtr{AdoptRef{}, new PortVal(PortVal::Mask(port_num, port_type))}; - } return ports[key]; } diff --git a/src/logging/writers/sqlite/SQLite.cc b/src/logging/writers/sqlite/SQLite.cc index 9eee5596e9..5471bf949a 100644 --- a/src/logging/writers/sqlite/SQLite.cc +++ b/src/logging/writers/sqlite/SQLite.cc @@ -63,7 +63,7 @@ string SQLite::GetTableType(int arg_type, int arg_subtype) case TYPE_INT: case TYPE_COUNT: case TYPE_PORT: // note that we do not save the protocol at the moment. Just like in the - // case of the ascii-writer + // case of the ascii-writer type = "integer"; break; diff --git a/src/net_util.h b/src/net_util.h index 7031eed81e..e3bd265640 100644 --- a/src/net_util.h +++ b/src/net_util.h @@ -64,7 +64,7 @@ struct ip6_ext struct ip6_hdr { - union { + union { struct ip6_hdrctl { uint32_t ip6_un1_flow; /* 4 bits version, 8 bits TC, 20 bits @@ -335,7 +335,7 @@ inline float htonf(float f) inline uint64_t ntohll(uint64_t i) { u_char c; - union { + union { uint64_t i; u_char c[8]; } x; diff --git a/src/packet_analysis/protocol/arp/ARP.cc b/src/packet_analysis/protocol/arp/ARP.cc index 40ee22247b..5d9396c6d0 100644 --- a/src/packet_analysis/protocol/arp/ARP.cc +++ b/src/packet_analysis/protocol/arp/ARP.cc @@ -94,7 +94,7 @@ bool ARPAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* packet) auto ah = (const struct arp_pkthdr*)data; // Check the size. - size_t min_length = (ar_tpa(ah) - (caddr_t) data) + ah->ar_pln; + size_t min_length = (ar_tpa(ah) - (caddr_t)data) + ah->ar_pln; if ( min_length > len ) { Weird("truncated_ARP", packet); diff --git a/src/packet_analysis/protocol/ip/IP.cc b/src/packet_analysis/protocol/ip/IP.cc index 19d5908d67..ebed559a2b 100644 --- a/src/packet_analysis/protocol/ip/IP.cc +++ b/src/packet_analysis/protocol/ip/IP.cc @@ -62,7 +62,8 @@ bool IPAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* packet) return false; } - ip_hdr = std::make_shared((const struct ip6_hdr*)data, false, static_cast(len)); + packet->ip_hdr = std::make_shared((const struct ip6_hdr*)data, false, + static_cast(len)); packet->l3_proto = L3_IPV6; } else diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index e52185bd33..7828af3656 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -3,7 +3,7 @@ #include "zeek/plugin/Manager.h" #include -#if !defined(_MSC_VER) +#if ! defined(_MSC_VER) #include #include #endif @@ -355,7 +355,7 @@ void Manager::ActivateDynamicPlugins(bool all) for ( const auto& x : requested_plugins ) { if ( ! x.empty() ) - plugins_to_activate.emplace(x, false); + plugins_to_activate.emplace(x, false); } // Activate plugins that our environment tells us to. diff --git a/src/plugin/Plugin.h b/src/plugin/Plugin.h index e49d80b461..333df81538 100644 --- a/src/plugin/Plugin.h +++ b/src/plugin/Plugin.h @@ -661,7 +661,7 @@ public: private: Type type; - union { + union { bool bool_; double double_; const Event* event; diff --git a/src/statistics/Plugin.cc b/src/statistics/Plugin.cc index 18cb71645c..8870580d82 100644 --- a/src/statistics/Plugin.cc +++ b/src/statistics/Plugin.cc @@ -1,54 +1,57 @@ -#include - #include "Plugin.h" -#include "zeek/Func.h" -#include "zeek/Event.h" +#include + #include "zeek/Conn.h" #include "zeek/Desc.h" -#include "zeek/threading/Formatter.h" +#include "zeek/Event.h" +#include "zeek/Func.h" #include "zeek/RunState.h" +#include "zeek/threading/Formatter.h" #include "statistics.bif.h" -namespace zeek::plugin::statistics { Plugin plugin; } +namespace zeek::plugin::statistics + { +Plugin plugin; + } using namespace zeek::plugin::statistics; zeek::plugin::Configuration Plugin::Configure() - { - zeek::plugin::Configuration config; - config.name = "Statistics"; - config.description = "Statistics module"; - config.version.major = 1; - config.version.minor = 0; - config.version.patch = 0; - return config; - } + { + zeek::plugin::Configuration config; + config.name = "Statistics"; + config.description = "Statistics module"; + config.version.major = 1; + config.version.minor = 0; + config.version.patch = 0; + return config; + } bool Plugin::HookQueueEvent(zeek::Event* event) - { - const char* name = event->Handler()->Name(); + { + const char* name = event->Handler()->Name(); - std::lock_guard scopedLock(m_lock); - if (m_eventNameCounters.find(name) == m_eventNameCounters.end()) - { - m_eventNameCounters[name] = 0; - } - m_eventNameCounters[name]++; - return false; - } + std::lock_guard scopedLock(m_lock); + if ( m_eventNameCounters.find(name) == m_eventNameCounters.end() ) + { + m_eventNameCounters[name] = 0; + } + m_eventNameCounters[name]++; + return false; + } std::unordered_map Plugin::GetAndResetEventStatistics() - { - std::lock_guard scopedLock(m_lock); - std::unordered_map result(m_eventNameCounters); - m_eventNameCounters.clear(); - return result; - } + { + std::lock_guard scopedLock(m_lock); + std::unordered_map result(m_eventNameCounters); + m_eventNameCounters.clear(); + return result; + } void Plugin::StartEventNamesStatisticsMonitor() - { - EnableHook(zeek::plugin::HOOK_QUEUE_EVENT); - } + { + EnableHook(zeek::plugin::HOOK_QUEUE_EVENT); + } diff --git a/src/statistics/Plugin.h b/src/statistics/Plugin.h index 0c396d8b07..d0eea47f82 100644 --- a/src/statistics/Plugin.h +++ b/src/statistics/Plugin.h @@ -1,15 +1,17 @@ #pragma once -#include #include #include +#include + #include "zeek/plugin/Plugin.h" -namespace zeek::plugin::statistics { +namespace zeek::plugin::statistics + { class Plugin : public zeek::plugin::Plugin -{ + { protected: bool HookQueueEvent(zeek::Event* event) override; @@ -18,12 +20,11 @@ protected: public: std::unordered_map GetAndResetEventStatistics(); void StartEventNamesStatisticsMonitor(); - + private: std::unordered_map m_eventNameCounters; std::mutex m_lock; - -}; + }; extern Plugin plugin; -} + } diff --git a/src/supervisor/Supervisor.cc b/src/supervisor/Supervisor.cc index 8d7fc4f3ab..6516bec699 100644 --- a/src/supervisor/Supervisor.cc +++ b/src/supervisor/Supervisor.cc @@ -995,12 +995,14 @@ std::optional Stem::Poll() node_pollfd_indices[name] = pfd_idx; if ( node.stdout_pipe.pipe ) - pfds[pfd_idx++] = {static_cast(node.stdout_pipe.pipe->ReadFD()), POLLIN, 0}; + pfds[pfd_idx++] = {static_cast(node.stdout_pipe.pipe->ReadFD()), + POLLIN, 0}; else pfds[pfd_idx++] = {static_cast(-1), POLLIN, 0}; if ( node.stderr_pipe.pipe ) - pfds[pfd_idx++] = {static_cast(node.stderr_pipe.pipe->ReadFD()), POLLIN, 0}; + pfds[pfd_idx++] = {static_cast(node.stderr_pipe.pipe->ReadFD()), + POLLIN, 0}; else pfds[pfd_idx++] = {static_cast(-1), POLLIN, 0}; } diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc index 457906a152..37bc3b6cba 100644 --- a/src/threading/BasicThread.cc +++ b/src/threading/BasicThread.cc @@ -49,8 +49,9 @@ void BasicThread::SetName(const char* arg_name) void BasicThread::SetOSName(const char* arg_name) { // Do it only if libc++ supports pthread_t. - if constexpr(std::is_same::value) - zeek::util::detail::set_thread_name(arg_name, reinterpret_cast(thread.native_handle())); + if constexpr ( std::is_same::value ) + zeek::util::detail::set_thread_name(arg_name, + reinterpret_cast(thread.native_handle())); } const char* BasicThread::Fmt(const char* format, ...) diff --git a/src/util.cc b/src/util.cc index 79da74bf9f..bc85bb166e 100644 --- a/src/util.cc +++ b/src/util.cc @@ -40,9 +40,9 @@ #include #include #include +#include #include #include -#include #include "zeek/3rdparty/ConvertUTF.h" #include "zeek/3rdparty/doctest.h" @@ -662,15 +662,16 @@ TEST_CASE("util normalize_path") string normalize_path(std::string_view path) { #ifdef _MSC_VER - if (0 == path.compare(zeek::detail::ScannedFile::canonical_stdin_path)) { + if ( 0 == path.compare(zeek::detail::ScannedFile::canonical_stdin_path) ) + { return string(path); - } + } // "//" interferes with std::weakly_canonical string stringPath = string(path); - if (stringPath._Starts_with("//")) - { + if ( stringPath._Starts_with("//") ) + { stringPath.erase(0, 2); - } + } return zeek::filesystem::path(stringPath).lexically_normal().string(); #else if ( path.find("/.") == std::string_view::npos && path.find("//") == std::string_view::npos ) @@ -1805,37 +1806,35 @@ FILE* open_file(const string& path, const string& mode) return rval; } -TEST_CASE("util path ops") - { +TEST_CASE("util path ops"){ #ifdef _MSC_VER - // TODO: adapt these tests to Windows paths +// TODO: adapt these tests to Windows paths #else - SUBCASE("SafeDirname") - { - SafeDirname d("/this/is/a/path", false); - CHECK(d.result == "/this/is/a"); + SUBCASE("SafeDirname"){SafeDirname d("/this/is/a/path", false); +CHECK(d.result == "/this/is/a"); - SafeDirname d2("invalid", false); - CHECK(d2.result == "."); +SafeDirname d2("invalid", false); +CHECK(d2.result == "."); - SafeDirname d3("./filename", false); - CHECK(d2.result == "."); - } - - SUBCASE("SafeBasename") - { - SafeBasename b("/this/is/a/path", false); - CHECK(b.result == "path"); - CHECK(! b.error); - - SafeBasename b2("justafile", false); - CHECK(b2.result == "justafile"); - CHECK(! b2.error); - } -#endif +SafeDirname d3("./filename", false); +CHECK(d2.result == "."); } -SafeDirname::SafeDirname(const char* path, bool error_aborts) : SafePathOp() +SUBCASE("SafeBasename") + { + SafeBasename b("/this/is/a/path", false); + CHECK(b.result == "path"); + CHECK(! b.error); + + SafeBasename b2("justafile", false); + CHECK(b2.result == "justafile"); + CHECK(! b2.error); + } +#endif +} + +SafeDirname::SafeDirname(const char* path, bool error_aborts) + : SafePathOp() { DoFunc(path ? path : "", error_aborts); } diff --git a/src/util.h b/src/util.h index 7019560058..38d812f04b 100644 --- a/src/util.h +++ b/src/util.h @@ -16,7 +16,6 @@ #include #include - #include #include #include @@ -82,13 +81,19 @@ extern "C" #ifdef _MSC_VER #include #include -namespace zeek { namespace filesystem = std::filesystem; } +namespace zeek + { +namespace filesystem = std::filesystem; + } inline constexpr std::string_view path_list_separator = ";"; #else // Expose ghc::filesystem as zeek::filesystem until we can // switch to std::filesystem on all platforms. #include "zeek/3rdparty/ghc/filesystem.hpp" -namespace zeek { namespace filesystem = ghc::filesystem; } +namespace zeek + { +namespace filesystem = ghc::filesystem; + } inline constexpr std::string_view path_list_separator = ":"; #endif diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index fbef570fca..7f89378a16 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -1,7 +1,5 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include - #include "zeek/zeek-setup.h" #include "zeek/zeek-config.h" @@ -10,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -888,229 +887,228 @@ SetupResult setup(int argc, char** argv, Options* zopts) #ifdef USE_PERFTOOLS_DEBUG } #endif - set_signal_mask(false); + set_signal_mask(false); - if ( reporter->Errors() > 0 ) + if ( reporter->Errors() > 0 ) + { + early_shutdown(); + exit(1); + } + + reporter->InitOptions(); + KeyedHash::InitOptions(); + zeekygen_mgr->GenerateDocs(); + + if ( options.pcap_filter ) + { + const auto& id = global_scope()->Find("cmd_line_bpf_filter"); + + if ( ! id ) + reporter->InternalError("global cmd_line_bpf_filter not defined"); + + id->SetVal(make_intrusive(*options.pcap_filter)); + } + + std::vector all_signature_files; + + // Append signature files given on the command line + for ( const auto& sf : options.signature_files ) + all_signature_files.emplace_back(sf); + + // Append signature files defined in "signature_files" script option + for ( auto&& sf : get_script_signature_files() ) + all_signature_files.emplace_back(std::move(sf)); + + // Append signature files defined in @load-sigs + for ( const auto& sf : zeek::detail::sig_files ) + all_signature_files.emplace_back(sf); + + if ( ! all_signature_files.empty() ) + { + rule_matcher = new RuleMatcher(options.signature_re_level); + if ( ! rule_matcher->ReadFiles(all_signature_files) ) { early_shutdown(); exit(1); } - reporter->InitOptions(); - KeyedHash::InitOptions(); - zeekygen_mgr->GenerateDocs(); + if ( options.print_signature_debug_info ) + rule_matcher->PrintDebug(); - if ( options.pcap_filter ) + file_mgr->InitMagic(); + } + + if ( g_policy_debug ) + // ### Add support for debug command file. + dbg_init_debugger(nullptr); + + if ( ! options.pcap_file && ! options.interface ) + { + const auto& interfaces_val = id::find_val("interfaces"); + if ( interfaces_val ) { - const auto& id = global_scope()->Find("cmd_line_bpf_filter"); + char* interfaces_str = interfaces_val->AsString()->Render(); - if ( ! id ) - reporter->InternalError("global cmd_line_bpf_filter not defined"); + if ( interfaces_str[0] != '\0' ) + options.interface = interfaces_str; - id->SetVal(make_intrusive(*options.pcap_filter)); + delete[] interfaces_str; } + } - std::vector all_signature_files; + if ( options.parse_only ) + { + if ( analysis_options.usage_issues > 0 ) + analyze_scripts(options.no_unused_warnings); - // Append signature files given on the command line - for ( const auto& sf : options.signature_files ) - all_signature_files.emplace_back(sf); + early_shutdown(); + exit(reporter->Errors() != 0); + } - // Append signature files defined in "signature_files" script option - for ( auto&& sf : get_script_signature_files() ) - all_signature_files.emplace_back(std::move(sf)); + auto init_stmts = stmts ? analyze_global_stmts(stmts) : nullptr; - // Append signature files defined in @load-sigs - for ( const auto& sf : zeek::detail::sig_files ) - all_signature_files.emplace_back(sf); + analyze_scripts(options.no_unused_warnings); - if ( ! all_signature_files.empty() ) - { - rule_matcher = new RuleMatcher(options.signature_re_level); - if ( ! rule_matcher->ReadFiles(all_signature_files) ) - { - early_shutdown(); - exit(1); - } + if ( analysis_options.report_recursive ) + { + // This option is report-and-exit. + early_shutdown(); + exit(0); + } - if ( options.print_signature_debug_info ) - rule_matcher->PrintDebug(); + if ( dns_type != DNS_PRIME ) + run_state::detail::init_run(options.interface, options.pcap_file, options.pcap_output_file, + options.use_watchdog); - file_mgr->InitMagic(); - } + if ( ! g_policy_debug ) + { + (void)setsignal(SIGTERM, sig_handler); + (void)setsignal(SIGINT, sig_handler); + (void)setsignal(SIGPIPE, SIG_IGN); + } - if ( g_policy_debug ) - // ### Add support for debug command file. - dbg_init_debugger(nullptr); + // Cooperate with nohup(1). + if ( (oldhandler = setsignal(SIGHUP, sig_handler)) != SIG_DFL ) + (void)setsignal(SIGHUP, oldhandler); - if ( ! options.pcap_file && ! options.interface ) - { - const auto& interfaces_val = id::find_val("interfaces"); - if ( interfaces_val ) - { - char* interfaces_str = interfaces_val->AsString()->Render(); + // If we were priming the DNS cache (i.e. -P was passed as an argument), flush anything + // remaining to be resolved and save the cache to disk. We can just exit now because + // we've done everything we need to do. The run loop isn't started in this case, so + // nothing else should be happening. + if ( dns_type == DNS_PRIME ) + { + dns_mgr->Resolve(); - if ( interfaces_str[0] != '\0' ) - options.interface = interfaces_str; + if ( ! dns_mgr->Save() ) + reporter->FatalError("can't update DNS cache"); - delete[] interfaces_str; - } - } - - if ( options.parse_only ) - { - if ( analysis_options.usage_issues > 0 ) - analyze_scripts(options.no_unused_warnings); - - early_shutdown(); - exit(reporter->Errors() != 0); - } - - auto init_stmts = stmts ? analyze_global_stmts(stmts) : nullptr; - - analyze_scripts(options.no_unused_warnings); - - if ( analysis_options.report_recursive ) - { - // This option is report-and-exit. - early_shutdown(); - exit(0); - } - - if ( dns_type != DNS_PRIME ) - run_state::detail::init_run(options.interface, options.pcap_file, - options.pcap_output_file, options.use_watchdog); - - if ( ! g_policy_debug ) - { - (void)setsignal(SIGTERM, sig_handler); - (void)setsignal(SIGINT, sig_handler); - (void)setsignal(SIGPIPE, SIG_IGN); - } - - // Cooperate with nohup(1). - if ( (oldhandler = setsignal(SIGHUP, sig_handler)) != SIG_DFL ) - (void)setsignal(SIGHUP, oldhandler); - - // If we were priming the DNS cache (i.e. -P was passed as an argument), flush anything - // remaining to be resolved and save the cache to disk. We can just exit now because - // we've done everything we need to do. The run loop isn't started in this case, so - // nothing else should be happening. - if ( dns_type == DNS_PRIME ) - { - dns_mgr->Resolve(); - - if ( ! dns_mgr->Save() ) - reporter->FatalError("can't update DNS cache"); - - event_mgr.Drain(); - early_shutdown(); - exit(0); - } - - // Print the ID. - if ( options.identifier_to_print ) - { - const auto& id = global_scope()->Find(*options.identifier_to_print); - if ( ! id ) - reporter->FatalError("No such ID: %s\n", options.identifier_to_print->data()); - - ODesc desc; - desc.SetQuotes(true); - desc.SetIncludeStats(true); - id->DescribeExtended(&desc); - - fprintf(stdout, "%s\n", desc.Description()); - early_shutdown(); - exit(0); - } - - if ( profiling_interval > 0 ) - { - const auto& profiling_file = id::find_val("profiling_file"); - profiling_logger = std::make_shared(profiling_file->AsFile(), - profiling_interval); - - if ( segment_profiling ) - segment_logger = profiling_logger; - } - - if ( ! run_state::reading_live && ! run_state::reading_traces ) - // Set up network_time to track real-time, since - // we don't have any other source for it. - run_state::detail::update_network_time(util::current_time()); - - if ( CPP_activation_hook ) - (*CPP_activation_hook)(); - - if ( zeek_init ) - event_mgr.Enqueue(zeek_init, Args{}); - - EventRegistry::string_list dead_handlers = event_registry->UnusedHandlers(); - - if ( ! dead_handlers.empty() && check_for_unused_event_handlers ) - { - for ( const string& handler : dead_handlers ) - reporter->Warning("event handler never invoked: %s", handler.c_str()); - } - - // Enable LeakSanitizer before zeek_init() and even before executing - // top-level statements. Even though it's not bad if a leak happens only - // once at initialization, we have to assume that script-layer code causing - // such a leak can be placed in any arbitrary event handler and potentially - // cause more severe problems. - ZEEK_LSAN_ENABLE(); - - if ( init_stmts ) - { - StmtFlowType flow; - Frame f(init_stmts->Scope()->Length(), nullptr, nullptr); - g_frame_stack.push_back(&f); - - try - { - init_stmts->Body()->Exec(&f, flow); - } - catch ( InterpreterException& ) - { - reporter->FatalError("failed to execute script statements at top-level scope"); - } - - g_frame_stack.pop_back(); - } - - if ( options.ignore_checksums ) - ignore_checksums = 1; - - if ( zeek_script_loaded ) - { - // Queue events reporting loaded scripts. - for ( const auto& file : zeek::detail::files_scanned ) - { - if ( file.skipped ) - continue; - - event_mgr.Enqueue(zeek_script_loaded, make_intrusive(file.name.c_str()), - val_mgr->Count(file.include_level)); - } - } - - reporter->ReportViaEvents(true); - - // Drain the event queue here to support the protocols framework configuring DPM event_mgr.Drain(); + early_shutdown(); + exit(0); + } - if ( reporter->Errors() > 0 && ! getenv("ZEEK_ALLOW_INIT_ERRORS") ) - reporter->FatalError("errors occurred while initializing"); + // Print the ID. + if ( options.identifier_to_print ) + { + const auto& id = global_scope()->Find(*options.identifier_to_print); + if ( ! id ) + reporter->FatalError("No such ID: %s\n", options.identifier_to_print->data()); - run_state::detail::zeek_init_done = true; - packet_mgr->DumpDebug(); - analyzer_mgr->DumpDebug(); + ODesc desc; + desc.SetQuotes(true); + desc.SetIncludeStats(true); + id->DescribeExtended(&desc); - run_state::detail::have_pending_timers = ! run_state::reading_traces && - timer_mgr->Size() > 0; + fprintf(stdout, "%s\n", desc.Description()); + early_shutdown(); + exit(0); + } - return {0, std::move(options)}; + if ( profiling_interval > 0 ) + { + const auto& profiling_file = id::find_val("profiling_file"); + profiling_logger = std::make_shared(profiling_file->AsFile(), + profiling_interval); + + if ( segment_profiling ) + segment_logger = profiling_logger; + } + + if ( ! run_state::reading_live && ! run_state::reading_traces ) + // Set up network_time to track real-time, since + // we don't have any other source for it. + run_state::detail::update_network_time(util::current_time()); + + if ( CPP_activation_hook ) + (*CPP_activation_hook)(); + + if ( zeek_init ) + event_mgr.Enqueue(zeek_init, Args{}); + + EventRegistry::string_list dead_handlers = event_registry->UnusedHandlers(); + + if ( ! dead_handlers.empty() && check_for_unused_event_handlers ) + { + for ( const string& handler : dead_handlers ) + reporter->Warning("event handler never invoked: %s", handler.c_str()); + } + + // Enable LeakSanitizer before zeek_init() and even before executing + // top-level statements. Even though it's not bad if a leak happens only + // once at initialization, we have to assume that script-layer code causing + // such a leak can be placed in any arbitrary event handler and potentially + // cause more severe problems. + ZEEK_LSAN_ENABLE(); + + if ( init_stmts ) + { + StmtFlowType flow; + Frame f(init_stmts->Scope()->Length(), nullptr, nullptr); + g_frame_stack.push_back(&f); + + try + { + init_stmts->Body()->Exec(&f, flow); + } + catch ( InterpreterException& ) + { + reporter->FatalError("failed to execute script statements at top-level scope"); + } + + g_frame_stack.pop_back(); + } + + if ( options.ignore_checksums ) + ignore_checksums = 1; + + if ( zeek_script_loaded ) + { + // Queue events reporting loaded scripts. + for ( const auto& file : zeek::detail::files_scanned ) + { + if ( file.skipped ) + continue; + + event_mgr.Enqueue(zeek_script_loaded, make_intrusive(file.name.c_str()), + val_mgr->Count(file.include_level)); + } + } + + reporter->ReportViaEvents(true); + + // Drain the event queue here to support the protocols framework configuring DPM + event_mgr.Drain(); + + if ( reporter->Errors() > 0 && ! getenv("ZEEK_ALLOW_INIT_ERRORS") ) + reporter->FatalError("errors occurred while initializing"); + + run_state::detail::zeek_init_done = true; + packet_mgr->DumpDebug(); + analyzer_mgr->DumpDebug(); + + run_state::detail::have_pending_timers = ! run_state::reading_traces && timer_mgr->Size() > 0; + + return {0, std::move(options)}; } int cleanup(bool did_run_loop) From a105ea9d80298112aa40d58f9af480f45e1d280a Mon Sep 17 00:00:00 2001 From: Tomer Lev Date: Tue, 1 Nov 2022 19:52:22 +0200 Subject: [PATCH 59/71] Rewrote usages of define(_MSC_VER) to ifdef _MSC_VER --- src/Flare.cc | 18 ++++++++---------- src/Flare.h | 6 +++--- src/main.cc | 4 ++-- src/plugin/Manager.cc | 4 ++-- src/plugin/Plugin.h | 2 +- src/util.cc | 2 +- 6 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/Flare.cc b/src/Flare.cc index 0ab45b7452..99ac2d6ede 100644 --- a/src/Flare.cc +++ b/src/Flare.cc @@ -8,7 +8,7 @@ #include "zeek/Reporter.h" -#if defined(_MSC_VER) +#ifdef _MSC_VER #include @@ -31,10 +31,8 @@ namespace zeek::detail { Flare::Flare() -#if ! defined(_MSC_VER) - : pipe(FD_CLOEXEC, FD_CLOEXEC, O_NONBLOCK, O_NONBLOCK) - { - } +#ifndef _MSC_VER + : pipe(FD_CLOEXEC, FD_CLOEXEC, O_NONBLOCK, O_NONBLOCK){} #else { WSADATA wsaData; @@ -64,7 +62,7 @@ Flare::Flare() } #endif -[[noreturn]] static void bad_pipe_op(const char* which, bool signal_safe) + [[noreturn]] static void bad_pipe_op(const char* which, bool signal_safe) { if ( signal_safe ) abort(); @@ -87,7 +85,7 @@ void Flare::Fire(bool signal_safe) for ( ;; ) { -#if ! defined(_MSC_VER) +#ifndef _MSC_VER int n = write(pipe.WriteFD(), &tmp, 1); #else @@ -99,7 +97,7 @@ void Flare::Fire(bool signal_safe) if ( n < 0 ) { -#if defined(_MSC_VER) +#ifdef _MSC_VER errno = WSAGetLastError(); bad_pipe_op("send", signal_safe); #endif @@ -125,7 +123,7 @@ int Flare::Extinguish(bool signal_safe) for ( ;; ) { -#if ! defined(_MSC_VER) +#ifndef _MSC_VER int n = read(pipe.ReadFD(), &tmp, sizeof(tmp)); #else int n = recv(recvfd, tmp, sizeof(tmp), 0); @@ -136,7 +134,7 @@ int Flare::Extinguish(bool signal_safe) // Pipe may not be empty yet: try again. continue; } -#if defined(_MSC_VER) +#ifdef _MSC_VER if ( WSAGetLastError() == WSAEWOULDBLOCK ) break; errno = WSAGetLastError(); diff --git a/src/Flare.h b/src/Flare.h index 6ded8af25e..affae42dfd 100644 --- a/src/Flare.h +++ b/src/Flare.h @@ -2,7 +2,7 @@ #pragma once -#if ! defined(_MSC_VER) +#ifndef _MSC_VER #include "Pipe.h" #endif @@ -25,7 +25,7 @@ public: * Fire()'d and not yet Extinguished()'d. */ int FD() const -#if ! defined(_MSC_VER) +#ifndef _MSC_VER { return pipe.ReadFD(); } @@ -52,7 +52,7 @@ public: int Extinguish(bool signal_safe = false); private: -#if ! defined(_MSC_VER) +#ifndef _MSC_VER Pipe pipe; #else int sendfd, recvfd; diff --git a/src/main.cc b/src/main.cc index 2796ebf1ad..1de09fc9e7 100644 --- a/src/main.cc +++ b/src/main.cc @@ -9,13 +9,13 @@ #include "zeek/supervisor/Supervisor.h" #include "zeek/zeek-setup.h" -#if defined(_MSC_VER) +#ifdef _MSC_VER #include // For _O_BINARY. #endif int main(int argc, char** argv) { -#if defined(_MSC_VER) +#ifdef _MSC_VER _setmode(_fileno(stdout), _O_BINARY); _setmode(_fileno(stderr), _O_BINARY); #endif diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index 7828af3656..a535d0f43e 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -3,7 +3,7 @@ #include "zeek/plugin/Manager.h" #include -#if ! defined(_MSC_VER) +#ifndef _MSC_VER #include #include #endif @@ -163,7 +163,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ std::vector* errors) { // Loading dynamic plugins is not currently supported on Windows platform. -#if defined(_MSC_VER) +#ifdef _MSC_VER return false; #else errors->clear(); // caller should pass it in empty, but just to be sure diff --git a/src/plugin/Plugin.h b/src/plugin/Plugin.h index 333df81538..415bc867ae 100644 --- a/src/plugin/Plugin.h +++ b/src/plugin/Plugin.h @@ -25,7 +25,7 @@ struct Field; namespace zeek { -#if defined(_MSC_VER) +#ifdef _MSC_VER #undef VOID #endif diff --git a/src/util.cc b/src/util.cc index bc85bb166e..ca17648fc1 100644 --- a/src/util.cc +++ b/src/util.cc @@ -2539,7 +2539,7 @@ static void strerror_r_helper(int result, char* buf, size_t buflen) void zeek_strerror_r(int zeek_errno, char* buf, size_t buflen) { -#if defined(_MSC_VER) +#ifdef _MSC_VER auto str = "Error number: " + std::to_string(zeek_errno); auto res = str.data(); #else From 73e749a16232fb42df1857368447305269dd5abc Mon Sep 17 00:00:00 2001 From: Tomer Lev Date: Wed, 2 Nov 2022 10:13:58 +0200 Subject: [PATCH 60/71] Clang format again but now with v13.0.1 --- src/DNS_Mapping.cc | 4 +- src/DNS_Mgr.cc | 2 +- src/Flare.cc | 6 +- src/Hash.h | 2 +- src/IP.cc | 2 +- src/iosource/BPF_Program.cc | 3 + src/logging/writers/ascii/Ascii.cc | 2 +- src/logging/writers/sqlite/SQLite.cc | 2 +- src/net_util.h | 4 +- src/plugin/Plugin.h | 2 +- src/util.cc | 4 +- src/zeek-setup.cc | 391 ++++++++++++++------------- 12 files changed, 215 insertions(+), 209 deletions(-) diff --git a/src/DNS_Mapping.cc b/src/DNS_Mapping.cc index f5264f5b03..ce9da8bb4b 100644 --- a/src/DNS_Mapping.cc +++ b/src/DNS_Mapping.cc @@ -296,8 +296,8 @@ TEST_CASE("dns_mapping init addr") TEST_CASE("dns_mapping save reload") { - // TODO: this test uses fmemopen and mkdtemp, both of which aren't available on - // Windows. We'll have to figure out another way to do this test there. + // TODO: this test uses fmemopen and mkdtemp, both of which aren't available on + // Windows. We'll have to figure out another way to do this test there. #ifndef _MSC_VER IPAddr addr("1.2.3.4"); in4_addr in4; diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 70dff59df0..6390d1de48 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -1613,7 +1613,7 @@ void TestDNS_Mgr::Process() TEST_CASE("dns_mgr priming" * doctest::skip(true)) { - // TODO: This test uses mkdtemp, which isn't available on Windows. + // TODO: This test uses mkdtemp, which isn't available on Windows. #ifndef _MSC_VER char prefix[] = "/tmp/zeek-unit-test-XXXXXX"; auto tmpdir = mkdtemp(prefix); diff --git a/src/Flare.cc b/src/Flare.cc index 99ac2d6ede..f2dbc60155 100644 --- a/src/Flare.cc +++ b/src/Flare.cc @@ -32,7 +32,9 @@ namespace zeek::detail Flare::Flare() #ifndef _MSC_VER - : pipe(FD_CLOEXEC, FD_CLOEXEC, O_NONBLOCK, O_NONBLOCK){} + : pipe(FD_CLOEXEC, FD_CLOEXEC, O_NONBLOCK, O_NONBLOCK) + { + } #else { WSADATA wsaData; @@ -62,7 +64,7 @@ Flare::Flare() } #endif - [[noreturn]] static void bad_pipe_op(const char* which, bool signal_safe) +[[noreturn]] static void bad_pipe_op(const char* which, bool signal_safe) { if ( signal_safe ) abort(); diff --git a/src/Hash.h b/src/Hash.h index b727fc7243..796763d509 100644 --- a/src/Hash.h +++ b/src/Hash.h @@ -372,7 +372,7 @@ protected: void Set(double d); void Set(const void* p); - union { + union { bool b; int i; zeek_int_t bi; diff --git a/src/IP.cc b/src/IP.cc index d7ec2dbeb1..4de4506387 100644 --- a/src/IP.cc +++ b/src/IP.cc @@ -546,7 +546,7 @@ void IPv6_Hdr_Chain::Init(const struct ip6_hdr* ip6, int total_len, bool set_nex total_len -= cur_len; } while ( current_type != IPPROTO_FRAGMENT && current_type != IPPROTO_ESP && - current_type != IPPROTO_MOBILITY && isIPv6ExtHeader(next_type) ); + current_type != IPPROTO_MOBILITY && isIPv6ExtHeader(next_type) ); } bool IPv6_Hdr_Chain::IsFragment() const diff --git a/src/iosource/BPF_Program.cc b/src/iosource/BPF_Program.cc index 3717ce0d97..31da7536f9 100644 --- a/src/iosource/BPF_Program.cc +++ b/src/iosource/BPF_Program.cc @@ -4,8 +4,11 @@ #include "zeek/zeek-config.h" +// clang-format off +// Include order is required here for a working build on Windows. #include #include +// clang-format on #include #include "zeek/util.h" diff --git a/src/logging/writers/ascii/Ascii.cc b/src/logging/writers/ascii/Ascii.cc index 9247ec6a0b..33793ada4a 100644 --- a/src/logging/writers/ascii/Ascii.cc +++ b/src/logging/writers/ascii/Ascii.cc @@ -111,7 +111,7 @@ static std::string prefix_basename_with(const std::string& path, const std::stri TEST_CASE("writers.ascii prefix_basename_with") { #ifdef _MSC_VER - // TODO: adapt this test to Windows paths + // TODO: adapt this test to Windows paths #else CHECK(prefix_basename_with("a/conn.log", ".shadow.") == "a/.shadow.conn.log"); CHECK(prefix_basename_with("/a/conn.log", ".shadow.") == "/a/.shadow.conn.log"); diff --git a/src/logging/writers/sqlite/SQLite.cc b/src/logging/writers/sqlite/SQLite.cc index 5471bf949a..9eee5596e9 100644 --- a/src/logging/writers/sqlite/SQLite.cc +++ b/src/logging/writers/sqlite/SQLite.cc @@ -63,7 +63,7 @@ string SQLite::GetTableType(int arg_type, int arg_subtype) case TYPE_INT: case TYPE_COUNT: case TYPE_PORT: // note that we do not save the protocol at the moment. Just like in the - // case of the ascii-writer + // case of the ascii-writer type = "integer"; break; diff --git a/src/net_util.h b/src/net_util.h index e3bd265640..7031eed81e 100644 --- a/src/net_util.h +++ b/src/net_util.h @@ -64,7 +64,7 @@ struct ip6_ext struct ip6_hdr { - union { + union { struct ip6_hdrctl { uint32_t ip6_un1_flow; /* 4 bits version, 8 bits TC, 20 bits @@ -335,7 +335,7 @@ inline float htonf(float f) inline uint64_t ntohll(uint64_t i) { u_char c; - union { + union { uint64_t i; u_char c[8]; } x; diff --git a/src/plugin/Plugin.h b/src/plugin/Plugin.h index 415bc867ae..ef1ae23308 100644 --- a/src/plugin/Plugin.h +++ b/src/plugin/Plugin.h @@ -661,7 +661,7 @@ public: private: Type type; - union { + union { bool bool_; double double_; const Event* event; diff --git a/src/util.cc b/src/util.cc index ca17648fc1..8f6f83538e 100644 --- a/src/util.cc +++ b/src/util.cc @@ -631,7 +631,7 @@ string flatten_script_name(const string& name, const string& prefix) TEST_CASE("util normalize_path") { #ifdef _MSC_VER - // TODO: adapt these tests to Windows + // TODO: adapt these tests to Windows #else CHECK(normalize_path("/1/2/3") == "/1/2/3"); CHECK(normalize_path("/1/./2/3") == "/1/2/3"); @@ -2737,7 +2737,7 @@ string json_escape_utf8(const char* val, size_t val_size, bool escape_printable_ TEST_CASE("util filesystem") { #ifdef _MSC_VER - // TODO: adapt these tests to Windows paths + // TODO: adapt these tests to Windows paths #else zeek::filesystem::path path1("/a/b"); CHECK(path1.is_absolute()); diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index 7f89378a16..cd7ef7f839 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -887,228 +887,229 @@ SetupResult setup(int argc, char** argv, Options* zopts) #ifdef USE_PERFTOOLS_DEBUG } #endif - set_signal_mask(false); + set_signal_mask(false); - if ( reporter->Errors() > 0 ) - { - early_shutdown(); - exit(1); - } - - reporter->InitOptions(); - KeyedHash::InitOptions(); - zeekygen_mgr->GenerateDocs(); - - if ( options.pcap_filter ) - { - const auto& id = global_scope()->Find("cmd_line_bpf_filter"); - - if ( ! id ) - reporter->InternalError("global cmd_line_bpf_filter not defined"); - - id->SetVal(make_intrusive(*options.pcap_filter)); - } - - std::vector all_signature_files; - - // Append signature files given on the command line - for ( const auto& sf : options.signature_files ) - all_signature_files.emplace_back(sf); - - // Append signature files defined in "signature_files" script option - for ( auto&& sf : get_script_signature_files() ) - all_signature_files.emplace_back(std::move(sf)); - - // Append signature files defined in @load-sigs - for ( const auto& sf : zeek::detail::sig_files ) - all_signature_files.emplace_back(sf); - - if ( ! all_signature_files.empty() ) - { - rule_matcher = new RuleMatcher(options.signature_re_level); - if ( ! rule_matcher->ReadFiles(all_signature_files) ) + if ( reporter->Errors() > 0 ) { early_shutdown(); exit(1); } - if ( options.print_signature_debug_info ) - rule_matcher->PrintDebug(); + reporter->InitOptions(); + KeyedHash::InitOptions(); + zeekygen_mgr->GenerateDocs(); - file_mgr->InitMagic(); - } - - if ( g_policy_debug ) - // ### Add support for debug command file. - dbg_init_debugger(nullptr); - - if ( ! options.pcap_file && ! options.interface ) - { - const auto& interfaces_val = id::find_val("interfaces"); - if ( interfaces_val ) + if ( options.pcap_filter ) { - char* interfaces_str = interfaces_val->AsString()->Render(); + const auto& id = global_scope()->Find("cmd_line_bpf_filter"); - if ( interfaces_str[0] != '\0' ) - options.interface = interfaces_str; + if ( ! id ) + reporter->InternalError("global cmd_line_bpf_filter not defined"); - delete[] interfaces_str; + id->SetVal(make_intrusive(*options.pcap_filter)); } - } - if ( options.parse_only ) - { - if ( analysis_options.usage_issues > 0 ) - analyze_scripts(options.no_unused_warnings); + std::vector all_signature_files; - early_shutdown(); - exit(reporter->Errors() != 0); - } + // Append signature files given on the command line + for ( const auto& sf : options.signature_files ) + all_signature_files.emplace_back(sf); - auto init_stmts = stmts ? analyze_global_stmts(stmts) : nullptr; + // Append signature files defined in "signature_files" script option + for ( auto&& sf : get_script_signature_files() ) + all_signature_files.emplace_back(std::move(sf)); - analyze_scripts(options.no_unused_warnings); + // Append signature files defined in @load-sigs + for ( const auto& sf : zeek::detail::sig_files ) + all_signature_files.emplace_back(sf); - if ( analysis_options.report_recursive ) - { - // This option is report-and-exit. - early_shutdown(); - exit(0); - } + if ( ! all_signature_files.empty() ) + { + rule_matcher = new RuleMatcher(options.signature_re_level); + if ( ! rule_matcher->ReadFiles(all_signature_files) ) + { + early_shutdown(); + exit(1); + } - if ( dns_type != DNS_PRIME ) - run_state::detail::init_run(options.interface, options.pcap_file, options.pcap_output_file, - options.use_watchdog); + if ( options.print_signature_debug_info ) + rule_matcher->PrintDebug(); - if ( ! g_policy_debug ) - { - (void)setsignal(SIGTERM, sig_handler); - (void)setsignal(SIGINT, sig_handler); - (void)setsignal(SIGPIPE, SIG_IGN); - } + file_mgr->InitMagic(); + } - // Cooperate with nohup(1). - if ( (oldhandler = setsignal(SIGHUP, sig_handler)) != SIG_DFL ) - (void)setsignal(SIGHUP, oldhandler); + if ( g_policy_debug ) + // ### Add support for debug command file. + dbg_init_debugger(nullptr); - // If we were priming the DNS cache (i.e. -P was passed as an argument), flush anything - // remaining to be resolved and save the cache to disk. We can just exit now because - // we've done everything we need to do. The run loop isn't started in this case, so - // nothing else should be happening. - if ( dns_type == DNS_PRIME ) - { - dns_mgr->Resolve(); + if ( ! options.pcap_file && ! options.interface ) + { + const auto& interfaces_val = id::find_val("interfaces"); + if ( interfaces_val ) + { + char* interfaces_str = interfaces_val->AsString()->Render(); - if ( ! dns_mgr->Save() ) - reporter->FatalError("can't update DNS cache"); + if ( interfaces_str[0] != '\0' ) + options.interface = interfaces_str; + delete[] interfaces_str; + } + } + + if ( options.parse_only ) + { + if ( analysis_options.usage_issues > 0 ) + analyze_scripts(options.no_unused_warnings); + + early_shutdown(); + exit(reporter->Errors() != 0); + } + + auto init_stmts = stmts ? analyze_global_stmts(stmts) : nullptr; + + analyze_scripts(options.no_unused_warnings); + + if ( analysis_options.report_recursive ) + { + // This option is report-and-exit. + early_shutdown(); + exit(0); + } + + if ( dns_type != DNS_PRIME ) + run_state::detail::init_run(options.interface, options.pcap_file, + options.pcap_output_file, options.use_watchdog); + + if ( ! g_policy_debug ) + { + (void)setsignal(SIGTERM, sig_handler); + (void)setsignal(SIGINT, sig_handler); + (void)setsignal(SIGPIPE, SIG_IGN); + } + + // Cooperate with nohup(1). + if ( (oldhandler = setsignal(SIGHUP, sig_handler)) != SIG_DFL ) + (void)setsignal(SIGHUP, oldhandler); + + // If we were priming the DNS cache (i.e. -P was passed as an argument), flush anything + // remaining to be resolved and save the cache to disk. We can just exit now because + // we've done everything we need to do. The run loop isn't started in this case, so + // nothing else should be happening. + if ( dns_type == DNS_PRIME ) + { + dns_mgr->Resolve(); + + if ( ! dns_mgr->Save() ) + reporter->FatalError("can't update DNS cache"); + + event_mgr.Drain(); + early_shutdown(); + exit(0); + } + + // Print the ID. + if ( options.identifier_to_print ) + { + const auto& id = global_scope()->Find(*options.identifier_to_print); + if ( ! id ) + reporter->FatalError("No such ID: %s\n", options.identifier_to_print->data()); + + ODesc desc; + desc.SetQuotes(true); + desc.SetIncludeStats(true); + id->DescribeExtended(&desc); + + fprintf(stdout, "%s\n", desc.Description()); + early_shutdown(); + exit(0); + } + + if ( profiling_interval > 0 ) + { + const auto& profiling_file = id::find_val("profiling_file"); + profiling_logger = std::make_shared(profiling_file->AsFile(), + profiling_interval); + + if ( segment_profiling ) + segment_logger = profiling_logger; + } + + if ( ! run_state::reading_live && ! run_state::reading_traces ) + // Set up network_time to track real-time, since + // we don't have any other source for it. + run_state::detail::update_network_time(util::current_time()); + + if ( CPP_activation_hook ) + (*CPP_activation_hook)(); + + if ( zeek_init ) + event_mgr.Enqueue(zeek_init, Args{}); + + EventRegistry::string_list dead_handlers = event_registry->UnusedHandlers(); + + if ( ! dead_handlers.empty() && check_for_unused_event_handlers ) + { + for ( const string& handler : dead_handlers ) + reporter->Warning("event handler never invoked: %s", handler.c_str()); + } + + // Enable LeakSanitizer before zeek_init() and even before executing + // top-level statements. Even though it's not bad if a leak happens only + // once at initialization, we have to assume that script-layer code causing + // such a leak can be placed in any arbitrary event handler and potentially + // cause more severe problems. + ZEEK_LSAN_ENABLE(); + + if ( init_stmts ) + { + StmtFlowType flow; + Frame f(init_stmts->Scope()->Length(), nullptr, nullptr); + g_frame_stack.push_back(&f); + + try + { + init_stmts->Body()->Exec(&f, flow); + } + catch ( InterpreterException& ) + { + reporter->FatalError("failed to execute script statements at top-level scope"); + } + + g_frame_stack.pop_back(); + } + + if ( options.ignore_checksums ) + ignore_checksums = 1; + + if ( zeek_script_loaded ) + { + // Queue events reporting loaded scripts. + for ( const auto& file : zeek::detail::files_scanned ) + { + if ( file.skipped ) + continue; + + event_mgr.Enqueue(zeek_script_loaded, make_intrusive(file.name.c_str()), + val_mgr->Count(file.include_level)); + } + } + + reporter->ReportViaEvents(true); + + // Drain the event queue here to support the protocols framework configuring DPM event_mgr.Drain(); - early_shutdown(); - exit(0); - } - // Print the ID. - if ( options.identifier_to_print ) - { - const auto& id = global_scope()->Find(*options.identifier_to_print); - if ( ! id ) - reporter->FatalError("No such ID: %s\n", options.identifier_to_print->data()); + if ( reporter->Errors() > 0 && ! getenv("ZEEK_ALLOW_INIT_ERRORS") ) + reporter->FatalError("errors occurred while initializing"); - ODesc desc; - desc.SetQuotes(true); - desc.SetIncludeStats(true); - id->DescribeExtended(&desc); + run_state::detail::zeek_init_done = true; + packet_mgr->DumpDebug(); + analyzer_mgr->DumpDebug(); - fprintf(stdout, "%s\n", desc.Description()); - early_shutdown(); - exit(0); - } + run_state::detail::have_pending_timers = ! run_state::reading_traces && + timer_mgr->Size() > 0; - if ( profiling_interval > 0 ) - { - const auto& profiling_file = id::find_val("profiling_file"); - profiling_logger = std::make_shared(profiling_file->AsFile(), - profiling_interval); - - if ( segment_profiling ) - segment_logger = profiling_logger; - } - - if ( ! run_state::reading_live && ! run_state::reading_traces ) - // Set up network_time to track real-time, since - // we don't have any other source for it. - run_state::detail::update_network_time(util::current_time()); - - if ( CPP_activation_hook ) - (*CPP_activation_hook)(); - - if ( zeek_init ) - event_mgr.Enqueue(zeek_init, Args{}); - - EventRegistry::string_list dead_handlers = event_registry->UnusedHandlers(); - - if ( ! dead_handlers.empty() && check_for_unused_event_handlers ) - { - for ( const string& handler : dead_handlers ) - reporter->Warning("event handler never invoked: %s", handler.c_str()); - } - - // Enable LeakSanitizer before zeek_init() and even before executing - // top-level statements. Even though it's not bad if a leak happens only - // once at initialization, we have to assume that script-layer code causing - // such a leak can be placed in any arbitrary event handler and potentially - // cause more severe problems. - ZEEK_LSAN_ENABLE(); - - if ( init_stmts ) - { - StmtFlowType flow; - Frame f(init_stmts->Scope()->Length(), nullptr, nullptr); - g_frame_stack.push_back(&f); - - try - { - init_stmts->Body()->Exec(&f, flow); - } - catch ( InterpreterException& ) - { - reporter->FatalError("failed to execute script statements at top-level scope"); - } - - g_frame_stack.pop_back(); - } - - if ( options.ignore_checksums ) - ignore_checksums = 1; - - if ( zeek_script_loaded ) - { - // Queue events reporting loaded scripts. - for ( const auto& file : zeek::detail::files_scanned ) - { - if ( file.skipped ) - continue; - - event_mgr.Enqueue(zeek_script_loaded, make_intrusive(file.name.c_str()), - val_mgr->Count(file.include_level)); - } - } - - reporter->ReportViaEvents(true); - - // Drain the event queue here to support the protocols framework configuring DPM - event_mgr.Drain(); - - if ( reporter->Errors() > 0 && ! getenv("ZEEK_ALLOW_INIT_ERRORS") ) - reporter->FatalError("errors occurred while initializing"); - - run_state::detail::zeek_init_done = true; - packet_mgr->DumpDebug(); - analyzer_mgr->DumpDebug(); - - run_state::detail::have_pending_timers = ! run_state::reading_traces && timer_mgr->Size() > 0; - - return {0, std::move(options)}; + return {0, std::move(options)}; } int cleanup(bool did_run_loop) From 2e3425000f3443e243a43c34b266f95d51bf8fe8 Mon Sep 17 00:00:00 2001 From: voidbar Date: Thu, 3 Nov 2022 19:26:44 +0200 Subject: [PATCH 61/71] Apply suggestions from code review Co-authored-by: Tim Wojtulewicz --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e811f2948f..2b960d522b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -420,7 +420,7 @@ FindRequiredPackage(FLEX) FindRequiredPackage(BISON) FindRequiredPackage(PCAP) FindRequiredPackage(OpenSSL) -if ( NOT WIN32 ) +if ( NOT MSVC ) FindRequiredPackage(BIND) endif () FindRequiredPackage(ZLIB) @@ -733,7 +733,7 @@ if ( ${CMAKE_SYSTEM_NAME} MATCHES Linux ) endif () set(DEFAULT_ZEEKPATH_PATHS . ${ZEEK_SCRIPT_INSTALL_PATH} ${ZEEK_SCRIPT_INSTALL_PATH}/policy ${ZEEK_SCRIPT_INSTALL_PATH}/site ${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins) -if ( WIN32 ) +if ( MSVC ) list(JOIN DEFAULT_ZEEKPATH_PATHS ";" DEFAULT_ZEEKPATH) else () list(JOIN DEFAULT_ZEEKPATH_PATHS ":" DEFAULT_ZEEKPATH) @@ -791,7 +791,7 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/cmake DESTINATION share/zeek USE_SOURCE_PERMISSIONS PATTERN ".git" EXCLUDE) -if ( NOT WIN32 ) +if ( NOT MSVC ) # Install wrapper script for Bro-to-Zeek renaming. include(InstallShellScript) include(InstallSymlink) From 9a74be155869bb07d6c7d093d8e9518441b8785a Mon Sep 17 00:00:00 2001 From: voidbar Date: Thu, 10 Nov 2022 18:43:47 +0200 Subject: [PATCH 62/71] Update src/packet_analysis/protocol/ip/IP.cc Co-authored-by: Tim Wojtulewicz --- src/packet_analysis/protocol/ip/IP.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packet_analysis/protocol/ip/IP.cc b/src/packet_analysis/protocol/ip/IP.cc index ebed559a2b..d06d14a7a5 100644 --- a/src/packet_analysis/protocol/ip/IP.cc +++ b/src/packet_analysis/protocol/ip/IP.cc @@ -62,7 +62,7 @@ bool IPAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* packet) return false; } - packet->ip_hdr = std::make_shared((const struct ip6_hdr*)data, false, + ip_hdr = std::make_shared((const struct ip6_hdr*)data, false, static_cast(len)); packet->l3_proto = L3_IPV6; } From e2be5ddc0c1af6b3e2146efae3b010b25e6cd0da Mon Sep 17 00:00:00 2001 From: Tomer Lev Date: Thu, 10 Nov 2022 19:01:29 +0200 Subject: [PATCH 63/71] Added fixes suggested in PR --- src/File.cc | 10 ++++++++-- src/util.cc | 8 +++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/File.cc b/src/File.cc index b2a167740f..e9767deb29 100644 --- a/src/File.cc +++ b/src/File.cc @@ -202,10 +202,16 @@ void File::SetBuf(bool arg_buffered) if ( ! f ) return; +#ifndef _MSC_VER if ( setvbuf(f, NULL, arg_buffered ? _IOFBF : _IOLBF, 0) != 0 ) - reporter->Error("setvbuf failed"); +#else + // TODO: this turns off buffering altogether because Windows wants us to pass a valid + // buffer and length if we're going to pass one of the other modes. We need to + // investigate the performance ramifications of this. + if ( setvbuf(f, NULL, _IONBF, 0) != 0 ) +#endif reporter->Error("setvbuf failed"); - buffered = arg_buffered; + buffered = arg_buffered; } bool File::Close() diff --git a/src/util.cc b/src/util.cc index 8f6f83538e..c3c8506bf7 100644 --- a/src/util.cc +++ b/src/util.cc @@ -2038,9 +2038,15 @@ RETSIGTYPE sig_handler(int signo); double current_time(bool real) { struct timeval tv; +#ifdef _MSC_VER + auto now = std::chrono::system_clock::now(); + auto ms = std::chrono::duration_cast(now.time_since_epoch()); + tv.tv_sec = ms.count() / 1000; + tv.tv_usec = (ms.count() % 1000) * 1000; +#else if ( gettimeofday(&tv, 0) < 0 ) reporter->InternalError("gettimeofday failed in current_time()"); - +#endif double t = double(tv.tv_sec) + double(tv.tv_usec) / 1e6; if ( ! run_state::pseudo_realtime || real || ! iosource_mgr || ! iosource_mgr->GetPktSrc() ) From d7474e2aa2904a3d2201c0c4de97d868b4ee4a88 Mon Sep 17 00:00:00 2001 From: Tomer Lev Date: Thu, 10 Nov 2022 19:04:37 +0200 Subject: [PATCH 64/71] Removed const, Windows build is now working --- src/ScriptValidation.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ScriptValidation.cc b/src/ScriptValidation.cc index 73942246ea..1555346985 100644 --- a/src/ScriptValidation.cc +++ b/src/ScriptValidation.cc @@ -85,7 +85,7 @@ private: bool NextStmtIsValid() { return stmt_depths[STMT_FOR] > 0 || stmt_depths[STMT_WHILE] > 0; } - std::unordered_map stmt_depths; + std::unordered_map stmt_depths; int hook_depth = 0; }; From 9a3855cc382e538237620654a0fc12271aa0a100 Mon Sep 17 00:00:00 2001 From: Tomer Lev Date: Thu, 10 Nov 2022 19:09:57 +0200 Subject: [PATCH 65/71] Changed implementation from std::map to std::unordered_map of Val.cc --- src/Val.cc | 7 +++---- src/Val.h | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Val.cc b/src/Val.cc index a20e00570d..33c2cb2b0a 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -3972,11 +3972,10 @@ const PortValPtr& ValManager::Port(uint32_t port_num, TransportProto port_type) port_num = 0; } - std::pair key{port_num, port_type}; - if ( ports.find(key) == ports.end() ) - ports[key] = IntrusivePtr{AdoptRef{}, new PortVal(PortVal::Mask(port_num, port_type))}; + if ( ports.find(port_num) == ports.end() ) + ports[port_num] = IntrusivePtr{AdoptRef{}, new PortVal(PortVal::Mask(port_num, port_type))}; - return ports[key]; + return ports[port_num]; } const PortValPtr& ValManager::Port(uint32_t port_num) diff --git a/src/Val.h b/src/Val.h index 60e1aa73dd..8cd0da05a2 100644 --- a/src/Val.h +++ b/src/Val.h @@ -339,7 +339,7 @@ public: const PortValPtr& Port(uint32_t port_num); private: - std::map, PortValPtr> ports; + std::unordered_map ports; std::array counts; std::array ints; StringValPtr empty_string; From 194960eafa2e9aa68fbe1e2b97d9572a24b8d5a4 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 10 Nov 2022 11:17:53 -0700 Subject: [PATCH 66/71] Mark a few EventHandler methods const --- src/EventHandler.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/EventHandler.h b/src/EventHandler.h index 64f134786e..b2137be402 100644 --- a/src/EventHandler.h +++ b/src/EventHandler.h @@ -38,19 +38,19 @@ public: explicit operator bool() const; void SetUsed() { used = true; } - bool Used() { return used; } + bool Used() const { return used; } // Handlers marked as error handlers will not be called recursively to // avoid infinite loops if they trigger a similar error themselves. void SetErrorHandler() { error_handler = true; } - bool ErrorHandler() { return error_handler; } + bool ErrorHandler() const { return error_handler; } void SetEnable(bool arg_enable) { enabled = arg_enable; } // Flags the event as interesting even if there is no body defined. In // particular, this will then still pass the event on to plugins. void SetGenerateAlways() { generate_always = true; } - bool GenerateAlways() { return generate_always; } + bool GenerateAlways() const { return generate_always; } private: void NewEvent(zeek::Args* vl); // Raise new_event() meta event. From 3a963f080e238a63c46e648cb4ce70ccc58e0281 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 10 Nov 2022 11:20:07 -0700 Subject: [PATCH 67/71] Add EventHandler version of stats plugin --- src/EventHandler.cc | 2 ++ src/EventHandler.h | 3 +++ src/stats.bif | 25 +++++++++++++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/src/EventHandler.cc b/src/EventHandler.cc index 09ba72c27d..62a616a619 100644 --- a/src/EventHandler.cc +++ b/src/EventHandler.cc @@ -52,6 +52,8 @@ void EventHandler::SetFunc(FuncPtr f) void EventHandler::Call(Args* vl, bool no_remote) { + call_count++; + if ( new_event ) NewEvent(vl); diff --git a/src/EventHandler.h b/src/EventHandler.h index b2137be402..c8947f1082 100644 --- a/src/EventHandler.h +++ b/src/EventHandler.h @@ -52,6 +52,8 @@ public: void SetGenerateAlways() { generate_always = true; } bool GenerateAlways() const { return generate_always; } + uint64_t CallCount() const { return call_count; } + private: void NewEvent(zeek::Args* vl); // Raise new_event() meta event. @@ -62,6 +64,7 @@ private: bool enabled; bool error_handler; // this handler reports error messages. bool generate_always; + uint64_t call_count = 0; std::unordered_set auto_publish; }; diff --git a/src/stats.bif b/src/stats.bif index ba5b94efae..d7131da482 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -484,3 +484,28 @@ function get_reporter_stats%(%): ReporterStats return r; %} + +function get_event_handler_call_counts%(%): EventNameStats + %{ + auto rval = zeek::make_intrusive(zeek::id::find_type("EventNameStats")); + const auto& recordType = zeek::id::find_type("EventNameCounter"); + + int i = 0; + const auto& events = event_registry->UsedHandlers(); + for ( const auto& name : events ) + { + auto handler = event_registry->Lookup(name); + auto call_count = handler->CallCount(); + + if ( call_count > 0 ) + { + auto eventStatRecord = zeek::make_intrusive(recordType); + eventStatRecord->Assign(0, zeek::make_intrusive(name)); + eventStatRecord->Assign(1, zeek::val_mgr->Count(handler->CallCount())); + rval->Assign(i, std::move(eventStatRecord)); + i++; + } + } + + return rval; + %} From 5d5f5de1d1dcfbfa78692f5d55ef5e4bc9818707 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 10 Nov 2022 12:39:44 -0700 Subject: [PATCH 68/71] Remove statistics plugin in favor of stats bif --- scripts/base/init-bare.zeek | 2 +- src/CMakeLists.txt | 3 +- src/statistics/CMakeLists.txt | 9 --- src/statistics/Plugin.cc | 57 ------------------- src/statistics/Plugin.h | 30 ---------- src/statistics/statistics.bif | 29 ---------- src/stats.bif | 5 ++ .../canonified_loaded_scripts.log | 1 - .../canonified_loaded_scripts.log | 1 - testing/btest/Baseline/plugins.hooks/output | 6 -- .../plugins.plugin-load-dependency/output | 1 - 11 files changed, 7 insertions(+), 137 deletions(-) delete mode 100644 src/statistics/CMakeLists.txt delete mode 100644 src/statistics/Plugin.cc delete mode 100644 src/statistics/Plugin.h delete mode 100644 src/statistics/statistics.bif diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index f6f87769df..767c33cb1a 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -794,7 +794,7 @@ type ReporterStats: record { ## Statistics about how many times each event name is queued. ## -## .. zeek:see:: Statistics::get_and_reset_event_name_statistics +## .. zeek:see:: get_event_handler_call_counts type EventNameCounter: record { ## Name of the zeek event. name: string &log; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2fe0a31c1e..91c7e6a7f6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -197,7 +197,6 @@ add_subdirectory(file_analysis) add_subdirectory(input) add_subdirectory(iosource) add_subdirectory(logging) -add_subdirectory(statistics) add_subdirectory(probabilistic) add_subdirectory(session) @@ -545,7 +544,7 @@ if (ZEEK_STANDALONE) ${bro_PLUGIN_LIBS} ) target_link_libraries(zeek ${bro_PLUGIN_LINK_LIBS} ${zeekdeps} ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) - + # Export symbols from zeek executable for use by plugins set_target_properties(zeek PROPERTIES ENABLE_EXPORTS TRUE) diff --git a/src/statistics/CMakeLists.txt b/src/statistics/CMakeLists.txt deleted file mode 100644 index 11538b5cf0..0000000000 --- a/src/statistics/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ - -include(ZeekPlugin) - -include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) - -zeek_plugin_begin(Zeek Statistics) -zeek_plugin_cc(Plugin.cc) -bif_target(statistics.bif) -zeek_plugin_end() diff --git a/src/statistics/Plugin.cc b/src/statistics/Plugin.cc deleted file mode 100644 index 8870580d82..0000000000 --- a/src/statistics/Plugin.cc +++ /dev/null @@ -1,57 +0,0 @@ - -#include "Plugin.h" - -#include - -#include "zeek/Conn.h" -#include "zeek/Desc.h" -#include "zeek/Event.h" -#include "zeek/Func.h" -#include "zeek/RunState.h" -#include "zeek/threading/Formatter.h" - -#include "statistics.bif.h" - -namespace zeek::plugin::statistics - { -Plugin plugin; - } - -using namespace zeek::plugin::statistics; - -zeek::plugin::Configuration Plugin::Configure() - { - zeek::plugin::Configuration config; - config.name = "Statistics"; - config.description = "Statistics module"; - config.version.major = 1; - config.version.minor = 0; - config.version.patch = 0; - return config; - } - -bool Plugin::HookQueueEvent(zeek::Event* event) - { - const char* name = event->Handler()->Name(); - - std::lock_guard scopedLock(m_lock); - if ( m_eventNameCounters.find(name) == m_eventNameCounters.end() ) - { - m_eventNameCounters[name] = 0; - } - m_eventNameCounters[name]++; - return false; - } - -std::unordered_map Plugin::GetAndResetEventStatistics() - { - std::lock_guard scopedLock(m_lock); - std::unordered_map result(m_eventNameCounters); - m_eventNameCounters.clear(); - return result; - } - -void Plugin::StartEventNamesStatisticsMonitor() - { - EnableHook(zeek::plugin::HOOK_QUEUE_EVENT); - } diff --git a/src/statistics/Plugin.h b/src/statistics/Plugin.h deleted file mode 100644 index d0eea47f82..0000000000 --- a/src/statistics/Plugin.h +++ /dev/null @@ -1,30 +0,0 @@ - -#pragma once - -#include -#include -#include - -#include "zeek/plugin/Plugin.h" - -namespace zeek::plugin::statistics - { - -class Plugin : public zeek::plugin::Plugin - { -protected: - bool HookQueueEvent(zeek::Event* event) override; - - zeek::plugin::Configuration Configure() override; - -public: - std::unordered_map GetAndResetEventStatistics(); - void StartEventNamesStatisticsMonitor(); - -private: - std::unordered_map m_eventNameCounters; - std::mutex m_lock; - }; - -extern Plugin plugin; - } diff --git a/src/statistics/statistics.bif b/src/statistics/statistics.bif deleted file mode 100644 index 7f2f14c9a2..0000000000 --- a/src/statistics/statistics.bif +++ /dev/null @@ -1,29 +0,0 @@ - -%%{ -#include "zeek/statistics/Plugin.h" -%%} - -function Statistics::get_and_reset_event_name_statistics%(%): EventNameStats - %{ - auto rval = zeek::make_intrusive(zeek::id::find_type("EventNameStats")); - auto stats = zeek::plugin::statistics::plugin.GetAndResetEventStatistics(); - const auto& recordType = zeek::id::find_type("EventNameCounter"); - - auto i = 0; - for (auto& eventCounter : stats) - { - auto eventStatRecord = zeek::make_intrusive(recordType); - eventStatRecord->Assign(0, zeek::make_intrusive(eventCounter.first)); - eventStatRecord->Assign(1, zeek::val_mgr->Count(eventCounter.second)); - rval->Assign(i, std::move(eventStatRecord)); - ++i; - } - - return rval; - %} - -function Statistics::start_event_name_statistics_monitor%(%): any - %{ - zeek::plugin::statistics::plugin.StartEventNamesStatisticsMonitor(); - return nullptr; - %} \ No newline at end of file diff --git a/src/stats.bif b/src/stats.bif index d7131da482..9adc49b16c 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -485,6 +485,11 @@ function get_reporter_stats%(%): ReporterStats return r; %} +## Returns a list of event handlers that were called and the number of times +## each was called. +## +## Returns: A record with event call statistics. +## function get_event_handler_call_counts%(%): EventNameStats %{ auto rval = zeek::make_intrusive(zeek::id::find_type("EventNameStats")); diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index cda12d3d6a..429bc44f5a 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -125,7 +125,6 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/telemetry.bif.zeek build/scripts/base/bif/zeekygen.bif.zeek build/scripts/base/bif/pcap.bif.zeek - build/scripts/base/bif/statistics.bif.zeek build/scripts/base/bif/bloom-filter.bif.zeek build/scripts/base/bif/cardinality-counter.bif.zeek build/scripts/base/bif/top-k.bif.zeek diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 66c433fb47..75cc1c72d4 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -125,7 +125,6 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/telemetry.bif.zeek build/scripts/base/bif/zeekygen.bif.zeek build/scripts/base/bif/pcap.bif.zeek - build/scripts/base/bif/statistics.bif.zeek build/scripts/base/bif/bloom-filter.bif.zeek build/scripts/base/bif/cardinality-counter.bif.zeek build/scripts/base/bif/top-k.bif.zeek diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index dfa36e5ae6..f6098e74b3 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -935,7 +935,6 @@ 0.000000 MetaHookPost LoadFile(0, ./site, <...>/site.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./smb1-main, <...>/smb1-main.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./smb2-main, <...>/smb2-main.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, ./statistics.bif.zeek, <...>/statistics.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./std-dev, <...>/std-dev.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./store, <...>/store.zeek) -> -1 @@ -1321,7 +1320,6 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./site, <...>/site.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./smb1-main, <...>/smb1-main.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./smb2-main, <...>/smb2-main.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, ./statistics.bif.zeek, <...>/statistics.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./std-dev, <...>/std-dev.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./store, <...>/store.zeek) -> (-1, ) @@ -2453,7 +2451,6 @@ 0.000000 MetaHookPre LoadFile(0, ./site, <...>/site.zeek) 0.000000 MetaHookPre LoadFile(0, ./smb1-main, <...>/smb1-main.zeek) 0.000000 MetaHookPre LoadFile(0, ./smb2-main, <...>/smb2-main.zeek) -0.000000 MetaHookPre LoadFile(0, ./statistics.bif.zeek, <...>/statistics.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./stats.bif.zeek, <...>/stats.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./std-dev, <...>/std-dev.zeek) 0.000000 MetaHookPre LoadFile(0, ./store, <...>/store.zeek) @@ -2839,7 +2836,6 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./site, <...>/site.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./smb1-main, <...>/smb1-main.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./smb2-main, <...>/smb2-main.zeek) -0.000000 MetaHookPre LoadFileExtended(0, ./statistics.bif.zeek, <...>/statistics.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./stats.bif.zeek, <...>/stats.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./std-dev, <...>/std-dev.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./store, <...>/store.zeek) @@ -3981,7 +3977,6 @@ 0.000000 | HookLoadFile ./site <...>/site.zeek 0.000000 | HookLoadFile ./smb1-main <...>/smb1-main.zeek 0.000000 | HookLoadFile ./smb2-main <...>/smb2-main.zeek -0.000000 | HookLoadFile ./statistics.bif.zeek <...>/statistics.bif.zeek 0.000000 | HookLoadFile ./stats.bif.zeek <...>/stats.bif.zeek 0.000000 | HookLoadFile ./std-dev <...>/std-dev.zeek 0.000000 | HookLoadFile ./store <...>/store.zeek @@ -4367,7 +4362,6 @@ 0.000000 | HookLoadFileExtended ./site <...>/site.zeek 0.000000 | HookLoadFileExtended ./smb1-main <...>/smb1-main.zeek 0.000000 | HookLoadFileExtended ./smb2-main <...>/smb2-main.zeek -0.000000 | HookLoadFileExtended ./statistics.bif.zeek <...>/statistics.bif.zeek 0.000000 | HookLoadFileExtended ./stats.bif.zeek <...>/stats.bif.zeek 0.000000 | HookLoadFileExtended ./std-dev <...>/std-dev.zeek 0.000000 | HookLoadFileExtended ./store <...>/store.zeek diff --git a/testing/btest/Baseline/plugins.plugin-load-dependency/output b/testing/btest/Baseline/plugins.plugin-load-dependency/output index 5d6ae02103..e788232bd8 100644 --- a/testing/btest/Baseline/plugins.plugin-load-dependency/output +++ b/testing/btest/Baseline/plugins.plugin-load-dependency/output @@ -9,7 +9,6 @@ Testing::Plugin2 - Plugin2 provides a load dependency for Plugin1 and Plugin3 (d in Plugin1 in Plugin2 -Statistics - Statistics module (built-in) Testing::Plugin1 - Plugin1 has a load dependency on Plugin2 (dynamic, version 1.0.0) Testing::Plugin2 - Plugin2 provides a load dependency for Plugin1 and Plugin3 (dynamic, version 1.0.0) Testing::Plugin3 - Plugin3 has a load dependency on Plugin2 (dynamic, version 1.0.0) From a26e98f170d2eb536c8e59278cc290da012e4a43 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 10 Nov 2022 12:52:56 -0700 Subject: [PATCH 69/71] Fix compiler warning from applied patch --- src/File.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/File.cc b/src/File.cc index e9767deb29..44c78ccd91 100644 --- a/src/File.cc +++ b/src/File.cc @@ -204,14 +204,16 @@ void File::SetBuf(bool arg_buffered) #ifndef _MSC_VER if ( setvbuf(f, NULL, arg_buffered ? _IOFBF : _IOLBF, 0) != 0 ) + reporter->Error("setvbuf failed"); #else // TODO: this turns off buffering altogether because Windows wants us to pass a valid // buffer and length if we're going to pass one of the other modes. We need to // investigate the performance ramifications of this. if ( setvbuf(f, NULL, _IONBF, 0) != 0 ) -#endif reporter->Error("setvbuf failed"); + reporter->Error("setvbuf failed"); +#endif - buffered = arg_buffered; + buffered = arg_buffered; } bool File::Close() From 5996520cc7dc307a380252cd59f30de320392257 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 11 Nov 2022 08:22:08 -0700 Subject: [PATCH 70/71] Mask ports before inserting them into the map --- src/Val.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Val.cc b/src/Val.cc index 33c2cb2b0a..f706902e31 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -3972,10 +3972,11 @@ const PortValPtr& ValManager::Port(uint32_t port_num, TransportProto port_type) port_num = 0; } - if ( ports.find(port_num) == ports.end() ) - ports[port_num] = IntrusivePtr{AdoptRef{}, new PortVal(PortVal::Mask(port_num, port_type))}; + uint32_t port_masked = PortVal::Mask(port_num, port_type); + if ( ports.find(port_masked) == ports.end() ) + ports[port_masked] = IntrusivePtr{AdoptRef{}, new PortVal(PortVal::Mask(port_num, port_type))}; - return ports[port_num]; + return ports[port_masked]; } const PortValPtr& ValManager::Port(uint32_t port_num) From 642d44009aa967cd227406fbe613953f7793fcd4 Mon Sep 17 00:00:00 2001 From: Tomer Lev Date: Fri, 11 Nov 2022 18:54:05 +0200 Subject: [PATCH 71/71] Clang formatting --- src/Val.cc | 3 ++- src/packet_analysis/protocol/ip/IP.cc | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Val.cc b/src/Val.cc index f706902e31..48f30348ee 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -3974,7 +3974,8 @@ const PortValPtr& ValManager::Port(uint32_t port_num, TransportProto port_type) uint32_t port_masked = PortVal::Mask(port_num, port_type); if ( ports.find(port_masked) == ports.end() ) - ports[port_masked] = IntrusivePtr{AdoptRef{}, new PortVal(PortVal::Mask(port_num, port_type))}; + ports[port_masked] = IntrusivePtr{AdoptRef{}, + new PortVal(PortVal::Mask(port_num, port_type))}; return ports[port_masked]; } diff --git a/src/packet_analysis/protocol/ip/IP.cc b/src/packet_analysis/protocol/ip/IP.cc index d06d14a7a5..ec2c6dcdc0 100644 --- a/src/packet_analysis/protocol/ip/IP.cc +++ b/src/packet_analysis/protocol/ip/IP.cc @@ -63,7 +63,7 @@ bool IPAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* packet) } ip_hdr = std::make_shared((const struct ip6_hdr*)data, false, - static_cast(len)); + static_cast(len)); packet->l3_proto = L3_IPV6; } else