diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index cee9ed9970..d22d370914 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -34,15 +34,6 @@ jobs: with: submodules: "recursive" - # Only reset the submodule pointer for scheduled builds. The reason to do - # this is to pick up any merge commits or anything that may have been - # missed in a merge, but not have any actual content. We don't want to do - # it otherwise because PRs should just use the submodule they're pointing - # at. - - name: Switch doc submodule to master - if: github.event_name == 'schedule' - run: cd doc && git checkout master - - name: Fetch Dependencies run: | sudo apt-get update @@ -119,9 +110,6 @@ jobs: cd doc - echo "*** Running pre-commit ***" - pre-commit run -a --show-diff-on-failure --color=always - echo "*** Generating Sphinx Docs ***" make > make.out 2>&1 make_status=$? @@ -132,7 +120,7 @@ jobs: grep -q WARNING make.out && exit 1 rm make.out - - name: Push zeek-docs Changes + - name: Push docs Changes if: github.event_name == 'schedule' run: | cd doc @@ -142,16 +130,6 @@ jobs: # with a check that detects whether there's anything staged. git diff-index --cached --quiet HEAD || { git commit -m "Generate docs" && git push; } - - name: Update zeek-docs Submodule - if: github.event_name == 'schedule' - run: | - git config --global user.name zeek-bot - git config --global user.email info@zeek.org - git add doc - git status - # Similar logic here: proceed only if there's a change in the submodule. - git diff-index --cached --quiet HEAD || { git commit -m 'Update doc submodule [nomail] [skip ci]' && git push; } - - name: Send email # Only send notifications for scheduled runs. Runs from pull requests # show failures in the GitHub UI. diff --git a/.gitignore b/.gitignore index 27d0bc390b..d1586f6fc8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ build* !ci/windows/build.cmd +# Don't ignore things in the docs directory +!doc/** + tmp *.gcov diff --git a/.gitmodules b/.gitmodules index b9cfdcb91b..01d43a6468 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,9 +16,6 @@ [submodule "auxil/netcontrol-connectors"] path = auxil/netcontrol-connectors url = https://github.com/zeek/zeek-netcontrol -[submodule "doc"] - path = doc - url = https://github.com/zeek/zeek-docs [submodule "auxil/paraglob"] path = auxil/paraglob url = https://github.com/zeek/paraglob diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a557470b29..d77a7008be 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: language: python files: '\.(h|c|cpp|cc|spicy|evt)$' types: [file] - exclude: '^(testing/btest/(Baseline|plugins|spicy|scripts)/.*|testing/builtin-plugins/.*|src/3rdparty/.*)$' + exclude: '^(testing/btest/(Baseline|plugins|spicy|scripts)/.*|testing/builtin-plugins/.*|src/3rdparty/.*|doc/.*)$' - id: btest-command-commented name: Check that all BTest command lines are commented out @@ -56,4 +56,4 @@ repos: rev: v0.26.0 hooks: - id: spicy-format - exclude: '^testing/.*' + exclude: '^(testing/.*|doc/devel/spicy/autogen/.*)' diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000000..fde3c16ef8 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,16 @@ +version: 2 + +formats: + - htmlzip + +build: + os: ubuntu-24.04 + tools: + python: "3.13" + +python: + install: + - requirements: doc/requirements.txt + +sphinx: + configuration: doc/conf.py diff --git a/doc b/doc deleted file mode 160000 index 2731def915..0000000000 --- a/doc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2731def9159247e6da8a3191783c89683363689c diff --git a/doc/.gitignore b/doc/.gitignore new file mode 100644 index 0000000000..6f97ca1afc --- /dev/null +++ b/doc/.gitignore @@ -0,0 +1,2 @@ +build +*.pyc diff --git a/doc/.typos.toml b/doc/.typos.toml new file mode 100644 index 0000000000..f54d93bcad --- /dev/null +++ b/doc/.typos.toml @@ -0,0 +1,66 @@ +[default] +extend-ignore-re = [ + # seh too close to she + "registered SEH to support IDL", + # ALLO is a valid FTP command + "\"ALLO\".*[0-9]{3}", + "des-ede3-cbc-Env-OID", + # On purpose + "\"THE NETBIOS NAM\"", + # NFS stuff. + "commited :zeek:type:`NFS3::stable_how_t`", + "\\/fo\\(o", + " nd\\./dev/null; done + @ + @echo Checking whether docs for Spicy integration are up-to-date + @./devel/spicy/autogen-spicy-docs spicy-tftp + @ + @git diff --quiet devel/spicy/autogen/ \ + || (echo "Spicy docs are not up-to-date, rerun './devel/spicy/autogen-spicy-docs'." && exit 1) + +.PHONY : all doc builddir clean html livehtml diff --git a/doc/README b/doc/README new file mode 100644 index 0000000000..8204c11a4e --- /dev/null +++ b/doc/README @@ -0,0 +1,132 @@ +.. _zeek-docs: https://github.com/zeek/zeek-docs +.. _Read the Docs: https://docs.readthedocs.io/en/stable/index.html +.. _Zeek repo: https://github.com/zeek/zeek +.. _Sphinx: https://www.sphinx-doc.org/en/master +.. _pip: https://pypi.org/project/pip + +Zeek Documentation +================== + +The documentation repo at zeek-docs_ +contains version-specific Zeek documentation source files that are ultimately +used as the basis for content hosted at https://docs.zeek.org. + +Markup Format, Style, and Conventions +------------------------------------- + +For general guidance on the basics of how the documentation is written, +consult this Zeek wiki: + +https://github.com/zeek/zeek/wiki/Documentation-Style-and-Conventions + +Source-Tree Organization +------------------------ + +The zeek-docs_ repo containing this README file is the root of a Sphinx_ source +tree and can be modified to add more documentation, style sheets, JavaScript, +etc. The Sphinx config file is ``conf.py``. The typical way new documents get +integrated is from them being referenced directly in ``index.rst`` or +indirectly from something in the ``toctree`` (Table of Contents Tree) specified +in that main index. + +There is also a custom Sphinx domain implemented in ``ext/zeek.py`` which adds +some reStructureText (reST) directives and roles that aid in generating useful +index entries and cross-references. This primarily supports integration with +the script-reference sections, some of which are auto-generated by Zeek's +Doxygen-like feature, named "Zeekygen". The bulk of auto-generated content +lives under the ``scripts/`` directory or has a file name starting with +"autogenerated", so if you find yourself wanting to change those, you should +actually look at at doing those changes within the `Zeek repo`_ itself rather +than here, so see the next section for how Zeekygen docs can be (re)generated. + +Generating Zeekygen Reference Docs +---------------------------------- + +All Zeekygen-generated docs get committed into Git, so if you don't have to +perform any changes on it and just want to preview what's already existing, +you can skip down to the next :ref:`Local Previewing ` section. + +The Zeekygen documentation-generation feature is a part of Zeek itself, so +you'll want to obtain the `Zeek repo`_ from Git, read the :doc:`INSTALL +` file directions to install required dependencies, and build Zeek:: + + git clone --recursive https://github.com/zeek/zeek + cd zeek + # Read INSTALL file and get dependencies here + ./configure && make -j $(nproc) + # Make desired edits to scripts/, src/, etc. + ./ci/update-zeekygen-docs.sh + +The last command runs a script to generate documentation, which will end up in +the ``doc/`` subdirectory. Note that ``doc/`` is just a Git submodule of this +this zeek-docs_ repository, so you can run ``git status`` there to find exactly +what changed. + +Also note that the documentation-generation script is run automatically +on a daily basis to incorporate up any documentation changes that people make +in Zeek itself without them having to necessarily be aware of the full +documentation process. The GitHub Action that does that daily task is +located in the Zeek repo's ``.github/workflows/generate-docs.yml`` file. + +.. _local-doc-preview: + +Local Previewing (How To Build) +------------------------------- + +First make sure you have the required dependencies used for building docs: + +* Python interpreter >= 3.9 +* Sphinx: https://www.sphinx-doc.org/en/master/ +* Read the Docs Sphinx Theme: https://github.com/rtfd/sphinx_rtd_theme +* GitPython: https://github.com/gitpython-developers/GitPython + +If you have pip_, you may just use the command ``pip3 install -r +requirements.txt`` to install all the dependencies using the +``requirements.txt`` from zeek-docs_. + +Now run ``make`` within the zeek-docs_ repository's top-level to locally render +its reST files into HTML. After the build completes, HTML documentation is +symlinked in ``build/html`` and you can open the ``index.html`` found there in +your web browser. + +There's also a ``make livehtml`` (requires ``pip3 install sphinx-autobuild``) +target in the top-level Makefile that is useful for editing the reST files and +seeing changes rendered out live to a separate browser. + +Hosting +------- + +Documentation is hosted by `Read the Docs`_ (RTD), so you can generally read +about how it works there. The web-interface is accessible via +https://readthedocs.org/projects/zeek-docs. + +How zeek-docs_ is configured to use RTD is a combination of some custom +settings in its ``.readthedocs.yml`` file and others only accessible through +RTD's web-interface (e.g. domain and subproject settings). Most config +settings are likely understandable just by browsing the web-interface and +RTD's guides, but a few particular points to mention: + +* There is an associated, always-failing project at + https://readthedocs.org/projects/zeek. It's always-failing because + RTD redirects only activate when pages 404 and this project exists so that + all attempts to use https://zeek.rtfd.io or https://zeek.readthedocs.io + get redirected to https://docs.zeek.org. Those would have been the project + URLs if ownership of the RTD 'zeek' project was had from the start, but + it was only obtained later, after documentation already started development + in the 'zeek-docs' RTD project slug. + +* Over time, page redirects have accrued into ``redirects.yml`` as a way to + help document what they are and why they happened and also as a potential + way to automate addition/reinstantiation of a large number of redirects, + but typically redirects can be manually added via the RTD web interface + first and then noted in ``redirects.yml`` + +* There are RTD subprojects for things like Broker, Package Manager, + and Spicy. The use of subprojects simply allows access to their RTD + docs via the custom domain of https://docs.zeek.org + +* RTD will auto-build any newly-pushed commits to zeek-docs_ (i.e. a webhook is + configured), but if a tag is changed to point somewhere different, you'll + typically have to go into the RTD web interface, "Edit" the associated + version under "Versions", "wipe" the existing docs, and then manually trigger + a rebuild of that version tag under "Builds". diff --git a/doc/README.rst b/doc/README.rst new file mode 100644 index 0000000000..8204c11a4e --- /dev/null +++ b/doc/README.rst @@ -0,0 +1,132 @@ +.. _zeek-docs: https://github.com/zeek/zeek-docs +.. _Read the Docs: https://docs.readthedocs.io/en/stable/index.html +.. _Zeek repo: https://github.com/zeek/zeek +.. _Sphinx: https://www.sphinx-doc.org/en/master +.. _pip: https://pypi.org/project/pip + +Zeek Documentation +================== + +The documentation repo at zeek-docs_ +contains version-specific Zeek documentation source files that are ultimately +used as the basis for content hosted at https://docs.zeek.org. + +Markup Format, Style, and Conventions +------------------------------------- + +For general guidance on the basics of how the documentation is written, +consult this Zeek wiki: + +https://github.com/zeek/zeek/wiki/Documentation-Style-and-Conventions + +Source-Tree Organization +------------------------ + +The zeek-docs_ repo containing this README file is the root of a Sphinx_ source +tree and can be modified to add more documentation, style sheets, JavaScript, +etc. The Sphinx config file is ``conf.py``. The typical way new documents get +integrated is from them being referenced directly in ``index.rst`` or +indirectly from something in the ``toctree`` (Table of Contents Tree) specified +in that main index. + +There is also a custom Sphinx domain implemented in ``ext/zeek.py`` which adds +some reStructureText (reST) directives and roles that aid in generating useful +index entries and cross-references. This primarily supports integration with +the script-reference sections, some of which are auto-generated by Zeek's +Doxygen-like feature, named "Zeekygen". The bulk of auto-generated content +lives under the ``scripts/`` directory or has a file name starting with +"autogenerated", so if you find yourself wanting to change those, you should +actually look at at doing those changes within the `Zeek repo`_ itself rather +than here, so see the next section for how Zeekygen docs can be (re)generated. + +Generating Zeekygen Reference Docs +---------------------------------- + +All Zeekygen-generated docs get committed into Git, so if you don't have to +perform any changes on it and just want to preview what's already existing, +you can skip down to the next :ref:`Local Previewing ` section. + +The Zeekygen documentation-generation feature is a part of Zeek itself, so +you'll want to obtain the `Zeek repo`_ from Git, read the :doc:`INSTALL +` file directions to install required dependencies, and build Zeek:: + + git clone --recursive https://github.com/zeek/zeek + cd zeek + # Read INSTALL file and get dependencies here + ./configure && make -j $(nproc) + # Make desired edits to scripts/, src/, etc. + ./ci/update-zeekygen-docs.sh + +The last command runs a script to generate documentation, which will end up in +the ``doc/`` subdirectory. Note that ``doc/`` is just a Git submodule of this +this zeek-docs_ repository, so you can run ``git status`` there to find exactly +what changed. + +Also note that the documentation-generation script is run automatically +on a daily basis to incorporate up any documentation changes that people make +in Zeek itself without them having to necessarily be aware of the full +documentation process. The GitHub Action that does that daily task is +located in the Zeek repo's ``.github/workflows/generate-docs.yml`` file. + +.. _local-doc-preview: + +Local Previewing (How To Build) +------------------------------- + +First make sure you have the required dependencies used for building docs: + +* Python interpreter >= 3.9 +* Sphinx: https://www.sphinx-doc.org/en/master/ +* Read the Docs Sphinx Theme: https://github.com/rtfd/sphinx_rtd_theme +* GitPython: https://github.com/gitpython-developers/GitPython + +If you have pip_, you may just use the command ``pip3 install -r +requirements.txt`` to install all the dependencies using the +``requirements.txt`` from zeek-docs_. + +Now run ``make`` within the zeek-docs_ repository's top-level to locally render +its reST files into HTML. After the build completes, HTML documentation is +symlinked in ``build/html`` and you can open the ``index.html`` found there in +your web browser. + +There's also a ``make livehtml`` (requires ``pip3 install sphinx-autobuild``) +target in the top-level Makefile that is useful for editing the reST files and +seeing changes rendered out live to a separate browser. + +Hosting +------- + +Documentation is hosted by `Read the Docs`_ (RTD), so you can generally read +about how it works there. The web-interface is accessible via +https://readthedocs.org/projects/zeek-docs. + +How zeek-docs_ is configured to use RTD is a combination of some custom +settings in its ``.readthedocs.yml`` file and others only accessible through +RTD's web-interface (e.g. domain and subproject settings). Most config +settings are likely understandable just by browsing the web-interface and +RTD's guides, but a few particular points to mention: + +* There is an associated, always-failing project at + https://readthedocs.org/projects/zeek. It's always-failing because + RTD redirects only activate when pages 404 and this project exists so that + all attempts to use https://zeek.rtfd.io or https://zeek.readthedocs.io + get redirected to https://docs.zeek.org. Those would have been the project + URLs if ownership of the RTD 'zeek' project was had from the start, but + it was only obtained later, after documentation already started development + in the 'zeek-docs' RTD project slug. + +* Over time, page redirects have accrued into ``redirects.yml`` as a way to + help document what they are and why they happened and also as a potential + way to automate addition/reinstantiation of a large number of redirects, + but typically redirects can be manually added via the RTD web interface + first and then noted in ``redirects.yml`` + +* There are RTD subprojects for things like Broker, Package Manager, + and Spicy. The use of subprojects simply allows access to their RTD + docs via the custom domain of https://docs.zeek.org + +* RTD will auto-build any newly-pushed commits to zeek-docs_ (i.e. a webhook is + configured), but if a tag is changed to point somewhere different, you'll + typically have to go into the RTD web interface, "Edit" the associated + version under "Versions", "wipe" the existing docs, and then manually trigger + a rebuild of that version tag under "Builds". diff --git a/doc/_static/theme_overrides.css b/doc/_static/theme_overrides.css new file mode 100644 index 0000000000..cf1fa824f8 --- /dev/null +++ b/doc/_static/theme_overrides.css @@ -0,0 +1,32 @@ +/* override table width restrictions */ +@media screen and (min-width: 767px) { + + .wy-table-responsive table td { + /* !important prevents the common CSS stylesheets from overriding + this as on RTD they are loaded after this stylesheet */ + white-space: normal !important; + } + + .wy-table-responsive { + overflow: visible !important; + } +} + +h1, h2, h3, h4, h5, h6 { + color: #294488; + font-family: 'Open Sans',Helvetica,Arial,Lucida,sans-serif!important; +} + +a { + color: #2ea3f2; +} + +body { + font-family: "Open Sans",Arial,sans-serif; + color: #666; +} + +div.highlight pre strong { + font-weight: 800; + background-color: #ffffcc; +} diff --git a/doc/_templates/breadcrumbs.html b/doc/_templates/breadcrumbs.html new file mode 100644 index 0000000000..8a4aa54ae4 --- /dev/null +++ b/doc/_templates/breadcrumbs.html @@ -0,0 +1,15 @@ +{% extends "!breadcrumbs.html" %} + +{% block breadcrumbs_aside %} +
  • +{% if pagename != "search" %} + {% if display_github %} + {% if github_version == "master" %} + {{ _('Edit on GitHub') }} + {% endif %} + {% elif show_source and has_source and sourcename %} + {{ _('View page source') }} + {% endif %} +{% endif %} +
    • +{% endblock %} diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html new file mode 100644 index 0000000000..3a5449e99a --- /dev/null +++ b/doc/_templates/layout.html @@ -0,0 +1,14 @@ +{% extends "!layout.html" %} + +{% if READTHEDOCS and current_version %} + {% if current_version == "latest" or current_version == "stable" + or current_version == "master" or current_version == "current" + or current_version == "lts" or current_version == "LTS" %} + {% set current_version = current_version ~ " (" ~ version ~ ")" %} + {% endif %} +{% endif %} + +{% block menu %} + {{ super() }} + Index +{% endblock %} diff --git a/doc/about.rst b/doc/about.rst new file mode 100644 index 0000000000..85b3ddd5a8 --- /dev/null +++ b/doc/about.rst @@ -0,0 +1,256 @@ +========== +About Zeek +========== + +What Is Zeek? +============= + +Zeek is a passive, open-source network traffic analyzer. Many operators use +Zeek as a network security monitor (NSM) to support investigations of +suspicious or malicious activity. Zeek also supports a wide range of traffic +analysis tasks beyond the security domain, including performance measurement +and troubleshooting. + +The first benefit a new user derives from Zeek is the extensive set of logs +describing network activity. These logs include not only a comprehensive record +of every connection seen on the wire, but also application-layer transcripts. +These include all HTTP sessions with their requested URIs, key headers, MIME +types, and server responses; DNS requests with replies; SSL certificates; key +content of SMTP sessions; and much more. By default, Zeek writes all this +information into well-structured tab-separated or JSON log files suitable for +post-processing with external software. Users can also choose to have external +databases or SIEM products consume, store, process, and present the data for +querying. + +In addition to the logs, Zeek comes with built-in functionality for a range of +analysis and detection tasks, including extracting files from HTTP sessions, +detecting malware by interfacing to external registries, reporting vulnerable +versions of software seen on the network, identifying popular web applications, +detecting SSH brute-forcing, validating SSL certificate chains, and much more. + +In addition to shipping such powerful functionality “out of the box,” Zeek is a +fully customizable and extensible platform for traffic analysis. Zeek provides +users a domain-specific, Turing-complete scripting language for expressing +arbitrary analysis tasks. Think of the Zeek language as a “domain-specific +Python” (or Perl): just like Python, the system comes with a large set of +pre-built functionality (the “standard library”), yet users can also put Zeek +to use in novel ways by writing custom code. Indeed, all of Zeek’s default +analyses, including logging, are done via scripts; no specific analysis is +hard-coded into the core of the system. + +Zeek runs on commodity hardware and hence provides a low-cost alternative to +expensive proprietary solutions. In many ways Zeek exceeds the capabilities of +other network monitoring tools, which typically remain limited to a small set +of hard-coded analysis tasks. Zeek is not a classic signature-based intrusion +detection system (IDS); while it supports such standard functionality as well, +Zeek’s scripting language facilitates a much broader spectrum of very different +approaches to finding malicious activity. These include semantic misuse +detection, anomaly detection, and behavioral analysis. + +A large variety of sites deploy Zeek to protect their infrastructure, including +many universities, research labs, supercomputing centers, open-science +communities, major corporations, and government agencies. Zeek specifically +targets high-speed, high-volume network monitoring, and an increasing number of +sites are now using the system to monitor their 10GE networks, with some +already moving on to 100GE links. + +Zeek accommodates high-performance settings by supporting scalable +load-balancing. Large sites typically run “Zeek Clusters” in which a high-speed +front end load balancer distributes the traffic across an appropriate number of +back end PCs, all running dedicated Zeek instances on their individual traffic +slices. A central manager system coordinates the process, synchronizing state +across the back ends and providing the operators with a central management +interface for configuration and access to aggregated logs. Zeek’s integrated +management framework, ZeekControl, supports such cluster setups out-of-the-box. + +Zeek’s cluster features support single-system and multi-system setups. That's +part of Zeek’s scalability advantages. For example, administrators can scale +Zeek within one system for as long as possible, and then transparently add more +systems when necessary. + +In brief, Zeek is optimized for interpreting network traffic and generating +logs based on that traffic. It is not optimized for byte matching, and users +seeking signature detection approaches would be better served by trying +intrusion detection systems such as Suricata. Zeek is also not a protocol +analyzer in the sense of Wireshark, seeking to depict every element of network +traffic at the frame level, or a system for storing traffic in packet capture +(PCAP) form. Rather, Zeek sits at the “happy medium” representing compact yet +high fidelity network logs, generating better understanding of network traffic +and usage. + +Why Zeek? +========= + +Zeek offers many advantages for security and network teams who want to better +understand how their infrastructure is being used. + +Security teams generally depend upon four sorts of data sources when trying to +detect and respond to suspicious and malicious activity. These include *third +party* sources such as law enforcement, peers, and commercial or nonprofit +threat intelligence organizations; *network data*; *infrastructure and +application data*, including logs from cloud environments; and *endpoint data*. +Zeek is primarily a platform for collecting and analyzing the second form of +data -- network data. All four are important elements of any security team’s +program, however. + +When looking at data derived from the network, there are four types of data +available to analysts. As defined by the `network security monitoring paradigm +`_, these +four data types are *full content*, *transaction data*, *extracted content*, +and *alert data*. Using these data types, one can record traffic, summarize +traffic, extract traffic (or perhaps more accurately, extract content +in the form of files), and judge traffic, respectively. + +It’s critical to collect and analyze the four types of network security +monitoring data. The question becomes one of determining the best way to +accomplish this goal. Thankfully, Zeek as a NSM platform enables collection of +at least two, and in some ways three, of these data forms, namely transaction +data, extracted content, and alert data. + +Zeek is best known for its transaction data. By default, when run and told to +watch a network interface, Zeek will generate a collection of compact, +high-fidelity, richly-annotated set of transaction logs. These logs describe +the protocols and activity seen on the wire, in a judgement-free, +policy-neutral manner. This documentation will spend a considerable amount of +time describing the most common Zeek log files such that readers will become +comfortable with the format and learn to apply them to their environment. + +Zeek can also easily carve files from network traffic, thanks to its file +extraction capabilities. Analysts can then send those files to execution +sandboxes or other file examination tools for additional investigation. Zeek +has some capability to perform classical byte-centric intrusion detection, but +that job is best suited for packages like the open source Snort or Suricata +engines. Zeek has other capabilities however that are capable of providing +judgements in the form of alerts, through its notice mechanism. + +Zeek is not optimized for writing traffic to disk in the spirit of a full +content data collection, and that task is best handled by software written to +fulfill that requirement. + +Beyond the forms of network data that Zeek can natively collect and generate, +Zeek has advantages that appeared in the `What Is Zeek?`_ section. These +include its built-in functionality for a range of analysis and detection +tasks, and its status as a fully customizable and extensible platform for +traffic analysis. Zeek is also attractive because of its ability to run on +commodity hardware, giving users of all types the ability to at least try Zeek +in a low-cost manner. + +History +======= + +Zeek has a rich history stretching back to the 1990s. `Vern Paxson +`_ designed and implemented the initial version in +1995 as a researcher at the `Lawrence Berkeley National Laboratory (LBNL) +`_. The original software was called “Bro,” as an +“Orwellian reminder that monitoring comes hand in hand with the potential +for privacy violations”. + +LBNL first deployed Zeek in 1996, and the USENIX Security Symposium published +Vern’s original paper on Zeek in 1998, and awarded it the Best Paper Award that +year He published a refined version of the paper in 1999 as `Bro: A System for +Detecting Network Intruders in Real-Time +`_. + +In 2003, the `National Science Foundation (NSF) `_ began +supporting research and advanced development on Bro at the `International +Computer Science Institute (ICSI) `_. (Vern +still leads the ICSI `Networking and Security group `_.) + +Over the years, a growing team of ICSI researchers and students kept adding +novel functions to Zeek, while LBNL continued its support with funding from the +`Department of Energy (DOE) `_. Much of Zeek’s +capabilities originate in academic research projects, with results often +published at top-tier conferences. A key to Zeek’s success was the project’s +ability to bridge the gap between academia and operations. This relationship +helped ground research on Zeek in real-world challenges. + +With a growing operational user community, the research-centric development +model eventually became a bottleneck to the system’s evolution. Research +grants did not support the more mundane parts of software development and +maintenance. However, those elements were crucial for the end-user experience. +As a result, deploying Zeek required overcoming a steep learning curve. + +In 2010, NSF sought to address this challenge by awarding ICSI a grant from its +Software Development for Cyberinfrastructure fund. The `National Center for +Supercomputing Applications (NCSA) `_ joined the +team as a core partner, and the Zeek project began to overhaul many of the +user-visible parts of the system for the 2.0 release in 2012. + +After Zeek 2.0, the project enjoyed tremendous growth in new deployments across +a diverse range of settings, and the ongoing collaboration between ICSI (co-PI +Robin Sommer) and NCSA (co-PI Adam Slagell) brought a number of important +features. In 2012, Zeek added native IPv6 support, long before many enterprise +networking monitoring tools. In 2013, NSF renewed its support with a second +grant that established the Bro Center of Expertise at ICSI and NCSA, promoting +Zeek as a comprehensive, low-cost security capability for research and +education communities. To facilitate both debugging and education, +`try.zeek.org `_ (formerly try.bro.org) was launched in +2014. This provided an interactive way for users to test a script with their +own packet captures against a variety of Zeek versions and easily share +sample code with others. For Zeek clusters and external communication, +the Broker communication framework was added. Last, but not least, the +Zeek package manager was created in 2016, funded by an additional grant +from the Mozilla Foundation. + +In the fall of 2018, the project leadership team decided to change the name of +the software from Bro to Zeek. The leadership team desired a name that better +reflected the values of the community while avoiding the negative connotations +of so-called “bro culture” outside the computing world. The project released +version 3.0 in the fall of 2019, the first release bearing the name Zeek. The +year 2020 saw a renewed focus on community and growing the Zeek community, with +increased interaction via social media, webinars, Slack channels, and related +outreach efforts. + +For a history of the project from 1995 to 2015, see Vern Paxson’s talk from +BroCon 2015, `Reflecting on Twenty Years of Bro +`_. + +For background on the decision to rename Bro to Zeek, see Vern Paxson’s talk +from BroCon 2018, `Renaming Bro +`_. + +Architecture +============ + +.. image:: /images/architecture.png + :align: center + :scale: 75% + +At a very high level, Zeek is architecturally layered into two major +components. Its *event engine* (or *core*) reduces the incoming packet stream +into a series of higher-level *events*. These events reflect network activity +in policy-neutral terms, i.e., they describe *what* has been seen, but not +*why*, or whether it is significant. + +For example, every HTTP request on the wire turns into a corresponding +:zeek:see:`http_request` event that carries with it the involved IP addresses +and ports, the URI being requested, and the HTTP version in use. The event +however does not convey any further *interpretation*, such as whether that URI +corresponds to a known malware site. + +The event engine component comprises a number of subcomponents, including in +particular the packet processing pipeline consisting of: input sources, +packet analysis, session analysis, and file analysis. Input sources ingest +incoming network traffic from network interfaces. Packet analysis processes +lower-level protocols, starting all the way down at the link layer. Session +analysis handles application-layer protocols, such as HTTP, FTP, etc. File +analysis dissects the content of files transferred over sessions. The event +engine provides a plugin architecture for adding any of these from outside +of the core Zeek code base, allowing to expand Zeek’s capabilities as +needed. + +Semantics related to the events are derived by Zeek’s second main component, +the *script interpreter*, which executes a set of *event handlers* written in +Zeek’s custom scripting language. These scripts can express a site’s +security policy, such as what actions to take when the monitor detects +different types of activity. + +More generally scripts can derive any desired properties and statistics from +the input traffic. In fact, all of Zeek’s default output comes from scripts +included in the distribution. Zeek’s language comes with extensive +domain-specific types and support functionality. Crucially, Zeek’s language +allows scripts to maintain state over time, enabling them to track and +correlate the evolution of what they observe across connection and host +boundaries. Zeek scripts can generate real-time alerts and also execute +arbitrary external programs on demand. One might use this functionality to +trigger an active response to an attack. diff --git a/doc/acknowledgements.rst b/doc/acknowledgements.rst new file mode 100644 index 0000000000..7a0747e638 --- /dev/null +++ b/doc/acknowledgements.rst @@ -0,0 +1,22 @@ +================ +Acknowledgements +================ + +Thanks to everyone who contributed in making Zeek's documentation +(alphabetically): + +* Johanna Amann +* Richard Bejtlich +* Michael Dopheide +* Amber Graner +* Jan Grashöfer +* Christian Kreibich +* Terry Leach +* Aashish Sharma +* Jon Siwek +* Stephen Smoot +* Robin Sommer +* Aaron Soto +* Nick Turley +* Fatema Bannat Wala +* Tim Wojtulewicz diff --git a/doc/building-from-source.rst b/doc/building-from-source.rst new file mode 100644 index 0000000000..21f940406f --- /dev/null +++ b/doc/building-from-source.rst @@ -0,0 +1,392 @@ + +.. _CMake: https://www.cmake.org +.. _SWIG: https://www.swig.org +.. _Xcode: https://developer.apple.com/xcode/ +.. _MacPorts: https://www.macports.org +.. _Fink: https://www.finkproject.org +.. _Homebrew: https://brew.sh +.. _downloads page: https://zeek.org/get-zeek +.. _devtoolset: https://developers.redhat.com/products/developertoolset/hello-world +.. _zkg package manager: https://docs.zeek.org/projects/package-manager/en/stable/ +.. _crosstool-NG: https://crosstool-ng.github.io/ +.. _CMake toolchain: https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html +.. _contribute: https://github.com/zeek/zeek/wiki/Contribution-Guide +.. _Chocolatey: https://chocolatey.org +.. _Npcap: https://npcap.com/ + +.. _building-from-source: + +==================== +Building from Source +==================== + +Building Zeek from source provides the most control over your build and is the +preferred approach for advanced users. We support a wide range of operating +systems and distributions. Our `support policy +`_ is informed by +what we can run in our CI pipelines with reasonable effort, with the current +status captured in our `support matrix +`_. + +Required Dependencies +--------------------- + +Building Zeek from source requires the following dependencies, including +development headers for libraries: + + * Bash (for ZeekControl and BTest) + * BIND8 library or greater (if not covered by system's libresolv) + * Bison 3.3 or greater (https://www.gnu.org/software/bison/) + * C/C++ compiler with C++17 support (GCC 8+ or Clang 9+) + * CMake 3.15 or greater (https://www.cmake.org) + * Flex (lexical analyzer generator) 2.6 or greater (https://github.com/westes/flex) + * Libpcap (https://www.tcpdump.org) + * Make + * OpenSSL (https://www.openssl.org) + * Python 3.9 or greater (https://www.python.org/) + * SWIG (https://www.swig.org) + * ZeroMQ (https://zeromq.org) + * Zlib (https://zlib.net/) + +To install these, you can use: + +* RPM/RedHat-based Linux: + + .. code-block:: console + + sudo dnf install bison cmake cppzmq-devel gcc gcc-c++ flex libpcap-devel make openssl-devel python3 python3-devel swig zlib-devel + + On pre-``dnf`` systems, use ``yum`` instead. Additionally, on RHEL/CentOS 7, + you can install and activate a devtoolset_ to get access to recent GCC + versions. You will also have to install and activate CMake 3. For example: + + .. code-block:: console + + sudo yum install cmake3 devtoolset-7 + scl enable devtoolset-7 bash + +* DEB/Debian-based Linux: + + .. code-block:: console + + sudo apt-get install bison cmake cppzmq-dev gcc g++ flex libfl-dev libpcap-dev libssl-dev make python3 python3-dev swig zlib1g-dev + + If your platform doesn't offer ``cppzmq-dev``, try ``libzmq3-dev`` + instead. Zeek's build will fall back to an in-tree version of C++ + bindings to ZeroMQ in that case. + +* FreeBSD: + + Most required dependencies should come with a minimal FreeBSD install + except for the following. + + .. code-block:: console + + sudo pkg install -y base64 bash bison cmake cppzmq git python3 swig + pyver=`python3 -c 'import sys; print(f"py{sys.version_info[0]}{sys.version_info[1]}")'` + sudo pkg install -y $pyver-sqlite3 + +* macOS: + + Compiling source code on Macs requires first installing either Xcode_ + or the "Command Line Tools" (which is a much smaller download). To check + if either is installed, run the ``xcode-select -p`` command. If you see + an error message, then neither is installed and you can then run + ``xcode-select --install`` which will prompt you to either get Xcode (by + clicking "Get Xcode") or to install the command line tools (by + clicking "Install"). + + macOS comes with all required dependencies except for CMake_, SWIG_, + Bison, Flex, and OpenSSL (OpenSSL headers were removed in macOS 10.11, + therefore OpenSSL must be installed manually for macOS versions 10.11 + or newer). + + Distributions of these dependencies can likely be obtained from your + preferred macOS package management system (e.g. Homebrew_, + MacPorts_, or Fink_). Specifically for Homebrew, the ``bison``, ``cmake``, + ``cppzmq``, ``flex``, ``swig``, and ``openssl`` packages + provide the required dependencies. For MacPorts, use the ``bison``, ``cmake``, + ``cppzmq``, ``flex``, ``swig``, ``swig-python``, and ``openssl`` packages. + +* Windows + + Windows support is experimental. These instructions are meant as a starting + point for development on that platform, and might have issues or be missing + steps. Notify the Zeek team if any such problems arise. + + Compiling on Windows requires the installation of a development environment. + Zeek currently builds on Visual Studio 2019, and you can either install the + full version including the UI tools or you can install the command-line tools + and build from a shell. The instructions below describe how to install the + command-line tools, but are not necessary if you install the full VS2019 + package. You will need to install Chocolatey_ in order to install the + dependencies as instructed below. It's possible to install them from other + sources (msys2, cygwin, etc), which we leave to the reader. + + Cloning the repository will also require Developer Mode to be enabled in + Windows. This is due to the existence of a number of symbolic links in the + repository. Without Developer Mode, ``git`` on Windows will ignore these + links and builds will fail. There are a couple of different ways to enable + it, and the settings may differ depending on the version of Windows. + + .. code-block:: console + + choco install -y --no-progress visualstudio2019buildtools --version=16.11.11.0 + choco install -y --no-progress visualstudio2019-workload-vctools --version=1.0.0 --package-parameters '--add Microsoft.VisualStudio.Component.VC.ATLMFC' + choco install -y --no-progress sed + choco install -y --no-progress winflexbison3 + choco install -y --no-progress msysgit + choco install -y --no-progress python + choco install -y --no-progress openssl --version=3.1.1 + + Once the dependencies are installed, you will need to add the Git installation + to your PATH (``C:\Program Files\Git\bin`` by default). This is needed for the + ``sh`` command to be available during the build. Once all of the dependencies + are in place, you will need to open a shell (PowerShell or cmd) and add the + development environment to it. The following command is for running on an + x86_64 host. + + .. code-block:: console + + C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvarsall.bat x86_amd64 + + Now you can build via cmake: + + .. code-block:: console + + mkdir build + cd build + cmake.exe .. -DCMAKE_BUILD_TYPE=release -DENABLE_ZEEK_UNIT_TESTS=yes -DENABLE_CLUSTER_BACKEND_ZEROMQ=no -DVCPKG_TARGET_TRIPLET="x64-windows-static" -G Ninja + cmake.exe --build . + + All of this is duplicated in the CI configuration for Windows which lives in + the ``ci/windows`` directory, and can be used as a reference for running the + commands by hand. + + Note: By default, Windows links against the standard libpcap library from + vcpkg. This version of libpcap does not support packet capture on Windows, + unlike other platforms. In order to capture packets from live interfaces on + Windows, you will need to link against the Npcap_ library. This library is free + for personal use, but requires a paid license for commercial use or + redistribution. To link against Npcap, download the SDK from their website, + unzip it, and then pass ``-DPCAP_ROOT_DIR=""`` to the + initial CMake invocation for Zeek. + + Note also that the ZeroMQ cluster backend is not yet supported on Windows. + +Optional Dependencies +--------------------- + +Zeek can make use of some optional libraries and tools if they are found at +build time: + + * libmaxminddb (for geolocating IP addresses) + * sendmail (enables Zeek and ZeekControl to send mail) + * curl (used by a Zeek script that implements active HTTP) + * gperftools (tcmalloc is used to improve memory and CPU usage) + * jemalloc (https://github.com/jemalloc/jemalloc) + * PF_RING (Linux only, see :ref:`pf-ring-config`) + * krb5 libraries and headers + * ipsumdump (for trace-summary; https://github.com/kohler/ipsumdump) + * hiredis (for the Redis storage backend) + +Geolocation is probably the most interesting and can be installed on most +platforms by following the instructions for :ref:`address geolocation and AS +lookups `. + +The `zkg package manager`_, included in the Zeek installation, requires +two external Python modules: + + * GitPython: https://pypi.org/project/GitPython/ + * semantic-version: https://pypi.org/project/semantic-version/ + +These install easily via pip (``pip3 install GitPython +semantic-version``) and also ship with some distributions: + +* RPM/RedHat-based Linux: + + .. code-block:: console + + sudo yum install python3-GitPython python3-semantic_version + +* DEB/Debian-based Linux: + + .. code-block:: console + + sudo apt-get install python3-git python3-semantic-version + +``zkg`` also requires a ``git`` installation, which the above system packages +pull in as a dependency. If you install via pip, remember that you also need +``git`` itself. + +Retrieving the Sources +---------------------- + +Zeek releases are bundled into source packages for convenience and are +available on the `downloads page`_. The source code can be manually downloaded +from the link in the ``.tar.gz`` format to the target system for installation. + +If you plan to `contribute`_ to Zeek or just want to try out the latest +features under development, you should obtain Zeek's source code through its +Git repositories hosted at https://github.com/zeek: + +.. code-block:: console + + git clone --recurse-submodules https://github.com/zeek/zeek + +.. note:: If you choose to clone the ``zeek`` repository + non-recursively for a "minimal Zeek experience", be aware that + compiling it depends on several of the other submodules as well, so + you'll likely have to build/install those independently first. + +Configuring and Building +------------------------ + +The typical way to build and install from source is as follows: + +.. code-block:: console + + ./configure + make + make install + +If the ``configure`` script fails, then it is most likely because it either +couldn't find a required dependency or it couldn't find a sufficiently new +version of a dependency. Assuming that you already installed all required +dependencies, then you may need to use one of the ``--with-*`` options +that can be given to the ``configure`` script to help it locate a dependency. +To find out what all different options ``./configure`` supports, run +``./configure --help``. + +The default installation path is ``/usr/local/zeek``, which would typically +require root privileges when doing the ``make install``. A different +installation path can be chosen by specifying the ``configure`` script +``--prefix`` option. Note that ``/usr``, ``/opt/bro/``, and ``/opt/zeek`` are +the standard prefixes for binary Zeek packages to be installed, so those are +typically not good choices unless you are creating such a package. + +OpenBSD users, please see our `FAQ `_ if you are having +problems installing Zeek. + +Depending on the Zeek package you downloaded, there may be auxiliary +tools and libraries available in the ``auxil/`` directory. Some of them +will be automatically built and installed along with Zeek. There are +``--disable-*`` options that can be given to the configure script to +turn off unwanted auxiliary projects that would otherwise be installed +automatically. Finally, use ``make install-aux`` to install some of +the other programs that are in the ``auxil/zeek-aux`` directory. + +Finally, if you want to build the Zeek documentation (not required, because +all of the documentation for the latest Zeek release is available at +https://docs.zeek.org), there are instructions in ``doc/README`` in the source +distribution. + +Cross Compiling +--------------- + +Prerequisites +~~~~~~~~~~~~~ + +You need three things on the host system: + +1. The Zeek source tree. +2. A cross-compilation toolchain, such as one built via crosstool-NG_. +3. Pre-built Zeek dependencies from the target system. This usually + includes libpcap, zlib, OpenSSL, and Python development headers + and libraries. + +Configuration and Compiling +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You first need to compile a few build tools native to the host system +for use during the later cross-compile build. In the root of your +Zeek source tree: + +.. code-block:: console + + ./configure --builddir=../zeek-buildtools + ( cd ../zeek-buildtools && make binpac bifcl ) + +Next configure Zeek to use your cross-compilation toolchain (this example +uses a Raspberry Pi as the target system): + +.. code-block:: console + + ./configure --toolchain=/home/jon/x-tools/RaspberryPi-toolchain.cmake --with-binpac=$(pwd)/../zeek-buildtools/auxil/binpac/src/binpac --with-bifcl=$(pwd)/../zeek-buildtools/src/bifcl + +Here, the :file:`RaspberryPi-toolchain.cmake` file specifies a `CMake +toolchain`_. In the toolchain file, you need to point the toolchain and +compiler at the cross-compilation toolchain. It might look something the +following: + +.. code-block:: cmake + + # Operating System on which CMake is targeting. + set(CMAKE_SYSTEM_NAME Linux) + + # The CMAKE_STAGING_PREFIX option may not work. + # Given that Zeek is configured: + # + # ``./configure --prefix=`` + # + # The options are: + # + # (1) ``make install`` and then copy over the --prefix dir from host to + # target system. + # + # (2) ``DESTDIR= make install`` and then copy over the + # contents of that staging directory. + + set(toolchain /home/jon/x-tools/arm-rpi-linux-gnueabihf) + set(CMAKE_C_COMPILER ${toolchain}/bin/arm-rpi-linux-gnueabihf-gcc) + set(CMAKE_CXX_COMPILER ${toolchain}/bin/arm-rpi-linux-gnueabihf-g++) + + # The cross-compiler/linker will use these paths to locate dependencies. + set(CMAKE_FIND_ROOT_PATH + /home/jon/x-tools/zeek-rpi-deps + ${toolchain}/arm-rpi-linux-gnueabihf/sysroot + ) + + set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) + set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) + set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) + +If that configuration succeeds you are ready to build: + +.. code-block:: console + + make + +And if that works, install on your host system: + +.. code-block:: console + + make install + +Once installed, you can copy/move the files from the installation prefix on the +host system to the target system and start running Zeek as usual. + +Configuring the Run-Time Environment +==================================== + +You may want to adjust your :envvar:`PATH` environment variable +according to the platform/shell/package you're using since +neither :file:`/usr/local/zeek/bin/` nor :file:`/opt/zeek/bin/` +will reside in the default :envvar:`PATH`. For example: + +Bourne-Shell Syntax: + +.. code-block:: console + + export PATH=/usr/local/zeek/bin:$PATH + +C-Shell Syntax: + +.. code-block:: console + + setenv PATH /usr/local/zeek/bin:$PATH + +Or substitute ``/opt/zeek/bin`` instead if you installed from a binary package. + +Zeek supports several environment variables to adjust its behavior. Take a look +at the ``zeek --help`` output for details. diff --git a/doc/cluster-setup.rst b/doc/cluster-setup.rst new file mode 100644 index 0000000000..0bb5cc0790 --- /dev/null +++ b/doc/cluster-setup.rst @@ -0,0 +1,507 @@ + +.. _ZeekControl documentation: https://github.com/zeek/zeekctl + +================== +Zeek Cluster Setup +================== + +.. TODO: integrate BoZ revisions + +A *Zeek Cluster* is a set of systems jointly analyzing the traffic of +a network link in a coordinated fashion. You can operate such a setup from +a central manager system easily using ZeekControl because it +hides much of the complexity of the multi-machine installation. + +Cluster Architecture +==================== + +Zeek is not multithreaded, so once the limitations of a single processor core +are reached the only option currently is to spread the workload across many +cores, or even many physical computers. The cluster deployment scenario for +Zeek is the current solution to build these larger systems. The tools and +scripts that accompany Zeek provide the structure to easily manage many Zeek +processes examining packets and doing correlation activities but acting as +a singular, cohesive entity. This section describes the Zeek cluster +architecture. For information on how to configure a Zeek cluster, +see the documentation for `ZeekControl `_. + +Architecture +------------ + +The figure below illustrates the main components of a Zeek cluster. + +.. image:: /images/deployment.png + +For more specific information on the way Zeek processes are connected, +how they function, and how they communicate with each other, see the +:ref:`Broker Framework Documentation `. + +Tap +*** +The tap is a mechanism that splits the packet stream in order to make a copy +available for inspection. Examples include the monitoring port on a switch +and an optical splitter on fiber networks. + +Frontend +******** +The frontend is a discrete hardware device or on-host technique that splits +traffic into many streams or flows. The Zeek binary does not do this job. +There are numerous ways to accomplish this task, some of which are described +below in `Frontend Options`_. + +Manager +******* +The manager is a Zeek process that has two primary jobs. It receives log +messages and notices from the rest of the nodes in the cluster using the Zeek +communications protocol (note that if you use a separate logger node, then the +logger receives all logs instead of the manager). The result +is a single log instead of many discrete logs that you have to +combine in some manner with post-processing. +The manager also supports other functionality and analysis which +requires a centralized, global view of events or data. + +Logger +****** +A logger is an optional Zeek process that receives log messages from the +rest of the nodes in the cluster using the Zeek communications protocol. +The purpose of having a logger receive logs instead of the manager is +to reduce the load on the manager. If no logger is needed, then the +manager will receive logs instead. + +Proxy +***** +A proxy is a Zeek process that may be used to offload data storage or +any arbitrary workload. A cluster may contain multiple proxy nodes. +The default scripts that come with Zeek make minimal use of proxies, so +a single one may be sufficient, but customized use of them to partition +data or workloads provides greater cluster scalability potential than +just doing similar tasks on a single, centralized Manager node. + +Zeek processes acting as proxies don't tend to be extremely hard on CPU +or memory and users frequently run proxy processes on the same physical +host as the manager. + +Worker +****** +The worker is the Zeek process that sniffs network traffic and does protocol +analysis on the reassembled traffic streams. Most of the work of an active +cluster takes place on the workers and as such, the workers typically +represent the bulk of the Zeek processes that are running in a cluster. +The fastest memory and CPU core speed you can afford is recommended +since all of the protocol parsing and most analysis will take place here. +There are no particular requirements for the disks in workers since almost all +logging is done remotely to the manager, and normally very little is written +to disk. + +Frontend Options +---------------- + +There are many options for setting up a frontend flow distributor. In many +cases it is beneficial to do multiple stages of flow distribution +on the network and on the host. + +Discrete hardware flow balancers +******************************** + +cPacket +^^^^^^^ + +If you are monitoring one or more 10G physical interfaces, the recommended +solution is to use either a cFlow or cVu device from cPacket because they +are used successfully at a number of sites. These devices will perform +layer-2 load balancing by rewriting the destination Ethernet MAC address +to cause each packet associated with a particular flow to have the same +destination MAC. The packets can then be passed directly to a monitoring +host where each worker has a BPF filter to limit its visibility to only that +stream of flows, or onward to a commodity switch to split the traffic out to +multiple 1G interfaces for the workers. This greatly reduces +costs since workers can use relatively inexpensive 1G interfaces. + +On host flow balancing +********************** + +PF_RING +^^^^^^^ + +The PF_RING software for Linux has a "clustering" feature which will do +flow-based load balancing across a number of processes that are sniffing the +same interface. This allows you to easily take advantage of multiple +cores in a single physical host because Zeek's main event loop is single +threaded and can't natively utilize all of the cores. If you want to use +PF_RING, see the documentation on :ref:`how to configure Zeek with PF_RING +`. + + +AF_PACKET +^^^^^^^^^ + +On Linux, Zeek supports `AF_PACKET sockets `_ natively. +Currently, this is provided by including the `external Zeek::AF_Packet plugin `_ +in default builds of Zeek for Linux. Additional information can be found in +the project's README file. + +To check the availability of the ``af_packet`` packet source, print its information using ``zeek -N``:: + + zeek -N Zeek::AF_Packet + Zeek::AF_Packet - Packet acquisition via AF_Packet (dynamic, version 3.2.0) + +On FreeBSD, MacOSX, or if Zeek was built with ``--disable-af-packet``, the +plugin won't be available. + +Single worker mode +"""""""""""""""""" + +For the most basic usage, prefix the interface with ``af_packet::`` when invoking Zeek:: + + zeek -i af_packet::eth0 + +Generally, running Zeek this way requires a privileged user with CAP_NET_RAW +and CAP_NET_ADMIN capabilities. Linux supports file-based capabilities: A +process executing an executable with capabilities will receive these. +Using this mechanism allows to run Zeek as an unprivileged user once the file +capabilities have been added:: + + sudo setcap cap_net_raw,cap_net_admin=+eip /path/to/zeek + +Offloading and ethtool tuning +""""""""""""""""""""""""""""" + +While not specific to AF_PACKET, it is recommended to disable any offloading +features provided by the network card or Linux networking stack when running +Zeek. This allows to see network packets as they arrive on the wire. +See this `blog post `_ +for more background + +Toggling these features can be done with the ``ethtool -K`` command, for example:: + + IFACE=eth0 + for offload in rx tx sg tso ufo gso gro lro; do + ethtool -K $IFACE $offload off + done + +Detailed statistics about the interface can be gathered via ``ethtool -S``. + +For more details around the involved offloads consult the +`ethtool manpage `_. + +Load balancing +"""""""""""""" + +The more interesting use-case is to use AF_PACKET to run multiple Zeek workers +and have their packet sockets join what is called a fanout group. +In such a setup, the network traffic is load-balanced across Zeek workers. +By default load balancing is based on symmetric flow hashes [#]_. + +For example, running two Zeek workers listening on the same network interface, +each worker analyzing approximately half of the network traffic, can be done +as follows:: + + zeek -i af_packet::eth0 & + zeek -i af_packet::eth0 & + +The fanout group is identified by an id and configurable using the +``AF_Packet::fanout_id`` constant which defaults to 23. In the example +above, both Zeek workers join the same fanout group. + + +.. note:: + + As a caveat, within the same Linux network namespace, two Zeek processes can + not use the same fanout group id for listening on different network interfaces. + If this is a setup you're planning on running, configure the fanout group + ids explicitly. + For illustration purposes, the following starts two Zeek workers each using + a different network interface and fanout group id:: + + zeek -i af_packet::eth0 AF_Packet::fanout_id=23 & + zeek -i af_packet::eth1 AF_Packet::fanout_id=24 & + +.. warning:: + + Zeek workers crashing or restarting due to running out of memory can, + for a short period of time, disturb load balancing due to their packet + sockets being removed and later rejoining the fanout group. + This may be visible in Zeek logs as gaps and/or duplicated connection + entries produced by different Zeek workers. + +See :ref:`cluster-configuration` for instructions how to configure AF_PACKET +with ZeekControl. + + +Netmap +^^^^^^ + +`Netmap `_ is a framework for fast +packet I/O that is natively supported on FreeBSD since version 10. +On Linux it can be installed as an out-of-tree kernel module. + +FreeBSD +""""""" +FreeBSD's libpcap library supports netmap natively. This allows to prefix +interface names with ``netmap:`` to instruct libpcap to open the interface +in netmap mode. For example, a single Zeek worker can leverage netmap +transparently using Zeek's default packet source as follows:: + + zeek -i netmap:em0 + +.. warning:: + + Above command will put the em0 interface into kernel-bypass mode. Network + packets will pass directly to Zeek without being interpreted by the kernel. + If em0 is your primary network interface, this effectively disables + networking, including SSH connectivity. + +If your network card supports multiple rings, individual Zeek workers can be +attached to these as well (this assumes the NIC does proper flow hashing in hardware):: + + zeek -i netmap:em0-0 + zeek -i netmap:em0-1 + +For software load balancing support, the FreeBSD source tree includes the +``lb`` tool to distribute packets into netmap pipes doing flow hashing +in user-space. + +To compile and install ``lb``, ensure ``/usr/src`` is available on your +FreeBSD system, then run the following commands:: + + cd /usr/src/tools/tools/netmap/ + make + # Installs lb into /usr/local/bin + cp /usr/obj/usr/src/`uname -m`.`uname -m`/tools/tools/netmap/lb /usr/local/bin/ + + +To load-balance packets arriving on em0 into 4 different netmap pipes named +``zeek}0`` through ``zeek}3``, run ``lb`` as follows:: + + lb -i em0 -p zeek:4 + 410.154166 main [634] interface is em0 + 411.377220 main [741] successfully opened netmap:em0 + 411.377243 main [812] opening pipe named netmap:zeek{0/xT@1 + 411.379200 main [829] successfully opened pipe #1 netmap:zeek{0/xT@1 (tx slots: 1024) + 411.379242 main [838] zerocopy enabled + ... + +Now, Zeek workers can attach to these four netmap pipes. When starting Zeek +workers manually, the respective invocations would be as follows. The ``/x`` +suffix specifies exclusive mode to prevent two Zeek processes consuming packets +from the same netmap pipe:: + + zeek -i netmap:zeek}0/x + zeek -i netmap:zeek}1/x + zeek -i netmap:zeek}2/x + zeek -i netmap:zeek}3/x + +For packet-level debugging, you can attach ``tcpdump`` to any of the netmap +pipes in read monitor mode even while Zeek workers are consuming from them:: + + tcpdump -i netmap:zeek}1/r + +In case libpcap's netmap support is insufficient, the external +`Zeek netmap plugin `_ can be installed. + +.. warning:: + + When using the zeek-netmap plugin on FreeBSD, the interface specification given to Zeek + needs to change from ``netmap:zeek}0/x`` to ``netmap::zeek}0/x`` - a single colon more. + In the first case, Zeek uses the default libpcap packet source and passes ``netmap:zeek}0`` + as interface name. In the second case, ``netmap::`` is interpreted by Zeek and + the netmap packet source is instantiated. The ``zeek}0/x`` part is used as + interface name. + +Linux +""""" + +While netmap isn't included in the Linux kernel, it can be installed as +an out-of-tree kernel module. +See the project's `GitHub repository `_ +for detailed instructions. This includes the ``lb`` tool for load balancing. + +On Linux, the external `zeek-netmap `_ +packet source plugin is required, or the system's libpcap library as used by +Zeek needs to be recompiled with native netmap support. With the netmap kernel +module loaded and the Zeek plugin installed, running a Zeek worker as follows +will leverage netmap on Linux:: + + zeek -i netmap::eth1 + +For using ``lb`` or libpcap with netmap support, refer to the commands shown +in the FreeBSD section - these are essentially the same. + + +.. _cluster-configuration: + +Cluster Configuration +===================== + +A *Zeek Cluster* is a set of systems jointly analyzing the traffic of +a network link in a coordinated fashion. You can operate such a setup from +a central manager system easily using ZeekControl because it +hides much of the complexity of the multi-machine installation. + +This section gives examples of how to setup common cluster configurations +using ZeekControl. For a full reference on ZeekControl, see the +`ZeekControl documentation`_. + +Preparing to Setup a Cluster +---------------------------- + +We refer to the user account used to set up the cluster +as the "Zeek user". When setting up a cluster the Zeek user must be set up +on all hosts, and this user must have ssh access from the manager to all +machines in the cluster, and it must work without being prompted for a +password/passphrase (for example, using ssh public key authentication). +Also, on the worker nodes this user must have access to the target +network interface in promiscuous mode. + +Additional storage must be available on all hosts under the same path, +which we will call the cluster's prefix path. We refer to this directory +as ````. If you build Zeek from source, then ```` is +the directory specified with the ``--prefix`` configure option, +or ``/usr/local/zeek`` by default. The Zeek user must be able to either +create this directory or, where it already exists, must have write +permission inside this directory on all hosts. + +When trying to decide how to configure the Zeek nodes, keep in mind that +there can be multiple Zeek instances running on the same host. For example, +it's possible to run a proxy and the manager on the same host. However, it is +recommended to run workers on a different machine than the manager because +workers can consume a lot of CPU resources. The maximum recommended +number of workers to run on a machine should be one or two less than +the number of CPU cores available on that machine. Using a load-balancing +method (such as PF_RING) along with CPU pinning can decrease the load on +the worker machines. Also, in order to reduce the load on the manager +process, it is recommended to have a logger in your configuration. If a +logger is defined in your cluster configuration, then it will receive logs +instead of the manager process. + +Basic Cluster Configuration +--------------------------- + +With all prerequisites in place, perform the following steps to setup +a Zeek cluster (do this as the Zeek user on the manager host only): + +- Edit the ZeekControl configuration file, ``/etc/zeekctl.cfg``, + and change the value of any options to be more suitable for + your environment. You will most likely want to change the value of + the ``MailTo`` and ``LogRotationInterval`` options. A complete + reference of all ZeekControl options can be found in the + `ZeekControl documentation`_. + +- Edit the ZeekControl node configuration file, ``/etc/node.cfg`` + to define where logger, manager, proxies, and workers are to run. For a + cluster configuration, you must comment-out (or remove) the standalone node + in that file, and either uncomment or add node entries for each node + in your cluster (logger, manager, proxy, and workers). For example, if you + wanted to run five Zeek nodes (two workers, one proxy, a logger, and a + manager) on a cluster consisting of three machines, your cluster + configuration would look like this:: + + [logger] + type=logger + host=10.0.0.10 + + [manager] + type=manager + host=10.0.0.10 + + [proxy-1] + type=proxy + host=10.0.0.10 + + [worker-1] + type=worker + host=10.0.0.11 + interface=eth0 + + [worker-2] + type=worker + host=10.0.0.12 + interface=eth0 + + For a complete reference of all options that are allowed in the ``node.cfg`` + file, see the `ZeekControl documentation`_. + +- Edit the network configuration file ``/etc/networks.cfg``. This + file lists all of the networks which the cluster should consider as local + to the monitored environment. + +- Install Zeek on all machines in the cluster using ZeekControl:: + + > zeekctl install + +- See the `ZeekControl documentation`_ + for information on setting up a cron job on the manager host that can + monitor the cluster. + +AF_PACKET Cluster Configuration +------------------------------- + +Since version 5.2, Zeek includes AF_PACKET as a native packet source. This +provides an easy and efficient capture mechanism for Linux users. + +Adapt the worker section in ZeekControl's ``node.cfg`` file with the +following entries, assuming running four worker processes listening on ``eth0`` :: + + [worker-1] + type=worker + host=10.0.0.11 + interface=eth0 + lb_method=af_packet + lb_procs=4 + +The specific options are ``lb_method=af_packet`` and ``lb_procs=4``. +If listening on two or more interfaces on the same host is a requirement, +remember to set a unique ``fanout_id`` using the node option ``af_packet_fanout_id``:: + + [worker-1-eth0] + type=worker + host=10.0.0.11 + interface=eth0 + lb_method=af_packet + lb_procs=4 + af_packet_fanout_id=20 + + [worker-1-eth1] + type=worker + host=10.0.0.11 + interface=eth1 + lb_method=af_packet + lb_procs=4 + af_packet_fanout_id=21 + +Pinning the worker processes to individual CPU cores can improve performance. +Use the node's option ``pin_cpus=4,5,6,7``, listing as many CPU numbers as +processes at appropriate offsets. + +.. _pf-ring-config: + +PF_RING Cluster Configuration +----------------------------- + +`PF_RING `_ allows speeding up the +packet capture process by installing a new type of socket in Linux systems. +It supports 10Gbit hardware packet filtering using standard network adapters, +and user-space DNA (Direct NIC Access) for fast packet capture/transmission. + +.. note:: + + Unless you have evaluated to specifically require PF_RING, consider using + AF_PACKET first and test if it fulfills your requirements. AF_PACKET has + been integrated into Zeek since version 5.2. It's a bit easier to get + started with as it does not require an out of tree Linux kernel module. + +Head over to :ref:`cluster-pf-ring` for more details. + +.. toctree:: + :hidden: + + cluster/pf_ring + + +.. [#] Some Linux kernel versions between 3.10 and 4.7 might exhibit + a bug that prevents the required symmetric hashing. The script available + in the GitHub project `can-i-use-afpacket-fanout `_ + can be used to verify whether ``PACKET_FANOUT`` works as expected. + + This issue has been fixed in all stable kernels for at least 5 years. + You're unlikely to be affected. diff --git a/doc/cluster/pf_ring.rst b/doc/cluster/pf_ring.rst new file mode 100644 index 0000000000..aa124491c2 --- /dev/null +++ b/doc/cluster/pf_ring.rst @@ -0,0 +1,141 @@ +.. _cluster-pf-ring: + +=================== +PF_RING Setup Guide +=================== + +Installing PF_RING +****************** + +1. Download and install PF_RING for your system following the instructions + `here `_. The following + commands will install the PF_RING libraries and kernel module (replace + the version number 5.6.2 in this example with the version that you + downloaded):: + + cd /usr/src + tar xvzf PF_RING-5.6.2.tar.gz + cd PF_RING-5.6.2/userland/lib + ./configure --prefix=/opt/pfring + make install + + cd ../libpcap + ./configure --prefix=/opt/pfring + make install + + cd ../tcpdump-4.1.1 + ./configure --prefix=/opt/pfring + make install + + cd ../../kernel + make + make install + + modprobe pf_ring enable_tx_capture=0 min_num_slots=32768 + + Refer to the documentation for your Linux distribution on how to load the + pf_ring module at boot time. You will need to install the PF_RING + library files and kernel module on all of the workers in your cluster. + +2. Download the Zeek source code. + +3. Configure and install Zeek using the following commands:: + + ./configure --with-pcap=/opt/pfring + make + make install + +4. Make sure Zeek is correctly linked to the PF_RING libpcap libraries:: + + ldd /usr/local/zeek/bin/zeek | grep pcap + libpcap.so.1 => /opt/pfring/lib/libpcap.so.1 (0x00007fa6d7d24000) + +5. Configure ZeekControl to use PF_RING (explained below). + +6. Run "zeekctl install" on the manager. This command will install Zeek and + required scripts to all machines in your cluster. + +Using PF_RING +************* + +In order to use PF_RING, you need to specify the correct configuration +options for your worker nodes in ZeekControl's node configuration file. +Edit the ``node.cfg`` file and specify ``lb_method=pf_ring`` for each of +your worker nodes. Next, use the ``lb_procs`` node option to specify how +many Zeek processes you'd like that worker node to run, and optionally pin +those processes to certain CPU cores with the ``pin_cpus`` option (CPU +numbering starts at zero). The correct ``pin_cpus`` setting to use is +dependent on your CPU architecture (Intel and AMD systems enumerate +processors in different ways). Using the wrong ``pin_cpus`` setting +can cause poor performance. Here is what a worker node entry should +look like when using PF_RING and CPU pinning:: + + [worker-1] + type=worker + host=10.0.0.50 + interface=eth0 + lb_method=pf_ring + lb_procs=10 + pin_cpus=2,3,4,5,6,7,8,9,10,11 + + +Using PF_RING+DNA with symmetric RSS +************************************ + +You must have a PF_RING+DNA license in order to do this. You can sniff +each packet only once. + +1. Load the DNA NIC driver (i.e. ixgbe) on each worker host. + +2. Run "ethtool -L dna0 combined 10" (this will establish 10 RSS queues + on your NIC) on each worker host. You must make sure that you set the + number of RSS queues to the same as the number you specify for the + lb_procs option in the node.cfg file. + +3. On the manager, configure your worker(s) in node.cfg:: + + [worker-1] + type=worker + host=10.0.0.50 + interface=dna0 + lb_method=pf_ring + lb_procs=10 + + +Using PF_RING+DNA with pfdnacluster_master +****************************************** + +You must have a PF_RING+DNA license and a libzero license in order to do +this. You can load balance between multiple applications and sniff the +same packets multiple times with different tools. + +1. Load the DNA NIC driver (i.e. ixgbe) on each worker host. + +2. Run "ethtool -L dna0 1" (this will establish 1 RSS queues on your NIC) + on each worker host. + +3. Run the pfdnacluster_master command on each worker host. For example:: + + pfdnacluster_master -c 21 -i dna0 -n 10 + + Make sure that your cluster ID (21 in this example) matches the interface + name you specify in the node.cfg file. Also make sure that the number + of processes you're balancing across (10 in this example) matches + the lb_procs option in the node.cfg file. + +4. If you are load balancing to other processes, you can use the + pfringfirstappinstance variable in zeekctl.cfg to set the first + application instance that Zeek should use. For example, if you are running + pfdnacluster_master with "-n 10,4" you would set + pfringfirstappinstance=4. Unfortunately that's still a global setting + in zeekctl.cfg at the moment but we may change that to something you can + set in node.cfg eventually. + +5. On the manager, configure your worker(s) in node.cfg:: + + [worker-1] + type=worker + host=10.0.0.50 + interface=dnacluster:21 + lb_method=pf_ring + lb_procs=10 diff --git a/doc/components/index.rst b/doc/components/index.rst new file mode 100644 index 0000000000..45092f3baa --- /dev/null +++ b/doc/components/index.rst @@ -0,0 +1,33 @@ + +============= +Subcomponents +============= + +To find documentation for the various subcomponents of Zeek, see their +respective GitHub repositories or documentation: + +* `Spicy `__ + - C++ parser generator for dissecting protocols & files. +* `BinPAC `__ + - A protocol parser generator +* `ZeekControl `__ + - Interactive Zeek management shell +* `Zeek-Aux `__ + - Small auxiliary tools for Zeek +* `BTest `__ + - A system testing framework +* `Capstats `__ + - Command-line packet statistic tool +* `PySubnetTree `__ + - Python module for CIDR lookups +* `trace-summary `__ + - Script for generating break-downs of network traffic +* `Broker `__ + - Zeek's Messaging Library + - `(Docs) `__ +* `Package Manager `__ + - A package manager for Zeek + - `(Docs) `__ +* `Paraglob `__ + - A pattern matching data structure for Zeek. + - `(Docs) `__ diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 0000000000..69de553ac5 --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,305 @@ +# +# Zeek documentation build configuration file, created by sphinx-quickstart +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import os +import sys + +extensions = [] + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath("ext")) + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions += [ + "zeek", + "sphinx.ext.todo", + "zeek_pygments", + "spicy-pygments", + "literal-emph", + "sphinx.ext.extlinks", +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix of source filenames. +source_suffix = ".rst" + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = "Zeek" +copyright = "by the Zeek Project" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +# + +version = "source" + +try: + # Use the actual Zeek version if available + with open("../VERSION") as f: + version = f.readline().strip() +except: + try: + import re + + import git + + repo = git.Repo(os.path.abspath(".")) + version = "git/master" + + version_tag_re = r"v\d+\.\d+(\.\d+)?" + version_tags = [ + t + for t in repo.tags + if t.commit == repo.head.commit and re.match(version_tag_re, str(t)) + ] + # Note: sorting by tag date doesn't necessarily give correct + # order in terms of version numbers, but doubtful that will ever be + # a problem (if we ever do re-tag an old version number on a given + # commit such that it is incorrectly found as the most recent version, + # we can just re-tag all the other version numbers on that same commit) + version_tags = sorted(version_tags, key=lambda t: t.tag.tagged_date) + + if version_tags: + version = str(version_tags[-1]) + + except: + pass + +# The full version, including alpha/beta/rc tags. +release = version + +# In terms of the actual hyperlink URL, a more ideal/stable way to reference +# source code on GitHub would be by commit hash, but that can be tricky to +# update in a way that produces stable Sphinx/reST configuration: don't want +# to update the commit-hash for every Zeek commit unless it actually produces +# new content, and also don't want to accidentally make it easy for people to +# insert unreachable commits when manually running +# `zeek/ci/update-zeekygen-docs.sh`. +# +# We only have a few versions of docs that actually matter: `master` and +# `release/.*`, and the tip of those branches will always be in sync with +# auto-generated content by simply having `zeek/ci/update-zeekygen-docs.sh` +# change this to `release/.*` when needed. +zeek_code_version = "master" +zeek_code_url = f"https://github.com/zeek/zeek/blob/{zeek_code_version}" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +today_fmt = "%B %d, %Y" + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [".#*", "script-reference/autogenerated-*"] + +# The reST default role (used for this markup: `text`) to use for all documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +show_authors = True + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +highlight_language = "none" + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +html_theme = "sphinx_rtd_theme" + +# Set canonical URL from the Read the Docs Domain +html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "") + +# Tell Jinja2 templates the build is running on Read the Docs +if os.environ.get("READTHEDOCS", "") == "True": + if "html_context" not in globals(): + html_context = {} + html_context["READTHEDOCS"] = True + +html_last_updated_fmt = "%B %d, %Y" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { + "analytics_id": "UA-144186885-1", + "collapse_navigation": False, + "style_external_links": True, +} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v Documentation". +html_title = f"Book of Zeek ({release})" + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +html_logo = "images/zeek-logo-sidebar.png" + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +html_favicon = "images/zeek-favicon.ico" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + + +def setup(app): + app.add_css_file("theme_overrides.css") + from sphinx.highlighting import lexers + from zeek_pygments import ZeekLexer + + lexers["zeek"] = ZeekLexer() + app.add_config_value("zeek-code-url", zeek_code_url, "env") + + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = { +#'**': ['localtoc.html', 'sourcelink.html', 'searchbox.html'], +# } + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = "zeek-docs" + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +# latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +# latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ("index", "Zeek.tex", "Zeek Documentation", "The Zeek Project", "manual"), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Additional stuff for the LaTeX preamble. +# latex_preamble = '' + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [("index", "zeek", "Zeek Documentation", ["The Zeek Project"], 1)] + +# -- Options for todo plugin -------------------------------------------- +todo_include_todos = True + +extlinks = { + "slacklink": ("https://zeek.org/slack%s", None), + "discourselink": ("https://community.zeek.org/%s", None), + "spicylink": ("https://docs.zeek.org/projects/spicy/en/latest/%s", None), +} +extlinks_detect_hardcoded_links = True diff --git a/doc/customizations.rst b/doc/customizations.rst new file mode 100644 index 0000000000..fc6b15e826 --- /dev/null +++ b/doc/customizations.rst @@ -0,0 +1,318 @@ +.. _popular-customizations: + +====================== +Popular Customizations +====================== + +This page outlines customizations and additions that are popular +among Zeek users. + +.. note:: + + This page lists externally-maintained Zeek packages. The Zeek team does not + provide support or maintenance for these packages. If you find bugs or have + feature requests, please reach out to the respective package maintainers directly. + + You may also post in the :slacklink:`Zeek Slack <>` #packages + channel or :discourselink:`forum <>` to get help from the broader + Zeek community. + + +Log Enrichment +============== + +Community ID +------------ + +.. versionadded:: 6.0 + +Zeek includes native `Community ID Flow Hashing`_ support. This functionality +has previously been provided through the `zeek-community-id`_ package. + +.. note:: + + At this point, the external `zeek-community-id`_ package is still + available to support Zeek deployments running older versions. However, + the scripts provided by the package cause conflicts with those provided in + Zeek 6.0 - do not load both. + +Loading the +:doc:`/scripts/policy/protocols/conn/community-id-logging.zeek` +and +:doc:`/scripts/policy/frameworks/notice/community-id.zeek` +scripts adds an additional ``community_id`` field to the +:zeek:see:`Conn::Info` and :zeek:see:`Notice::Info` record. + +.. code-block:: console + + $ zeek -r ./traces/get.trace protocols/conn/community-id-logging LogAscii::use_json=T + $ jq < conn.log + { + "ts": 1362692526.869344, + "uid": "CoqLmg1Ds5TE61szq1", + "id.orig_h": "141.142.228.5", + "id.orig_p": 59856, + "id.resp_h": "192.150.187.43", + "id.resp_p": 80, + "proto": "tcp", + ... + "community_id": "1:yvyB8h+3dnggTZW0UEITWCst97w=" + } + + +The Community ID Flow Hash of a :zeek:see:`conn_id` instance can be computed +with the :zeek:see:`community_id_v1` builtin function directly on the command-line +or used in custom scripts. + +.. code-block:: console + + $ zeek -e 'print community_id_v1([$orig_h=141.142.228.5, $orig_p=59856/tcp, $resp_h=192.150.187.43, $resp_p=80/tcp])' + 1:yvyB8h+3dnggTZW0UEITWCst97w= + +.. _Community ID Flow Hashing: https://github.com/corelight/community-id-spec +.. _zeek-community-id: https://github.com/corelight/zeek-community-id/>`_ + +.. _geolocation: + +Address geolocation and AS lookups +---------------------------------- + +.. _libmaxminddb: https://github.com/maxmind/libmaxminddb + +Zeek supports IP address geolocation as well as AS (autonomous system) +lookups. This requires two things: + + * Compilation of Zeek with the `libmaxminddb`_ library and development + headers. If you're using our :ref:`Docker images ` or + :ref:`binary packages `, there's nothing to do: they ship + with GeoIP support. + * Installation of corresponding MaxMind database files on your + system. + +To check whether your Zeek supports geolocation, run ``zeek-config --have-geoip`` +(available since Zeek 6.2) or simply try an address lookup. The following +indicates that your Zeek lacks support: + +.. code-block:: console + + $ zeek -e 'lookup_location(1.2.3.4)' + error in , line 1: Zeek was not configured for GeoIP support (lookup_location(1.2.3.4)) + +Read on for more details about building Zeek with GeoIP support, and how to +configure access to the database files. + +Building Zeek with libmaxminddb +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you build Zeek yourself, you need to install libmaxminddb prior to +configuring your build. + +* RPM/RedHat-based Linux: + + .. code-block:: console + + sudo yum install libmaxminddb-devel + +* DEB/Debian-based Linux: + + .. code-block:: console + + sudo apt-get install libmaxminddb-dev + +* FreeBSD: + + .. code-block:: console + + sudo pkg install libmaxminddb + +* Mac OS X: + + You need to install from your preferred package management system + (e.g. Homebrew, MacPorts, or Fink). For Homebrew, the name of the package + that you need is libmaxminddb. + +The ``configure`` script's output indicates whether it successfully located +libmaxminddb. If your system's MaxMind library resides in a non-standard path, +you may need to specify it via ``./configure --with-geoip=``. + +Installing and configuring GeoIP databases +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +MaxMind's databases ship as individual files that you can `download +`_ from their +website after `signing up `_ for an +account. Some Linux distributions also offer free databases in their package +managers. + +There are three types of databases: city-level geolocation, country-level +geolocation, and mapping of IP addresses to autonomous systems (AS number and +organization). Download these and decide on a place to put them on your +file system. If you use automated tooling or system packages for the +installation, that path may be chosen for you, such as ``/usr/share/GeoIP``. + +Zeek provides three ways to configure access to the databases: + +* Specifying the path and filenames via script variables. Use the + :zeek:see:`mmdb_dir` variable, unset by default, to point to the directory + containing the database(s). By default Zeek looks for databases called + ``GeoLite2-City.mmdb``, ``GeoLite2-Country.mmdb``, and + ``GeoLite2-ASN.mmdb``. Starting with Zeek 6.2 you can adjust these names by + redefining the :zeek:see:`mmdb_city_db`, :zeek:see:`mmdb_country_db`, and + :zeek:see:`mmdb_asn_db` variables. +* Relying on Zeek's pre-configured search paths and filenames. The + :zeek:see:`mmdb_dir_fallbacks` variable contains default + search paths that Zeek will try in turn when :zeek:see:`mmdb_dir` is not + set. Prior to Zeek 6.2 these paths were hardcoded; they're now redefinable. + For geolocation, Zeek first attempts the city-level databases due to their + greater precision, and falls back to the city-level one. You can adjust the + database filenames via :zeek:see:`mmdb_city_db` and related variables, as + covered above. +* Opening databases explicitly via scripting. The + :zeek:see:`mmdb_open_location_db` and :zeek:see:`mmdb_open_asn_db` + functions take full paths to database files. Zeek only ever uses one + geolocation and one ASN database, and these loads override any databases + previously loaded. These loads can occur at any point. + +Querying the databases +^^^^^^^^^^^^^^^^^^^^^^ + +Two built-in functions provide GeoIP functionality: + +.. code-block:: zeek + + function lookup_location(a:addr): geo_location + function lookup_autonomous_system(a:addr): geo_autonomous_system + +:zeek:see:`lookup_location` returns a :zeek:see:`geo_location` record with +country/region/etc fields, while :zeek:see:`lookup_autonomous_system` returns a +:zeek:see:`geo_autonomous_system` record indicating the AS number and +organization. Depending on the queried IP address some fields may be +uninitialized, so you should guard access with an ``a?$b`` :ref:`existence test +`. + +Zeek tests the database files for staleness. If it detects that a database has +been updated, it will automatically reload it. Zeek does not automatically add +GeoIP intelligence to its logs, but several add-on scripts and packages provide +such functionality. These include: + +* The :ref:`notice framework ` lets you configure notice types + that you'd like to augment with location information. See + :zeek:see:`Notice::lookup_location_types` and + :zeek:see:`Notice::ACTION_ADD_GEODATA` for details. +* The :doc:`/scripts/policy/protocols/smtp/detect-suspicious-orig.zeek` and + :doc:`/scripts/policy/protocols/ssh/geo-data.zeek` policy scripts. +* Several `Zeek packages `_. + +Testing +^^^^^^^ + +Before using the GeoIP functionality it is a good idea to verify that +everything is setup correctly. You can quickly check if the GeoIP +functionality works by running commands like these: + +.. code-block:: console + + zeek -e "print lookup_location(8.8.8.8);" + +If you see an error message similar to "Failed to open GeoIP location database", +then your database configuration is broken. You may need to rename or move your +GeoIP database files. + +Example +^^^^^^^ + +The following shows every FTP connection from hosts in Ohio, US: + +.. code-block:: zeek + + event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) + { + local client = c$id$orig_h; + local loc = lookup_location(client); + + if (loc?$region && loc$region == "OH" && loc?$country_code && loc$country_code == "US") + { + local city = loc?$city ? loc$city : ""; + + print fmt("FTP Connection from:%s (%s,%s,%s)", client, city, + loc$region, loc$country_code); + } + } + + +Log Writers +=========== + +Kafka +----- + +For exporting logs to `Apache Kafka`_ in a streaming fashion, the externally-maintained +`zeek-kafka`_ package is a popular choice and easy to configure. It relies on `librdkafka`_. + +.. code-block:: zeek + + redef Log::default_writer = Log::WRITER_KAFKAWRITER; + + redef Kafka::kafka_conf += { + ["metadata.broker.list"] = "192.168.0.1:9092" + }; + +.. _Apache Kafka: https://kafka.apache.org/ +.. _zeek-kafka: https://github.com/SeisoLLC/zeek-kafka/ +.. _librdkafka: https://github.com/confluentinc/librdkafka + + +Logging +======= + +JSON Streaming Logs +------------------- + +The externally-maintained `json-streaming-logs`_ package tailors Zeek +for use with log shippers like `Filebeat`_ or `fluentd`_. It configures +additional log files prefixed with ``json_streaming_``, adds ``_path`` +and ``_write_ts`` fields to log records and configures log rotation +appropriately. + +If you do not use a logging archive and want to stream all logs away +from the system where Zeek is running without leveraging Kafka, this +package helps you with that. + +.. _json-streaming-logs: https://github.com/corelight/json-streaming-logs +.. _Filebeat: https://www.elastic.co/beats/filebeat +.. _fluentd: https://www.fluentd.org/ + + +Long Connections +---------------- + +Zeek logs connection entries into the :file:`conn.log` only upon termination +or due to expiration of inactivity timeouts. Depending on the protocol and +chosen timeout values this can significantly delay the appearance of a log +entry for a given connection. The delay may be up to an hour for lingering +SSH connections or connections where the final FIN or RST packets were missed. + +The `zeek-long-connections`_ package alleviates this by creating a :file:`conn_long.log` +log with the same format as :file:`conn.log`, but containing entries for connections +that have been existing for configurable intervals. +By default, the first entry for a connection is logged after 10mins. Depending on +the environment, this can be lowered as even a 10 minute delay may be significant +for detection purposes in streaming setup. + +.. _zeek-long-connections: https://github.com/corelight/zeek-long-connections + + +Profiling and Debugging +======================= + +jemalloc profiling +------------------ + +For investigation of memory leaks or state-growth issues within Zeek, +jemalloc's profiling is invaluable. A package providing a bit support +for configuring jemalloc's profiling facilities is `zeek-jemalloc-profiling`_. + +Some general information about memory profiling exists in the :ref:`Troubleshooting ` +section. + +.. _zeek-jemalloc-profiling: https://github.com/JustinAzoff/zeek-jemalloc-profiling diff --git a/doc/devel/cluster-backend-zeromq.rst b/doc/devel/cluster-backend-zeromq.rst new file mode 100644 index 0000000000..c07c523786 --- /dev/null +++ b/doc/devel/cluster-backend-zeromq.rst @@ -0,0 +1,120 @@ +.. _cluster_backend_zeromq: + +====================== +ZeroMQ Cluster Backend +====================== + +.. versionadded:: 7.1 + +*Experimental* + +Quickstart +========== + +To switch a Zeek cluster with a static cluster layout over to use ZeroMQ +as cluster backend, add the following snippet to ``local.zeek``: + +.. code-block:: zeek + + @load frameworks/cluster/backend/zeromq/connect + + +Note that the function :zeek:see:`Broker::publish` will be non-functional +and a warning emitted when used - use :zeek:see:`Cluster::publish` instead. + +By default, a configuration based on hard-coded endpoints and cluster layout +information is created. For more customization, refer to the module documentation +at :doc:`cluster/backend/zeromq/main.zeek `. + + +Architecture +============ + +Publish-Subscribe of Zeek Events +-------------------------------- + +The `ZeroMQ `_ based cluster backend uses a central +XPUB/XSUB broker for publish-subscribe functionality. Zeek events published +via :zeek:see:`Cluster::publish` are distributed by this central broker to +interested nodes. + +.. figure:: /images/cluster/zeromq-pubsub.png + + +As depicted in the figure above, each cluster node connects to the central +broker twice, once via its XPUB socket and once via its XSUB socket. This +results in two TCP connections from every cluster node to the central broker. +This setup allows every node in the cluster to see messages from all other +nodes, avoiding the need for cluster topology awareness. + +.. note:: + + Scalability of the central broker in production setups, but for small + clusters on a single node, may be fast enough. + +On a cluster node, the XPUB socket provides notifications about subscriptions +created by other nodes: For every subscription created by any node in +the cluster, the :zeek:see:`Cluster::Backend::ZeroMQ::subscription` event is +raised locally on every other node (unless another node had created the same +subscription previously). + +This mechanism is used to discover the existence of other cluster nodes by +matching the topics with the prefix for node specific subscriptions as produced +by :zeek:see:`Cluster::nodeid_topic`. + +As of now, the implementation of the central broker calls ZeroMQ's +``zmq::proxy()`` function to forward messages between the XPUB and +XSUB socket. + +While the diagram above indicates the central broker being deployed separately +from Zeek cluster nodes, by default the manager node will start and run this +broker using a separate thread. There's nothing that would prevent from running +a long running central broker independently from the Zeek cluster nodes, however. + +The serialization of Zeek events is done by the selected +:zeek:see:`Cluster::event_serializer` and is independent of ZeroMQ. +The central broker needs no knowledge about the chosen format, it is +only shuffling messages between nodes. + + +Logging +------- + +While remote events always pass through the central broker, nodes connect and +send log writes directly to logger nodes in a cluster. The ZeroMQ cluster backend +leverages ZeroMQ's pipeline pattern for this functionality. That is, logger nodes +(including the manager if configured using :zeek:see:`Cluster::manager_is_logger`) +open a ZeroMQ PULL socket to receive log writes. All other nodes connect their +PUSH socket to all available PULL sockets. These connections are separate from +the publish-subscribe setup outlined above. + +When sending log-writes over a PUSH socket, load balancing is done by ZeroMQ. +Individual cluster nodes do not have control over the decision which logger +node receives log writes at any given time. + +.. figure:: /images/cluster/zeromq-logging.png + +While the previous paragraph used "log writes", a single message to a logger +node actually contains a batch of log writes. The options :zeek:see:`Log::flush_interval` +and :zeek:see:`Log::write_buffer_size` control the frequency and maximum size +of these batches. + +The serialization format used to encode such batches is controlled by the +selected :zeek:see:`Cluster::log_serializer` and is independent of ZeroMQ. + +With the default serializer (:zeek:see:`Cluster::LOG_SERIALIZER_ZEEK_BIN_V1`), +every log batch on the wire has a header prepended that describes it. This allows +interpretation of log writes even by non-Zeek processes. This opens the possibility +to implement non-Zeek logger processes as long as the chosen serializer format +is understood by the receiving process. In the future, a JSON lines serialization +may be provided, allowing easier interpretation than a proprietary binary format. + + +Summary +------- + +Combining the diagrams above, the connections between the different socket +types in a Zeek cluster looks something like the following. + +.. figure:: /images/cluster/zeromq-cluster.png + diff --git a/doc/devel/contributors.rst b/doc/devel/contributors.rst new file mode 100644 index 0000000000..f1ec537a16 --- /dev/null +++ b/doc/devel/contributors.rst @@ -0,0 +1,111 @@ + +=================== +Contributor's Guide +=================== + +See below for selection of some of the more common contribution guidelines +maintained directly in `Zeek wiki +`_. + +General Contribution Process +============================ + +See https://github.com/zeek/zeek/wiki/Contribution-Guide + +Coding Style and Conventions +============================ + +See https://github.com/zeek/zeek/wiki/Coding-Style-and-Conventions + +General Documentation Structure/Process +======================================= + +See the :doc:`README ` file of https://github.com/zeek/zeek-docs + +Documentation Style and Conventions +=================================== + +See https://github.com/zeek/zeek/wiki/Documentation-Style-and-Conventions + +Checking for Memory Errors and Leaks +==================================== + +See https://github.com/zeek/zeek/wiki/Checking-for-Memory-Errors-and-Leaks + +Maintaining long-lived forks of Zeek +==================================== + +Consistent formatting of the Zeek codebase is enforced automatically by +configurations tracked in the repository. Upstream updates to these +configurations can lead to formatting changes which could cause merge conflicts +for long-lived forks. + +Currently the following configuration files in the root directory are used: + +- ``.pre-commit-config.yaml``: Configuration for `pre-commit `_. + We use pre-commit to manage and orchestrate formatters and linters. +- ``.clang-format``: Configuration for `clang-format + `_ for formatting C++ files. +- ``.style.yapf``: Configuration for `YAPF `_ + for formatting Python files. +- ``.cmake-format.json``: Configuration for `cmake-format + `_ for formatting CMake files. + +With these configuration files present ``pre-commit run --all-files`` will +install all needed formatters and reformat all files in the repository +according to the current configuration. + +.. rubric:: Workflow: Zeek ``master`` branch regularly merged into fork + +If Zeek's master branch is regularly merged into the fork, merge conflicts can +be resolved once and their resolution is tracked in the repository. Similarly, +we can explicitly reformat the fork once and then merge the upstream branch. + +.. code-block:: sh + + ## Get and stage latest versions of configuration files from master. + git checkout master -- .pre-commit-config.yaml .clang-format .style.yapf .cmake-format.json + + ## Reformat fork according to new configuration. + pre-commit run -a + + ## Record reformatted state of fork. + git add -u && git commit -m 'Reformat' + + # Merge in master, resolve merge conflicts as usual. + git merge master + +.. rubric:: Workflow: Fork regularly rebased onto Zeek ``master`` branch + +If the target for a rebase has been reformatted individual diff hunks might not +apply cleanly anymore. There are different approaches to work around that. The +approach with the least conflicts is likely to first reformat the fork +according to upstream style without pulling in changes, and only after that +rebase on upstream and resolve potential semantic conflicts. + +.. code-block:: sh + + # Create a commit updating the configuration files. + git checkout master -- .pre-commit-config.yaml .clang-format .style.yapf .cmake-format.json + git commit -m 'Bump formatter configurations' + + # With a fork branched from upstream at commit FORK_COMMIT, rebase the + # config update commit 'Bump formatter configurations' to the start of the + # fork, but do not yet rebase on master (interactively move the last patch + # to the start of the list of patches). + git rebase -i FORK_COMMIT + + # Reformat all commits according to configs at the base. We use the '--exec' + # flag of 'git rebase' to execute pre-commit after applying each patch. If + # 'git rebase' detects uncommitted changes it stops automatic progress so + # one can inspect and apply the changes. + git rebase -i FORK_COMMIT --exec 'pre-commit run --all-files' + # When this stops, inspect changes and stage them. + git add -u + # Continue rebasing. This prompts for a commit message and amends the last + # patch. + git rebase --continue + + # The fork is now formatted according to upstream style. Rebase on master, + # and drop the 'Bump formatter configurations' patch from the list of patches. + git rebase -i master diff --git a/doc/devel/index.rst b/doc/devel/index.rst new file mode 100644 index 0000000000..7ffe4baf86 --- /dev/null +++ b/doc/devel/index.rst @@ -0,0 +1,21 @@ + +================ +Developer Guides +================ + +In addition to documentation found or mentioned below, some developer-oriented +content is maintained directly in the `Zeek wiki +`_ due to the nature of +the content (e.g. the author finds it to be more dynamic, informal, meta, +transient, etc. compared to other documentation). + +.. toctree:: + :maxdepth: 2 + + plugins + spicy/index + websocket-api + Documentation Guide + contributors + maintainers + cluster-backend-zeromq diff --git a/doc/devel/maintainers.rst b/doc/devel/maintainers.rst new file mode 100644 index 0000000000..0bc179bcc9 --- /dev/null +++ b/doc/devel/maintainers.rst @@ -0,0 +1,13 @@ + +================== +Maintainer's Guide +================== + +Some notable guidelines for maintainers are linked below for convenience, but +they are generally maintained directly in the `Zeek wiki +`_. + +Release Process +=============== + +See https://github.com/zeek/zeek/wiki/Release-Process diff --git a/doc/devel/plugins.rst b/doc/devel/plugins.rst new file mode 100644 index 0000000000..5381c1ef24 --- /dev/null +++ b/doc/devel/plugins.rst @@ -0,0 +1,505 @@ +.. _zkg package manager: https://docs.zeek.org/projects/package-manager/en/stable/ + +.. _writing-plugins: + +=============== +Writing Plugins +=============== + +Zeek provides a plugin API that enables extending +the system dynamically, without modifying the core code base. That way, +custom code remains self-contained and can be maintained, compiled, +and installed independently. Currently, plugins can add the following +functionality to Zeek: + + - Zeek scripts. + + - Builtin functions/events/types for the scripting language. + + - Protocol analyzers. + + - File analyzers. + + - Packet sources and packet dumpers. + + - Logging framework backends. + + - Input framework readers. + +A plugin's functionality is available to the user just as if Zeek had +the corresponding code built-in. Indeed, internally many of Zeek's +pieces are structured as plugins as well, they are just statically +compiled into the binary rather than loaded dynamically at runtime. + +.. note:: + + Plugins and Zeek packages are related but separate concepts. Both extend + Zeek's functionality without modifying Zeek's source code. A plugin achieves + this via compiled, native code that Zeek links into its core at runtime. A Zeek + package, on the other hand, is a modular addition to Zeek, managed via the + `zkg package manager`_, that may or may not include a plugin. More commonly, + packages consist of script-layer additions to Zeek's functionality. Packages + also feature more elaborate metadata, enabling dependencies on other packages, + Zeek versions, etc. + +Quick Start +=========== + +Writing a basic plugin is quite straight-forward as long as one +follows a few conventions. In the following, we create a simple example +plugin that adds a new Built-In Function (BIF) to Zeek: we'll add +``rot13(s: string) : string``, a function that rotates every letter +in a string by 13 places. + +Generally, a plugin comes in the form of a directory following a +certain structure. To get started, Zeek's distribution provides a +helper script ``auxil/zeek-aux/plugin-support/init-plugin`` that creates +a skeleton plugin that can then be customized. Let's use that:: + + # init-plugin ./rot13-plugin Demo Rot13 + +As you can see, the script takes three arguments. The first is a +directory inside which the plugin skeleton will be created. The second +is the namespace the plugin will live in, and the third is a descriptive +name for the plugin itself relative to the namespace. Zeek uses the +combination of namespace and name to identify a plugin. The namespace +serves to avoid naming conflicts between plugins written by independent +developers; pick, e.g., the name of your organisation. The namespaces +``Bro`` (legacy) and ``Zeek`` are reserved for functionality distributed +by the Zeek Project. In +our example, the plugin will be called ``Demo::Rot13``. + +The ``init-plugin`` script puts a number of files in place. The full +layout is described later. For now, all we need is +``src/rot13.bif``. It's initially empty, but we'll add our new BIF +there as follows:: + + # cat src/rot13.bif + %%{ + #include + #include + #include "zeek/util.h" + #include "zeek/ZeekString.h" + #include "zeek/Val.h" + %%} + + module Demo; + + function rot13%(s: string%) : string + %{ + char* rot13 = util::copy_string(s->CheckString()); + + for ( char* p = rot13; *p; p++ ) + { + char b = islower(*p) ? 'a' : 'A'; + char d = *p - b + 13; + + if ( d >= 13 && d <= 38 ) + *p = d % 26 + b; + } + + zeek::String* zs = new zeek::String(1, reinterpret_cast(rot13), + strlen(rot13)); + return make_intrusive(zs); + %} + +The syntax of this file is just like any other ``*.bif`` file; we +won't go into it here. + +Now we are ready to compile our plugin. The configure script will just +need to be able to find the location of either a Zeek installation-tree or +a Zeek source-tree. + +When building a plugin against a Zeek installation-tree, simply have the +installation's associated ``zeek-config`` in your :envvar:`PATH` and the +configure script will detect it and use it to obtain all the information +it needs:: + + # which zeek-config + /usr/local/zeek/bin/zeek-config + # cd rot13-plugin + # ./configure && make + [... cmake output ...] + +When building a plugin against a Zeek source-tree (which itself needs +to have first been built), the configure script has to explicitly be +told its location:: + + # cd rot13-plugin + # ./configure --zeek-dist=/path/to/zeek/dist && make + [... cmake output ...] + +This builds the plugin in a subdirectory ``build/``. In fact, that +subdirectory *becomes* the plugin: when ``make`` finishes, ``build/`` +has everything it needs for Zeek to recognize it as a dynamic plugin. + +Let's try that. Once we point Zeek to the ``build/`` directory, it will +pull in our new plugin automatically, as we can check with the ``-N`` +option:: + + # export ZEEK_PLUGIN_PATH=/path/to/rot13-plugin/build + # zeek -N + [...] + Demo::Rot13 - (dynamic, version 0.1.0) + [...] + +That looks quite good, except for the dummy description that we should +replace with something nicer so that users will know what our plugin +is about. We do this by editing the ``config.description`` line in +``src/Plugin.cc``, like this:: + + [...] + plugin::Configuration Plugin::Configure() + { + plugin::Configuration config; + config.name = "Demo::Rot13"; + config.description = "Caesar cipher rotating a string's letters by 13 places."; + config.version.major = 0; + config.version.minor = 1; + config.version.patch = 0; + return config; + } + [...] + +Now rebuild and verify that the description is visible:: + + # make + [...] + # zeek -N | grep Rot13 + Demo::Rot13 - Caesar cipher rotating a string's letters by 13 places. (dynamic, version 0.1.0) + +Zeek can also show us what exactly the plugin provides with the +more verbose option ``-NN``:: + + # zeek -NN + [...] + Demo::Rot13 - Caesar cipher rotating a string's letters by 13 places. (dynamic, version 0.1.0) + [Function] Demo::rot13 + [...] + +There's our function. Now let's use it:: + + # zeek -e 'print Demo::rot13("Hello")' + Uryyb + +It works. We next install the plugin along with Zeek itself, so that it +will find it directly without needing the ``ZEEK_PLUGIN_PATH`` +environment variable. If we first unset the variable, the function +will no longer be available:: + + # unset ZEEK_PLUGIN_PATH + # zeek -e 'print Demo::rot13("Hello")' + error in , line 1: unknown identifier Demo::rot13, at or near "Demo::rot13" + +Once we install it, it works again:: + + # make install + # zeek -e 'print Demo::rot13("Hello")' + Uryyb + +The installed version went into +``/lib/zeek/plugins/Demo_Rot13``. + +One can distribute the plugin independently of Zeek for others to use. +To distribute in source form, just remove the ``build/`` directory +(``make distclean`` does that) and then tar up the whole ``rot13-plugin/`` +directory. Others then follow the same process as above after +unpacking. + +To distribute the plugin in binary form, the build process +conveniently creates a corresponding tarball in ``build/dist/``. In +this case, it's called ``Demo_Rot13-0.1.0.tar.gz``, with the version +number coming out of the ``VERSION`` file that ``init-plugin`` put +into place. The binary tarball has everything needed to run the +plugin, but no further source files. Optionally, one can include +further files by specifying them in the plugin's ``CMakeLists.txt`` +through the ``zeek_plugin_dist_files`` macro; the skeleton does that +for ``README``, ``VERSION``, ``CHANGES``, and ``COPYING``. To use the +plugin through the binary tarball, just unpack it into +``/lib/zeek/plugins/``. Alternatively, if you unpack +it in another location, then you need to point ``ZEEK_PLUGIN_PATH`` there. + +Before distributing your plugin, you should edit some of the meta +files that ``init-plugin`` puts in place. Edit ``README`` and +``VERSION``, and update ``CHANGES`` when you make changes. Also put a +license file in place as ``COPYING``; if BSD is fine, you will find a +template in ``COPYING.edit-me``. + +Plugin Directory Layout +======================= + +A plugin's directory needs to follow a set of conventions so that Zeek +(1) recognizes it as a plugin, and (2) knows what to load. While +``init-plugin`` takes care of most of this, the following is the full +story. We'll use ```` to represent a plugin's top-level +directory. With the skeleton, ```` corresponds to ``build/``. + +``/__zeek_plugin__`` + A file that marks a directory as containing a Zeek plugin. The file + must exist, and its content must consist of a single line with the + qualified name of the plugin (e.g., "Demo::Rot13"). + +``/lib/.-.so`` + The shared library containing the plugin's compiled code. Zeek will + load this in dynamically at run-time if OS and architecture match + the current platform. + +``scripts/`` + A directory with the plugin's custom Zeek scripts. When the plugin + gets activated, this directory will be automatically added to + ``ZEEKPATH``, so that any scripts/modules inside can be + "@load"ed. + +``scripts``/__load__.zeek + A Zeek script that will be loaded when the plugin gets activated. + When this script executes, any BIF elements that the plugin + defines will already be available. See below for more information + on activating plugins. + +``scripts``/__preload__.zeek + A Zeek script that will be loaded when the plugin gets activated, + but before any BIF elements become available. See below for more + information on activating plugins. + +``lib/bif/`` + Directory with auto-generated Zeek scripts that declare the plugin's + BIF elements. The files here are produced by ``bifcl``. + +Any other files in ```` are ignored by Zeek. + +By convention, a plugin should put its custom scripts into sub folders +of ``scripts/``, i.e., ``scripts///', + # format='html') + # signode += rawnode + + else: + signode += addnodes.desc_name("", sig) + + return sig + + +class ZeekNamespace(ZeekGeneric): + def add_target_and_index(self, name, sig, signode): + targetname = self.get_obj_name() + "-" + name + + if targetname not in self.state.document.ids: + signode["names"].append(targetname) + signode["ids"].append(targetname) + signode["first"] = not self.names + self.state.document.note_explicit_target(signode) + + objects = self.env.domaindata["zeek"]["objects"] + key = (self.get_obj_name(), name) + objects[key] = self.env.docname + self.update_type_map(name) + + indextext = self.get_index_text(name) + self.indexnode["entries"].append( + make_index_tuple("single", indextext, targetname, targetname) + ) + self.indexnode["entries"].append( + make_index_tuple("single", f"namespaces; {sig}", targetname, targetname) + ) + + def get_index_text(self, name): + return _("%s (namespace); %s") % (name, self.env.docname) + + def handle_signature(self, sig, signode): + signode += addnodes.desc_name("", sig) + return sig + + +class ZeekEnum(ZeekGeneric): + def add_target_and_index(self, name, sig, signode): + targetname = self.get_obj_name() + "-" + name + + if targetname not in self.state.document.ids: + self.process_signode(name, sig, signode, targetname) + + objects = self.env.domaindata["zeek"]["objects"] + key = (self.get_obj_name(), name) + objects[key] = self.env.docname + self.update_type_map(name) + + # indextext = self.get_index_text(name) + # self.indexnode['entries'].append(make_index_tuple('single', indextext, + # targetname, targetname)) + m = sig.split() + + if len(m) < 2: + logger.warning( + "%s: zeek:enum directive missing argument(s)", self.env.docname + ) + return + + if m[1] == "Notice::Type": + if "notices" not in self.env.domaindata["zeek"]: + self.env.domaindata["zeek"]["notices"] = [] + self.env.domaindata["zeek"]["notices"].append( + (m[0], self.env.docname, targetname) + ) + + self.indexnode["entries"].append( + make_index_tuple( + "single", f"{m[1]} (enum values); {m[0]}", targetname, targetname + ) + ) + + def handle_signature(self, sig, signode): + m = sig.split() + name = m[0] + signode += addnodes.desc_name("", name) + return name + + +class ZeekParamField(docfields.GroupedField): + has_arg = True + is_typed = True + + +class ZeekIdentifier(ZeekGeneric): + zeek_param_field = ZeekParamField("param", label="Parameters", can_collapse=True) + field_type_map = {"param": (zeek_param_field, False)} + + def get_index_text(self, name): + return name + + def get_field_type_map(self): + return self.field_type_map + + +class ZeekNative(ZeekGeneric): + def handle_signature(self, sig, signode): + # The run() method is overridden to drop signode anyway in favor of + # simply adding the index and a target nodes and leaving up + # to the .rst document to explicitly add things that need to + # be presented in the final rendering (e.g. a section header) + self.native_name = sig + return sig + + def process_signode(self, name, sig, signode, targetname): + pass + + def run(self): + ns = super().run() + index_node = ns[0] + + target_id = self.get_obj_name() + "-" + self.native_name + target_node = nodes.target("", "", ids=[target_id]) + self.state.document.note_explicit_target(target_node) + + # Replace the description node from Sphinx with a simple target node + return [index_node, target_node] + + +class ZeekKeyword(ZeekNative): + def get_index_text(self, name): + if name and name[0] == "@": + return _("%s (directive)") % (name) + else: + return _("%s (keyword)") % (name) + + +class ZeekAttribute(ZeekNative): + def get_index_text(self, name): + return _("%s (attribute)") % (name) + + +class ZeekType(ZeekGeneric): + """ + Put the type that's currently documented into env.ref_context + for usage with the ZeekField directive. + """ + + def before_content(self): + self.env.ref_context["zeek:type"] = self.arguments[0] + + def after_content(self): + self.env.ref_context.pop("zeek:type", None) + + +class ZeekField(ZeekGeneric): + def handle_signature(self, sig, signode): + """ + The signature for .. zeek:field: currently looks like the following: + + .. zeek:field:: ts :zeek:type:`time` :zeek:attr:`&log` :zeek:attr:`&optional` + """ + parts = sig.split(" ", 2) + name, type_str = parts[0:2] + record_type = self.env.ref_context["zeek:type"] + fullname = "$".join([record_type, name]) + attrs_str = "" + if len(parts) == 3: + attrs_str = parts[2] + + type_nodes, _ = self.state.inline_text(type_str, -1) + + signode += addnodes.desc_name(name, name) + signode += addnodes.desc_sig_punctuation("", ":") + signode += addnodes.desc_sig_space() + signode += type_nodes + + if attrs_str: + attr_nodes, _ = self.state.inline_text(attrs_str, -1) + signode += addnodes.desc_sig_space() + signode += attr_nodes + + signode["class"] = record_type + signode["fullname"] = fullname + + return fullname + + def run(self): + idx, signode = super().run() + + record_type = self.env.ref_context["zeek:type"] + + fields = self.env.domaindata["zeek"].setdefault("fields", {}) + rfields = fields.setdefault(record_type, collections.OrderedDict()) + rfields[signode[0]["fullname"]] = { + "idx": idx, + "signode": signode, + } + + return [] + + +class ZeekNativeType(ZeekNative): + def get_obj_name(self): + # As opposed to using 'native-type', just imitate 'type'. + return "type" + + +class ZeekFieldXRefRole(XRefRole): + def process_link(self, env, refnode, has_explicit_title, title, target): + title, target = super().process_link( + env, refnode, has_explicit_title, title, target + ) + + parts = title.split("$") + if len(parts) == 2 and parts[0] and parts[1]: + # If a field is in Type$field, form, strip Type. + title = parts[1] + + return title, target + + +class ZeekNotices(Index): + """ + Index subclass to provide the Zeek notices index. + """ + + name = "noticeindex" + localname = _("Zeek Notice Index") + shortname = _("notices") + + def generate(self, docnames=None): + content = {} + + if "notices" not in self.domain.env.domaindata["zeek"]: + return content, False + + for n in self.domain.env.domaindata["zeek"]["notices"]: + modname = n[0].split("::")[0] + entries = content.setdefault(modname, []) + entries.append([n[0], 0, n[1], n[2], "", "", ""]) + + content = sorted(content.items()) + + return content, False + + +class ZeekDomain(Domain): + """Zeek domain.""" + + name = "zeek" + label = "Zeek" + + object_types = { + "type": ObjType(_("type"), "type"), + "native-type": ObjType(_("type"), "type"), + "namespace": ObjType(_("namespace"), "namespace"), + "id": ObjType(_("id"), "id"), + "keyword": ObjType(_("keyword"), "keyword"), + "enum": ObjType(_("enum"), "enum"), + "attr": ObjType(_("attr"), "attr"), + "field": ObjType(_("field"), "field"), + } + + directives = { + "type": ZeekType, + "native-type": ZeekNativeType, + "namespace": ZeekNamespace, + "id": ZeekIdentifier, + "keyword": ZeekKeyword, + "enum": ZeekEnum, + "attr": ZeekAttribute, + "field": ZeekField, + } + + roles = { + "type": XRefRole(), + "namespace": XRefRole(), + "id": XRefRole(), + "keyword": XRefRole(), + "enum": XRefRole(), + "attr": XRefRole(), + "see": XRefRole(), + "field": ZeekFieldXRefRole(), + } + + indices = [ + ZeekNotices, + ] + + initial_data = { + "objects": {}, # fullname -> docname, objtype + } + + def clear_doc(self, docname): + to_delete = [] + + for (typ, name), doc in self.data["objects"].items(): + if doc == docname: + to_delete.append((typ, name)) + + for typ, name in to_delete: + del self.data["objects"][typ, name] + + def resolve_xref(self, env, fromdocname, builder, typ, target, node, contnode): + objects = self.data["objects"] + + if typ == "see": + if target not in self.data["idtypes"]: + logger.warning( + '%s: unknown target for ":zeek:see:`%s`"', fromdocname, target + ) + return [] + + objtype = self.data["idtypes"][target] + return make_refnode( + builder, + fromdocname, + objects[objtype, target], + objtype + "-" + target, + contnode, + target + " " + objtype, + ) + elif typ == "field" and "$" not in target: + # :zeek:field:`x` without a record type ends up just x, no ref. + return [] + else: + objtypes = self.objtypes_for_role(typ) + + for objtype in objtypes: + if (objtype, target) in objects: + return make_refnode( + builder, + fromdocname, + objects[objtype, target], + objtype + "-" + target, + contnode, + target + " " + objtype, + ) + else: + logger.warning( + '%s: unknown target for ":zeek:%s:`%s`"', + fromdocname, + typ, + target, + ) + + def get_objects(self): + for (typ, name), docname in self.data["objects"].items(): + yield name, name, typ, docname, typ + "-" + name, 1 + + def merge_domaindata(self, docnames, otherdata): + """ + Merge domaindata in multiprocess mode. + + I'm quite unclear how the objects dict works out okay in single + process mode. For example, the file_entropy() event is defined + in scripts/base/bif/plugins/Zeek_FileEntropy.events.bif.zeek.rst + *and* in script-reference/autogenerated-file-analyzer-index.rst. + The current documentation refers to the first one for :zeek:see:. + It seems in single process mode the reading sorts filenames and + just uses the last highest sorting one. That ends-up being the one + in scripts/base. + + In [4]: "script-reference/autogenerated" < "scripts/base" + Out[4]: True + + """ + for target, data in otherdata.items(): + if target == "version": + continue + elif hasattr(data, "items"): + target_data = self.env.domaindata["zeek"].setdefault(target, {}) + + # Iterate manually over the elements for debugging + for k, v in data.items(): + if k not in target_data: + target_data[k] = v + else: + # The > comparison below updates the objects domaindata + # to filenames that sort higher. See comment above. + if isinstance(v, str): + if v > target_data[k]: + target_data[k] = v + else: + # Otherwise assume it's a dict and we can merge + # using update() + target_data[k].update(v) + + elif hasattr(data, "extend"): + # notices are a list + target_data = self.env.domaindata["zeek"].setdefault(target, []) + target_data.extend(data) + else: + raise NotImplementedError(target, type(data)) diff --git a/doc/ext/zeek_pygments.py b/doc/ext/zeek_pygments.py new file mode 100644 index 0000000000..aaa9f449fe --- /dev/null +++ b/doc/ext/zeek_pygments.py @@ -0,0 +1,247 @@ +from pygments.lexer import RegexLexer, bygroups, include, words +from pygments.token import ( + Comment, + Keyword, + Literal, + Name, + Number, + Operator, + Punctuation, + String, + Text, +) + + +def setup(Sphinx): + return { + "parallel_read_safe": True, + } + + +class ZeekLexer(RegexLexer): + """ + For `Zeek `_ scripts. + + .. versionadded:: 2.5 + """ + + name = "Zeek" + aliases = ["zeek"] + filenames = ["*.zeek"] + + _hex = r"[0-9a-fA-F]" + _float = r"((\d*\.?\d+)|(\d+\.?\d*))([eE][-+]?\d+)?" + _h = r"[A-Za-z0-9][-A-Za-z0-9]*" + + tokens = { + "root": [ + include("whitespace"), + include("comments"), + include("directives"), + include("attributes"), + include("types"), + include("keywords"), + include("literals"), + include("operators"), + include("punctuation"), + ( + r"\b((?:[A-Za-z_][A-Za-z_0-9]*)(?:::(?:[A-Za-z_][A-Za-z_0-9]*))*)(?=\s*\()", + Name.Function, + ), + include("identifiers"), + ], + "whitespace": [ + (r"\n", Text), + (r"\s+", Text), + (r"\\\n", Text), + ], + "comments": [ + (r"#.*$", Comment), + ], + "directives": [ + (r"(@(load-plugin|load-sigs|load|unload))\b.*$", Comment.Preproc), + ( + r"(@(DEBUG|DIR|FILENAME|deprecated|if|ifdef|ifndef|else|endif))\b", + Comment.Preproc, + ), + (r"(@prefixes)\s*(\+?=).*$", Comment.Preproc), + ], + "attributes": [ + ( + words( + ( + "redef", + "priority", + "log", + "optional", + "default", + "add_func", + "delete_func", + "expire_func", + "read_expire", + "write_expire", + "create_expire", + "synchronized", + "persistent", + "rotate_interval", + "rotate_size", + "encrypt", + "raw_output", + "mergeable", + "error_handler", + "broker_allow_complex_type", + "is_assigned", + "is_used", + "type_column", + "deprecated", + "on_change", + "backend", + "broker_store", + ), + prefix=r"&", + suffix=r"\b", + ), + Keyword.Pseudo, + ), + ], + "types": [ + ( + words( + ( + "any", + "enum", + "record", + "set", + "table", + "vector", + "function", + "hook", + "event", + "addr", + "bool", + "count", + "double", + "file", + "int", + "interval", + "pattern", + "port", + "string", + "subnet", + "time", + ), + prefix=r"\b", + suffix=r"\b", + ), + Keyword.Type, + ), + ( + r"\b(opaque)(\s+)(of)(\s+)((?:[A-Za-z_][A-Za-z_0-9]*)(?:::(?:[A-Za-z_][A-Za-z_0-9]*))*)\b", + bygroups(Keyword.Type, Text, Operator.Word, Text, Keyword.Type), + ), + ( + r"\b(type)(\s+)((?:[A-Za-z_][A-Za-z_0-9]*)(?:::(?:[A-Za-z_][A-Za-z_0-9]*))*)(\s*)(:)(\s*)\b(record|enum)\b", + bygroups(Keyword, Text, Name.Class, Text, Operator, Text, Keyword.Type), + ), + ( + r"\b(type)(\s+)((?:[A-Za-z_][A-Za-z_0-9]*)(?:::(?:[A-Za-z_][A-Za-z_0-9]*))*)(\s*)(:)", + bygroups(Keyword, Text, Name, Text, Operator), + ), + ( + r"\b(redef)(\s+)(record|enum)(\s+)((?:[A-Za-z_][A-Za-z_0-9]*)(?:::(?:[A-Za-z_][A-Za-z_0-9]*))*)\b", + bygroups(Keyword, Text, Keyword.Type, Text, Name.Class), + ), + ], + "keywords": [ + ( + words( + ( + "redef", + "export", + "if", + "else", + "for", + "while", + "return", + "break", + "next", + "continue", + "fallthrough", + "switch", + "default", + "case", + "add", + "delete", + "copy", + "when", + "timeout", + "schedule", + ), + prefix=r"\b", + suffix=r"\b", + ), + Keyword, + ), + (r"\b(print)\b", Keyword), + (r"\b(global|local|const|option)\b", Keyword.Declaration), + ( + r"\b(module)(\s+)(([A-Za-z_][A-Za-z_0-9]*)(?:::([A-Za-z_][A-Za-z_0-9]*))*)\b", + bygroups(Keyword.Namespace, Text, Name.Namespace), + ), + ], + "literals": [ + (r'"', String, "string"), + # Not the greatest match for patterns, but generally helps + # disambiguate between start of a pattern and just a division + # operator. + (r"/(?=.*/)", String.Regex, "regex"), + (r"\b(T|F)\b", Keyword.Constant), + # Port + (r"\b\d{1,5}/(udp|tcp|icmp|unknown)\b", Number), + # IPv4 Address + ( + r"\b(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[0-9]{1,2})\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[0-9]{1,2})\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[0-9]{1,2})\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[0-9]{1,2})\b", + Number, + ), + # IPv6 Address (not 100% correct: that takes more effort) + ( + r"\[([0-9a-fA-F]{0,4}:){2,7}([0-9a-fA-F]{0,4})?((25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[0-9]{1,2})\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[0-9]{1,2})\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[0-9]{1,2})\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[0-9]{1,2}))?\]", + Number, + ), + # Numeric + (r"\b0[xX]" + _hex + r"+\b", Number.Hex), + (r"\b" + _float + r"\s*(day|hr|min|sec|msec|usec)s?\b", Literal.Date), + (r"\b" + _float + r"\b", Number.Float), + (r"\b(\d+)\b", Number.Integer), + # Hostnames + (_h + r"(\." + _h + r")+", String), + ], + "operators": [ + (r"[!%*/+<=>~|&^-]", Operator), + (r"([-+=&|]{2}|[+=!><-]=)", Operator), + (r"\b(in|as|is|of)\b", Operator.Word), + (r"\??\$", Operator), + # Technically, colons are often used for punctuation/separation. + # E.g. field name/type separation. + (r"[?:]", Operator), + ], + "punctuation": [ + (r"\?\$", Punctuation), + (r"[{}()\[\],;:.]", Punctuation), + ], + "identifiers": [ + (r"([a-zA-Z_]\w*)(::)", bygroups(Name, Punctuation)), + (r"[a-zA-Z_]\w*", Name), + ], + "string": [ + (r"\\.", String.Escape), + (r"%-?[0-9]*(\.[0-9]+)?[DTdxsefg]", String.Escape), + (r'"', String, "#pop"), + (r".", String), + ], + "regex": [ + (r"\\.", String.Escape), + (r"/", String.Regex, "#pop"), + (r".", String.Regex), + ], + } diff --git a/doc/frameworks/broker.rst b/doc/frameworks/broker.rst new file mode 100644 index 0000000000..a70b042e5a --- /dev/null +++ b/doc/frameworks/broker.rst @@ -0,0 +1,644 @@ +.. _CAF: https://github.com/actor-framework/actor-framework + +.. _broker-framework: + +============================== +Broker Communication Framework +============================== + +.. rst-class:: opening + + Zeek uses the `Broker Library + `_ to exchange information with + other Zeek processes. Broker itself uses CAF_ (C++ Actor Framework) + internally for connecting nodes and exchanging arbitrary data over + networks. Broker then introduces, on top of CAF, a topic-based + publish/subscribe communication pattern using a data model that is + compatible to Zeek's. Broker itself can be utilized outside the + context of Zeek, with Zeek itself making use of only a few predefined + Broker message formats that represent Zeek events, log entries, etc. + + In summary, the Zeek's Broker framework provides basic facilities for + connecting broker-enabled peers (e.g. Zeek instances) to each other + and exchanging messages (e.g. events and logs). + +Cluster Layout / API +==================== + +Layout / Topology +----------------- + +In a Zeek cluster setup, every Zeek process is assigned a cluster role. +Such a process is then called a Zeek node, a cluster node, or just named +after the role of the process (the manager, the loggers, ...). A basic Zeek +cluster uses four different node types, enumerated in the script-level +variable :zeek:see:`Cluster::NodeType`. + +- Manager +- Logger +- Worker +- Proxy + +In small Zeek deployments, all nodes may run on a single host. In large +Zeek deployments, nodes may be distributed across multiple physical +systems for scaling. + +Currently, a single Manager node in a Zeek cluster exists. Further, connectivity +between nodes is determined statically based on their type: + +- Every node connects to all loggers and the manager. + +- Each worker connects to all proxies. + + +.. figure:: broker/cluster-layout.png + +Some general suggestions as to the purpose/utilization of each node type: + +- Workers: are a good first choice for doing the brunt of any work you need + done. They should be spending a lot of time performing the actual job + of parsing/analyzing incoming data from packets, so you might choose + to look at them as doing a "first pass" analysis and then deciding how + the results should be shared with other nodes in the cluster. + +- Proxies: serve as intermediaries for data storage and work/calculation + offloading. Good for helping offload work or data in a scalable and + distributed way. Since any given worker is connected to all + proxies and can agree on an "arbitrary key -> proxy node" mapping + (more on that later), you can partition work or data amongst them in a + uniform manner. e.g. you might choose to use proxies as a method of + sharing non-persistent state or as a "second pass" analysis for any + work that you don't want interfering with the workers' capacity to + keep up with capturing and parsing packets. Note that the default scripts + that come with Zeek make minimal use of proxies, so if you are coming + from a previous ZeekControl deployment, you may want to try reducing down + to a single proxy node. If you come to have custom/community scripts + that utilize proxies, that would be the time to start considering scaling + up the number of proxies to meet demands. + +- Manager: this node will be good at performing decisions that require a + global view of things since it is in a centralized location, connected + to everything. However, that also makes it easy to overload, so try + to use it sparingly and only for tasks that must be done in a + centralized or authoritative location. Optionally, for some + deployments, the Manager can also serve as the sole Logger. + +- Loggers: these nodes should simply be spending their time writing out + logs to disk and not used for much else. In the default cluster + configuration, logs get distributed among available loggers in a + round-robin fashion, providing failover capability should any given + logger temporarily go offline. + +Data Management/Sharing Strategies +================================== + +There's maybe no single, best approach or pattern to use when you need a +Zeek script to store or share long-term state and data. The two +approaches that were previously used were either using the ``&synchronized`` +attribute on tables/sets or by explicitly sending events to specific +nodes on which you wanted data to be stored. The former is no longer +possible, though there are several new possibilities that the new +Broker/Cluster framework offer, namely distributed data store and data +partitioning APIs. + +Data Stores +----------- + +Broker provides a distributed key-value store interface with optional +choice of using a persistent backend. For more detail, see +:ref:`this example `. + +Some ideas/considerations/scenarios when deciding whether to use +a data store for your use-case: + +* If you need the full data set locally in order to achieve low-latency + queries using data store "clones" can provide that. + +* If you need data that persists across restarts of Zeek processes, then + data stores can also provide that. + +* If the data you want to store is complex (tables, sets, records) or + you expect to read, modify, and store back, then data stores may not + be able to provide simple, race-free methods of performing the pattern + of logic that you want. + +* If the data set you want to store is excessively large, that's still + problematic even for stores that use a persistent backend as they are + implemented in a way that requires a full snapshot of the store's + contents to fit in memory (this limitation may change in the future). + +Data Partitioning +----------------- + +New data partitioning strategies are available using the API in +:doc:`/scripts/base/frameworks/cluster/pools.zeek`. Using that API, developers +of custom Zeek scripts can define a custom pool of nodes that best fits the +needs of their script. + +One example strategy is to use Highest Random Weight (HRW) hashing to +partition data tables amongst the pool of all proxy nodes. e.g. using +:zeek:see:`Cluster::publish_hrw`. This could allow clusters to +be scaled more easily than the approach of "the entire data set gets +synchronized to all nodes" as the solution to memory limitations becomes +"just add another proxy node". It may also take away some of the +messaging load that used to be required to synchronize data sets across +all nodes. + +The tradeoff of this approach, is that nodes that leave the pool (due to +crashing, etc.) cause a temporary gap in the total data set until +workers start hashing keys to a new proxy node that is still alive, +causing data to now be located and updated there. + +If the developer of a script expects its workload to be particularly +intensive, wants to ensure that their operations get exclusive +access to nodes, or otherwise set constraints on the number of nodes within +a pool utilized by their script, then the :zeek:see:`Cluster::PoolSpec` +structure will allow them to do that while still allowing users of that script +to override the default suggestions made by the original developer. + +Broker Framework Examples +========================= + +The broker framework provides basic facilities for connecting Zeek instances +to each other and exchanging messages, like events or logs. + +See :doc:`/scripts/base/frameworks/broker/main.zeek` for an overview +of the main Broker API. + +.. _broker_topic_naming: + +Topic Naming Conventions +------------------------ + +All Broker-based messaging involves two components: the information you +want to send (e.g. an event w/ its arguments) along with an associated +topic name string. The topic strings are used as a filtering mechanism: +Broker uses a publish/subscribe communication pattern where peers +advertise interest in topic **prefixes** and only receive messages which +match one of their prefix subscriptions. + +Broker itself supports arbitrary topic strings, however Zeek generally +follows certain conventions in choosing these topics to help avoid +conflicts and generally make them easier to remember. + +As a reminder of how topic subscriptions work, subscribers advertise +interest in a topic **prefix** and then receive any messages published by a +peer to a topic name that starts with that prefix. E.g. Alice +subscribes to the "alice/dogs" prefix, then would receive the following +message topics published by Bob: + +- topic "alice/dogs/corgi" +- topic "alice/dogs" +- topic "alice/dogsarecool/oratleastilikethem" + +Alice would **not** receive the following message topics published by Bob: + +- topic "alice/cats/siamese" +- topic "alice/cats" +- topic "alice/dog" +- topic "alice" + +Note that the topics aren't required to form a slash-delimited hierarchy, +the subscription matching is purely a byte-per-byte prefix comparison. + +However, Zeek scripts generally will follow a topic naming hierarchy and +any given script will make the topic names it uses apparent via some +redef'able constant in its export section. Generally topics that Zeek +scripts use will be along the lines of :samp:`zeek/{}/{}` +with :samp:`{}` being the script's module name (in all-undercase). +For example, you might expect an imaginary ``Pretend`` framework to +publish/subscribe using topic names like ``zeek/pretend/my_cool_event``. +For scripts that use Broker as a means of cluster-aware analysis, +it's usually sufficient for them to make use of the topics declared +by the cluster framework. For scripts that are meant to establish +communication flows unrelated to Zeek cluster, new topics are declared +(examples being the NetControl and Control frameworks). + +For cluster operation, see :doc:`/scripts/base/frameworks/cluster/main.zeek` +for a list of topics that are useful for steering published events to +the various node classes. E.g. you have the ability to broadcast +to all nodes of a given class (e.g. just workers) or just send to a +specific node within a class. + +The topic names that logs get published under are a bit nuanced. In the +default cluster configuration, they are round-robin published to +explicit topic names that identify a single logger. In standalone Zeek +processes, logs get published to the topic indicated by +:zeek:see:`Broker::default_log_topic_prefix`. + +For those writing their own scripts which need new topic names, a +suggestion would be to avoid prefixing any new topics/prefixes with +``zeek/`` as any changes in scripts shipping with Zeek will use that prefix +and it's better to not risk unintended conflicts. Again, it's +often less confusing to just re-use existing topic names instead +of introducing new topic names. The typical use case is writing +a cluster-enabled script, which usually just needs to route events +based upon node classes, and that already has usable topics in the +cluster framework. + +Connecting to Peers +------------------- + +Zeek can accept incoming connections by calling :zeek:see:`Broker::listen`. + +.. literalinclude:: broker/connecting-listener.zeek + :caption: connecting-listener.zeek + :language: zeek + :linenos: + :tab-width: 4 + +Zeek can initiate outgoing connections by calling :zeek:see:`Broker::peer`. + +.. literalinclude:: broker/connecting-connector.zeek + :caption: connecting-connector.zeek + :language: zeek + :linenos: + :tab-width: 4 + +In either case, connection status updates are monitored via the +:zeek:see:`Broker::peer_added` and :zeek:see:`Broker::peer_lost` events. + +Remote Events +------------- + +To receive remote events, you need to first subscribe to a "topic" to which +the events are being sent. A topic is just a string chosen by the sender, +and named in a way that helps organize events into various categories. +See the :ref:`topic naming conventions section ` for +more on how topics work and are chosen. + +Use the :zeek:see:`Broker::subscribe` function to subscribe to topics and +define any event handlers for events that peers will send. + +.. literalinclude:: broker/events-listener.zeek + :caption: events-listener.zeek + :language: zeek + :linenos: + :tab-width: 4 + +To send an event, call the :zeek:see:`Broker::publish` function which you can +supply directly with the event and its arguments or give it the return value of +:zeek:see:`Broker::make_event` in case you need to send the same event/args +multiple times. When publishing events like this, local event handlers for +the event are not called, even if a matching subscription exists. + +.. literalinclude:: broker/events-connector.zeek + :caption: events-connector.zeek + :language: zeek + :linenos: + :tab-width: 4 + +Note that the subscription model is prefix-based, meaning that if you subscribe +to the ``zeek/events`` topic prefix you would receive events that are published +to topic names ``zeek/events/foo`` and ``zeek/events/bar`` but not +``zeek/misc``. + +.. note:: + + In prior Zeek versions, ``Broker::auto_publish`` was available to automatically + send events to peers whenever the events were called locally via the normal + event invocation syntax. When auto-publishing events, local event handlers for + the event were called in addition to sending the event to any subscribed peers. + + ``Broker::auto_publish`` was removed due to its + `implicit nature `_. + + +Remote Logging +-------------- + +.. literalinclude:: broker/testlog.zeek + :caption: testlog.zeek + :language: zeek + :linenos: + :tab-width: 4 + +To toggle remote logs, redef :zeek:see:`Log::enable_remote_logging`. +Use the :zeek:see:`Broker::subscribe` function to advertise interest +in logs written by peers. The topic names that Zeek uses are determined by +:zeek:see:`Broker::log_topic`. + +.. literalinclude:: broker/logs-listener.zeek + :caption: logs-listener.zeek + :language: zeek + :linenos: + :tab-width: 4 + +.. literalinclude:: broker/logs-connector.zeek + :caption: logs-connector.zeek + :language: zeek + :linenos: + :tab-width: 4 + +Note that logging events are only raised locally on the node that performs +the :zeek:see:`Log::write` and not automatically published to peers. + +.. _data_store_example: + +Distributed Data Stores +----------------------- + +See :doc:`/scripts/base/frameworks/broker/store.zeek` for an overview +of the Broker data store API. + +There are two flavors of key-value data store interfaces: master and clone. + +A master data store can be cloned from remote peers which may then +perform lightweight, local queries against the clone, which +automatically stays synchronized with the master store. Clones cannot +modify their content directly, instead they send modifications to the +centralized master store which applies them and then broadcasts them to +all clones. + +Master stores get to choose what type of storage backend to +use. E.g. In-memory versus SQLite for persistence. + +Data stores also support expiration on a per-key basis using an amount of +time relative to the entry's last modification time. + +.. literalinclude:: broker/stores-listener.zeek + :caption: stores-listener.zeek + :language: zeek + :linenos: + :tab-width: 4 + +.. literalinclude:: broker/stores-connector.zeek + :caption: stores-connector.zeek + :language: zeek + :linenos: + :tab-width: 4 + +Note that all data store queries must be made within Zeek's asynchronous +``when`` statements and must specify a timeout block. + + +SQLite Data Store Tuning +^^^^^^^^^^^^^^^^^^^^^^^^ + +When leveraging the SQLite backend for persistence, SQLite's default journaling +and consistency settings are used. Concretely, ``journal_mode`` is set to +``DELETE`` and ``synchronous`` to ``FULL``. This in turn is not optimal for +`high INSERT or UPDATE rates `_ +due to SQLite waiting for the required IO to complete until data is safely +on disk. This can also have a non-negligible system effect when the +SQLite database is located on the same device as other IO critical processes. + +Starting with Zeek 5.2, it is possible to tune and relax these settings by +providing an appropriate :zeek:see:`Broker::BackendOptions` and +:zeek:see:`Broker::SQLiteOptions` instance to +:zeek:see:`Broker::create_master`. The following example changes the +data store to use `Write-Ahead Logging `_ +which should perform significantly faster than the default. + + +.. literalinclude:: broker/store-sqlite-tuning.zeek + :caption: store-sqlite-tuning.zeek + :language: zeek + :linenos: + :tab-width: 4 + +If your use-case turns out to require more and lower-level tuning around +SQLite options, please get in contact or open a feature request on GitHub. + + +Cluster Framework Examples +========================== + +This section contains a few brief examples of how various communication +patterns one might use when developing Zeek scripts that are to operate in +the context of a cluster. + +.. _event-namespacing-pitfall: + +A Reminder About Events and Module Namespaces +--------------------------------------------- + +For simplicity, the following examples do not use any modules/namespaces. +If you choose to use them within your own code, it's important to +remember that the ``event`` and ``schedule`` dispatching statements +should always use the fully-qualified event name. + +For example, this will likely not work as expected: + +.. code-block:: zeek + + module MyModule; + + export { + global my_event: event(); + } + + event my_event() + { + print "got my event"; + } + + event zeek_init() + { + event my_event(); + schedule 10sec { my_event() }; + } + +This code runs without errors, however, the local ``my_event`` handler +will never be called and also not any remote handlers either. Instead, at +minimum you would need change the ``zeek_init()`` handler: + +.. code-block:: zeek + + event zeek_init() + { + event MyModule::my_event(); + schedule 10sec { MyModule::my_event() }; + } + +Though, an easy rule of thumb to remember would be to always use the +explicit module namespace scoping and you can't go wrong: + +.. code-block:: zeek + + module MyModule; + + export { + global MyModule::my_event: event(); + } + + event MyModule::my_event() + { + print "got my event"; + } + + event zeek_init() + { + event MyModule::my_event(); + schedule 10sec { MyModule::my_event() }; + } + +Event types that reside in the default namespace (such as :zeek:id:`zeek_init` or +:zeek:id:`connection_established`) require no qualification, even when scheduled from +inside a module. Don't force qualification of such events by prefixing with +``GLOBAL::``. + +Note that other identifiers in Zeek do not have this inconsistency +related to module namespacing, it's just events that require +explicitness. + +Manager Sending Events To Workers +--------------------------------- + +This is fairly straightforward, we just need a topic name which we know +all workers are subscribed combined with the event we want to send them. + +.. code-block:: zeek + + event manager_to_workers(s: string) + { + print "got event from manager", s; + } + + event some_event_handled_on_manager() + { + Broker::publish(Cluster::worker_topic, manager_to_workers, + "hello v0"); + + # If you know this event is only handled on the manager, you don't + # need any of the following conditions, they're just here as an + # example of how you can further discriminate based on node identity. + + # Can check based on the name of the node. + if ( Cluster::node == "manager" ) + Broker::publish(Cluster::worker_topic, manager_to_workers, + "hello v1"); + + # Can check based on the type of the node. + if ( Cluster::local_node_type() == Cluster::MANAGER ) + Broker::publish(Cluster::worker_topic, manager_to_workers, + "hello v2"); + + # The run-time overhead of the above conditions can even be + # eliminated by using the following conditional directives. + # It's evaluated once per node at parse-time and, if false, + # any code within is just ignored / treated as not existing at all. + @if ( Cluster::local_node_type() == Cluster::MANAGER ) + Broker::publish(Cluster::worker_topic, manager_to_workers, + "hello v3"); + @endif + } + +Worker Sending Events To Manager +-------------------------------- + +This should look almost identical to the previous case of sending an event +from the manager to workers, except it simply changes the topic name to +one which the manager is subscribed. + +.. code-block:: zeek + + event worker_to_manager(worker_name: string) + { + print "got event from worker", worker_name; + } + + event some_event_handled_on_worker() + { + Broker::publish(Cluster::manager_topic, worker_to_manager, + Cluster::node); + } + +Worker Sending Events To All Workers +------------------------------------ + +Since workers are not directly connected to each other in the cluster +topology, this type of communication is a bit different than what we +did before since we have to manually relay the event via some node that *is* +connected to all workers. The manager or a proxy satisfies that requirement: + +.. code-block:: zeek + + event worker_to_workers(worker_name: string) + { + @if ( Cluster::local_node_type() == Cluster::MANAGER || + Cluster::local_node_type() == Cluster::PROXY ) + Broker::publish(Cluster::worker_topic, worker_to_workers, + worker_name); + @else + print "got event from worker", worker_name; + @endif + } + + event some_event_handled_on_worker() + { + # We know the manager is connected to all workers, so we could + # choose to relay the event across it. + Broker::publish(Cluster::manager_topic, worker_to_workers, + Cluster::node + " (via manager)"); + + # We also know that any given proxy is connected to all workers, + # though now we have a choice of which proxy to use. If we + # want to distribute the work associated with relaying uniformly, + # we can use a round-robin strategy. The key used here is simply + # used by the cluster framework internally to keep track of + # which node is up next in the round-robin. + local pt = Cluster::rr_topic(Cluster::proxy_pool, "example_key"); + Broker::publish(pt, worker_to_workers, + Cluster::node + " (via a proxy)"); + } + +Worker Distributing Events Uniformly Across Proxies +--------------------------------------------------- + +If you want to offload some data/work from a worker to your proxies, +we can make use of a `Highest Random Weight (HRW) hashing +`_ distribution strategy +to uniformly map an arbitrary key space across all available proxies. + +.. code-block:: zeek + + event worker_to_proxies(worker_name: string) + { + print "got event from worker", worker_name; + } + + global my_counter = 0; + + event some_event_handled_on_worker() + { + # The key here is used to choose which proxy shall receive + # the event. Different keys may map to different nodes, but + # any given key always maps to the same node provided the + # pool of nodes remains consistent. If a proxy goes offline, + # that key maps to a different node until the original comes + # back up. + Cluster::publish_hrw(Cluster::proxy_pool, + cat("example_key", ++my_counter), + worker_to_proxies, Cluster::node); + } + +Broker-backed Zeek Tables for Data Synchronization and Persistence +================================================================== + +Starting with Zeek 3.2, it is possible to "bind" a Zeek table to a backing +Broker store. Changes to the Zeek table are sent to the Broker store. Similarly, +changes of the Broker store are applied to the Zeek table. + +This feature allows easy distribution of table contents across a cluster. +It also offers persistence for tables (when using a persistent Broker store +backend like SQLite). + +To give a short example, to distribute a table over a cluster you can use +the :zeek:attr:`&backend` attribute. + +.. code-block:: zeek + + global t: table[string] of count &backend=Broker::MEMORY; + +The :zeek:attr:`&backend` attribute creates a master data store on the +manager and a clone data store on all other node on the cluster. This +in essence means that the table exists twice in each Zeek process. One +copy of the table is contained in a Broker data store (either a master +or a clone depending on the node), which data store distributes the +data across the cluster---and, depending on the backend, might also +make the data persistent. Since Broker data stores are only accessible +via asynchronous operations, and accessing them might not always be +immediate, a second copy of the table, which is immediately +accessible, is held inside the Zeek core. This is the copy that you +see and interact with on the Zeek side. diff --git a/doc/frameworks/broker/cluster-layout.png b/doc/frameworks/broker/cluster-layout.png new file mode 100644 index 0000000000..3813bfbfda Binary files /dev/null and b/doc/frameworks/broker/cluster-layout.png differ diff --git a/doc/frameworks/broker/cluster-layout.xml b/doc/frameworks/broker/cluster-layout.xml new file mode 100644 index 0000000000..4269c6723f --- /dev/null +++ b/doc/frameworks/broker/cluster-layout.xml @@ -0,0 +1,2 @@ + +7VxLc6M4EP41Po4LSUjAcZJJZg+7VVOVrd3ZowwKZgYjFyaxvb9+hZEwEhA/eITs2JdYrZYE0ve1ultyZuh+tfua0vXyDx6weAatYDdDX2ZQfDxL/Mkl+0Li2G4hCNMoKETgKHiK/mVSKNuFL1HANppixnmcRWtd6PMkYX6myWia8q2u9sxjfdQ1DVlN8OTTuC79OwqypZQCyzpW/MaicCmHdrGsWFD/Z5jyl0SON4Po+fApqldU9SX1N0sa8G1FhB5m6D7lPCu+rXb3LM7nVk1b0e6xpbZ87pQl2VkNGEXAcTwrgD71XPQJoKKLVxq/MPUOhyfN9mp2WCAmSxYTnog/d4dXZnmnQJSW2SqWX2O6YPFdOSv3PObpsdkmo2n2OV8wQ/YYxXkPlipLiGBRZkmgWvgx3Wwi/89llBQVshkoSpVGP1iW7WWZvmRciHiaLXnIExr/zvlattpkKf/J1FOK1fOIgz6TskahIdd95kn2SFdRnIP8L5YGNKFSLEcCUJYrHVqHj5An/KEyj4dH3kXZ9/yt51iW/pFzcHxWqVpfZ7n0G/6S+qxtcSWVaBqyrEXHK3TyNa50LNHzlfEVy9K9UEhZTLPoVScMlbwLSz3ZVKwY3VcU1jxKsk2l52+5QCgoE6KYJg0IcqCOY1MfQTFp1Ra2bVdbiC/FM6hS5WWOogM7zmWKfWPKL80UNAmmAHIZUwB0RmdKjSgrAYCQpTNIYjG7d4v8W5iVq1VlUByLTT9H+3YZZexpTQ9rthV+h06fI68OVFD7al7l81Xky4pTLGsANW0BtWBQRZOBADOnO9hpHIVJzliBVzFDbwD4laUZ270JPVWL9BVHyrpuj86NctmWFbdGQasJrBW4XIYG2GA2Cxhs1jRRQFinfLf/BJsQUkjEuFX9qQEncLyFZZ0DnOdnRnx/msCBUAcOwHXgOLgOHDwEcJo80zpwYh4Ky5LbnI+BE7Ig+CwDI4IIOFWcePoeZJN3hIlXg4mERIN7dlv7HjYXD7/b4t+CVQ1QXxzvrm3T6Qac0uGuuNvFg4sV+14tKEe87rT35JrDM1xzMIhrXvOlbYLmnmMjDCxgI9dBjsYED84RJB7GmCAHQ7334h1lh29Fth6YE9GLhRwxAlbOerkj2/Y8790Rr01sYBFjmGKaasO0Rxka9S5z/JsC4jbPDtw8u3e12kbUOKZjh29Ge1yjrTImNaNtDWW07enYaAKN/Agi1xlirLwN1RExOhrT1JIbh0biUMmVObIcjS9zizRSpjk52UimQ2ffWBqJpc8t+2Eqe2PYMKn8GjGQbTAM4OsYZqO+GdaSArWMiAVoh2SdE6DOjZwjRyU1plVoS1yDtfC8je56bl4VsgxzmlAnK3J1jlnOdWRFjt4RgEZHPZEV4mHJqixphZx63FBhYnu0ezU57lxs42ZyNMXcBdL7ctO8Oi7tcWBonu/a5lFDCwwvNvVQD5e9nsFDLrPsJp5OOeij4GxANE30dgFCRvRrGbdkTuirMHgo/d5tnNsNpsfNFI9q+Grb+phQhSNZQqLvo90tYYewEoFuQGmwZxXM1C3avJNjORGbNo17IMjM6J2yaeRCG9VR37BpdR4YUSS2+7WB9WPBpuT0lqc/i6PCD5qdXrRzwsxOuz6bana6TBKPcYAMG5BxC9ff4xDRmtumfzFooH5OEu0WlzeP4wwblt/uoU/nlGhOiJn5nmYObaSEN1Fucpnwdq/jKnb1jgA09rO+cmjuwDk0XOPmFTHCIKHk4EedgJwBzJFiSdfMqoGB8KTOUAfKqjmXmfoWhNXzpu8UabaA8PBI/WFsJOMHjPN0bOYrumLsVPx6Kv48s/5UPFp7z57jURWQdgX5W0dfo2TrhjSkw5xGDJM6s/pZT2M1p2WyejVYI0VW4NRRU0+b4qVnChem0zqp9x6dNd0/as2mfdy7nv+PbNqYv8ZoSrP+wmH7G1Z4oGTamCcfI13hhMZ9rauvcNrmTTUT8n1dMDN+EmPsBd19x/ovam8sGzU5dpELNUc5UrT8WemZX5ccO8e9Gomc5W1P5Wp4BqfOJWf5EwTl4pgd9UVO0is5RfH471oK9eP/xEEP/wE= \ No newline at end of file diff --git a/doc/frameworks/broker/connecting-connector.zeek b/doc/frameworks/broker/connecting-connector.zeek new file mode 100644 index 0000000000..f1f8cbf872 --- /dev/null +++ b/doc/frameworks/broker/connecting-connector.zeek @@ -0,0 +1,12 @@ +redef exit_only_after_terminate = T; + +event zeek_init() + { + Broker::peer("127.0.0.1"); + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + print "peer added", endpoint; + terminate(); + } diff --git a/doc/frameworks/broker/connecting-listener.zeek b/doc/frameworks/broker/connecting-listener.zeek new file mode 100644 index 0000000000..7802229996 --- /dev/null +++ b/doc/frameworks/broker/connecting-listener.zeek @@ -0,0 +1,17 @@ +redef exit_only_after_terminate = T; + +event zeek_init() + { + Broker::listen("127.0.0.1"); + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + print "peer added", endpoint; + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + print "peer lost", endpoint; + terminate(); + } diff --git a/doc/frameworks/broker/events-connector.zeek b/doc/frameworks/broker/events-connector.zeek new file mode 100644 index 0000000000..fb4bec92ef --- /dev/null +++ b/doc/frameworks/broker/events-connector.zeek @@ -0,0 +1,26 @@ +redef exit_only_after_terminate = T; +global my_event: event(msg: string, c: count); + +event zeek_init() + { + Broker::peer("127.0.0.1"); + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + print "peer added", endpoint; + Broker::publish("zeek/event/my_event", my_event, "hi", 0); + Broker::publish("zeek/event/my_event", my_event, "...", 1); + local e = Broker::make_event(my_event, "bye", 2); + Broker::publish("zeek/event/my_event", e); + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + terminate(); + } + +event my_event(msg: string, c: count) + { + print "got my_event", msg, c; + } diff --git a/doc/frameworks/broker/events-listener.zeek b/doc/frameworks/broker/events-listener.zeek new file mode 100644 index 0000000000..374dc5db11 --- /dev/null +++ b/doc/frameworks/broker/events-listener.zeek @@ -0,0 +1,24 @@ +redef exit_only_after_terminate = T; +global msg_count = 0; +global my_event: event(msg: string, c: count); +global my_auto_event: event(msg: string, c: count); + +event zeek_init() + { + Broker::subscribe("zeek/event/"); + Broker::listen("127.0.0.1"); + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + print "peer added", endpoint; + } + +event my_event(msg: string, c: count) + { + ++msg_count; + print "got my_event", msg, c; + + if ( msg_count == 5 ) + terminate(); + } diff --git a/doc/frameworks/broker/logs-connector.zeek b/doc/frameworks/broker/logs-connector.zeek new file mode 100644 index 0000000000..47d912e294 --- /dev/null +++ b/doc/frameworks/broker/logs-connector.zeek @@ -0,0 +1,36 @@ +@load ./testlog + +redef exit_only_after_terminate = T; +global n = 0; + +event zeek_init() + { + Broker::peer("127.0.0.1"); + } + +event do_write() + { + if ( n == 6 ) + return; + + Log::write(Test::LOG, [$msg = "ping", $num = n]); + ++n; + event do_write(); + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + print "peer added", endpoint; + event do_write(); + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + terminate(); + } + +event Test::log_test(rec: Test::Info) + { + print "wrote log", rec; + Broker::publish("zeek/logs/forward/test", Test::log_test, rec); + } diff --git a/doc/frameworks/broker/logs-listener.zeek b/doc/frameworks/broker/logs-listener.zeek new file mode 100644 index 0000000000..654551b940 --- /dev/null +++ b/doc/frameworks/broker/logs-listener.zeek @@ -0,0 +1,22 @@ +@load ./testlog + +redef exit_only_after_terminate = T; + +event zeek_init() + { + Broker::subscribe("zeek/logs"); + Broker::listen("127.0.0.1"); + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + print "peer added", endpoint; + } + +event Test::log_test(rec: Test::Info) + { + print "got log event", rec; + + if ( rec$num == 5 ) + terminate(); + } diff --git a/doc/frameworks/broker/store-sqlite-tuning.zeek b/doc/frameworks/broker/store-sqlite-tuning.zeek new file mode 100644 index 0000000000..4c59456013 --- /dev/null +++ b/doc/frameworks/broker/store-sqlite-tuning.zeek @@ -0,0 +1,19 @@ +global h: opaque of Broker::Store; + +event zeek_init() + { + # Use WAL mode. + local sqlite_options=Broker::SQLiteOptions( + $synchronous=Broker::SQLITE_SYNCHRONOUS_NORMAL, + $journal_mode=Broker::SQLITE_JOURNAL_MODE_WAL, + ); + local options = Broker::BackendOptions($sqlite=sqlite_options); + h = Broker::create_master("persistent-store", Broker::SQLITE, options); + + local c = 1000; + while (c > 0) + { + Broker::put(h, cat(c), rand(10000)); + --c; + } + } diff --git a/doc/frameworks/broker/stores-connector.zeek b/doc/frameworks/broker/stores-connector.zeek new file mode 100644 index 0000000000..4c09d9b950 --- /dev/null +++ b/doc/frameworks/broker/stores-connector.zeek @@ -0,0 +1,29 @@ +redef exit_only_after_terminate = T; + +global h: opaque of Broker::Store; + +global ready: event(); + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + terminate(); + } + +event zeek_init() + { + h = Broker::create_master("mystore"); + + local myset: set[string] = {"a", "b", "c"}; + local myvec: vector of string = {"alpha", "beta", "gamma"}; + Broker::put(h, "one", 110); + Broker::put(h, "two", 223); + Broker::put(h, "myset", myset); + Broker::put(h, "myvec", myvec); + Broker::increment(h, "one"); + Broker::decrement(h, "two"); + Broker::insert_into_set(h, "myset", "d"); + Broker::remove_from(h, "myset", "b"); + Broker::push(h, "myvec", "delta"); + + Broker::peer("127.0.0.1"); + } diff --git a/doc/frameworks/broker/stores-listener.zeek b/doc/frameworks/broker/stores-listener.zeek new file mode 100644 index 0000000000..059444226f --- /dev/null +++ b/doc/frameworks/broker/stores-listener.zeek @@ -0,0 +1,79 @@ +redef exit_only_after_terminate = T; + +global h: opaque of Broker::Store; +global expected_key_count = 4; +global key_count = 0; + +# Lookup a value in the store based on an arbitrary key string. +function do_lookup(key: string) + { + when ( local res = Broker::get(h, key) ) + { + ++key_count; + print "lookup", key, res; + + # End after we iterated over looking up each key in the store twice. + if ( key_count == expected_key_count * 2 ) + terminate(); + } + # All data store queries must specify a timeout + timeout 3sec + { print "timeout", key; } + } + +event check_keys() + { + # Here we just query for the list of keys in the store, and show how to + # look up each one's value. + when ( local res = Broker::keys(h) ) + { + print "clone keys", res; + + if ( res?$result ) + { + # Since we know that the keys we are storing are all strings, + # we can conveniently cast the result of Broker::keys to + # a native Bro type, namely 'set[string]'. + for ( k in res$result as string_set ) + do_lookup(k); + + # Alternatively, we can use a generic iterator to iterate + # over the results (which we know is of the 'set' type because + # that's what Broker::keys() always returns). If the keys + # we stored were not all of the same type, then you would + # likely want to use this method of inspecting the store's keys. + local i = Broker::set_iterator(res$result); + + while ( ! Broker::set_iterator_last(i) ) + { + do_lookup(Broker::set_iterator_value(i) as string); + Broker::set_iterator_next(i); + } + } + } + # All data store queries must specify a timeout. + # You also might see timeouts on connecting/initializing a clone since + # it hasn't had time to get fully set up yet. + timeout 1sec + { + print "timeout"; + schedule 1sec { check_keys() }; + } + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + print "peer added"; + # We could create a clone early, like in zeek_init and it will periodically + # try to synchronize with its master once it connects, however, we just + # create it now since we know the peer w/ the master store has just + # connected. + h = Broker::create_clone("mystore"); + + event check_keys(); + } + +event zeek_init() + { + Broker::listen("127.0.0.1"); + } diff --git a/doc/frameworks/broker/testlog.zeek b/doc/frameworks/broker/testlog.zeek new file mode 100644 index 0000000000..f52f177cf1 --- /dev/null +++ b/doc/frameworks/broker/testlog.zeek @@ -0,0 +1,17 @@ +module Test; + +export { + redef enum Log::ID += { LOG }; + + type Info: record { + msg: string &log; + num: count &log; + }; + + global log_test: event(rec: Test::Info); +} + +event zeek_init() &priority=5 + { + Log::create_stream(Test::LOG, [$columns=Test::Info, $ev=log_test, $path="test"]); + } diff --git a/doc/frameworks/cluster.rst b/doc/frameworks/cluster.rst new file mode 100644 index 0000000000..ed9766f45c --- /dev/null +++ b/doc/frameworks/cluster.rst @@ -0,0 +1,630 @@ + +.. _cluster-framework: + +================= +Cluster Framework +================= + +The basic premise of Zeek clusterization is to break down network traffic into +smaller pieces, while preserving the affinity of individual network sessions to +a single analysis process. Cluster architecture thus allows Zeek to distribute +that analysis across many dozens or hundreds of worker processes, allowing the +monitoring system to scale up to line speeds of 100G or more. + +.. figure:: /images/cluster-diagram.png + + Figure 1: Block diagram of cluster setup showing multiple network feeds to a + traffic aggregator. This device sends traffic to workers after symmetric + hashing/load-balancing. Traffic is then fed to the Zeek cluster using + load-balancing network cards. + +Zeek's Cluster Components +========================= + +By distributing network traffic across hosts and processes, overall traffic +finally reaches a volume that can be effectively analyzed by a single worker +process. Zeek then acts as a distributed network security monitor to perform +analysis across many dozens or hundreds of workers, all acting on a small +fraction of the overall traffic volume. The analysis of the worker process is +further facilitated by nodes such as manager and proxies, ultimately logging +the alerts and or relevant network logs. A Zeek cluster therefore consists of +four main components: a manager, workers, proxies, and a logger. + +Manager +------- + +The manager is a Zeek process that has two primary jobs. It normally receives +log messages and notices from the rest of the nodes in the cluster using the +Zeek communications protocol. It combines the individual logs that each worker +produces, so that the result is a set of joint logs instead of many discrete +logs that you would have to combine in some manner with post-processing. (Note +that if you use a separate logger node, then the logger receives all logs +instead of the manager.) The manager also supports other functionality and +analysis which requires a centralized, global view of events or data. + +Worker +------ + +The worker is the Zeek process that sniffs network traffic and does protocol +analysis on the reassembled traffic streams. Most of the work of an active +cluster takes place on the workers. Workers typically represent the bulk of the +Zeek processes that are running in a cluster. The fastest memory and CPU core +speed you can afford is recommended since all of the protocol parsing and most +analysis will take place here. There are no particular requirements for the +disks in workers since almost all logging is done remotely to the manager (or +dedicated logger). Normally, very little is written to disk. + +Proxy +----- + +A proxy is a Zeek process that may be used to offload data storage or any +arbitrary workload. A cluster may contain multiple proxy nodes. +Zeek's default scripts make only minimal use of proxies. +Custom scripts or third-party packages may exercise proxies more heavily +to partition data or workloads, providing greater cluster scalability potential. +The number of required proxy nodes in a cluster depends on the deployed scripts, +cluster size and traffic characteristics. For small clusters with four or less workers, +a single proxy node is usually sufficient. For larger clusters, you may want to +closely monitor :ref:`CPU and memory usage ` of proxy +nodes and increase their number as needed. + +Zeek processes acting as proxies don’t tend to be extremely hard on CPU or +memory, and users frequently run proxy processes on the same physical host as +the manager. + +Logger +------ + +A logger is an optional Zeek process that receives log messages from the rest +of the nodes in the cluster using the Zeek communications protocol. The purpose +of having a logger to receive logs instead of the manager is to reduce the load +on the manager. If no logger is needed, then the manager will receive logs +instead. + +Running a Zeek Cluster +====================== + +Zeek Cluster Setup +------------------ + +This :ref:`link ` describes the cluster setup in great +detail. + +General Usage and Deployment +---------------------------- + +The biggest advantage to using a Zeek cluster is that most of its inner +workings are transparent to the user. Clusterization is a clever trick to +divide-and-conquer ever increasing network traffic volume. + +As a practitioner one must know how to set up a cluster by defining components +such as the manager, proxies, loggers and workers in the +:samp:`{}/etc/node.cfg` file on the manager. + +Edit the ZeekControl node configuration file, :samp:`{}/etc/node.cfg`, +to define where the logger, manager, proxies, and workers will run. For a +cluster configuration, comment-out (or remove) the standalone node in that +file, and either uncomment or add node entries for each node in your cluster +(logger, manager, proxy, and workers). + +For example, to run five Zeek nodes (two workers, one proxy, a logger, and a +manager) on a cluster consisting of three machines, the cluster configuration +would look like this:: + + [logger] + type=logger + host=10.0.0.10 + + [manager] + type=manager + host=10.0.0.10 + + [proxy-1] + type=proxy + host=10.0.0.10 + + [worker-1] + type=worker + host=10.0.0.11 + interface=eth0 + + [worker-2] + type=worker + host=10.0.0.12 + interface=eth0 + + +To set up a cluster we need a network-aggregator/load balancing device which +can aggregate inputs from network sources, such as taps or span ports. This +device also performs the critical function of ensuring each TCP session is +distributed to a single link. This function is provided through symmetric +hashing. + +Once the tap aggregator is set, output from each port is sent to a “Zeek node” +which is typically built on commodity hardware. Zeek clusters have evolved from +running the manager, workers and proxies on individual servers, to most often +now running a “cluster-in-a-box” setup, where a powerful multi-core box with +dedicated cores hosts the workers, proxies logger and manager. We’ve seen +instances of 90 workers running on a single physical server. + +At present the preferred way to run a cluster is to use a load-balancing +network card such as Myricom NICs or Intel cards with PF_RING or AF_PACKET +support. The NIC (and associated software) further divides the traffic to +multiple Zeek worker processes running on the ‘Zeek- node’. + +While the Zeek cluster allows us to monitor traffic at scale, an optional +add-on technology called “shunting” is helpful to reduce the volume that needs +be processed.. Shunting can detect specific large data flows based on +predetermined characteristics and communicate with the network tap via an API +to stop sending those flows to Zeek for analysis. This allows Zeek to maintain +awareness and logs of these shunted large flows while dramatically reducing the +analysis load necessary to process traffic. + +The following links gives more specific information on how to set up +clusterization using one of the above approaches: :ref:`cluster-configuration`. + +Developing Scripts/Heuristics +============================= + +This section is for developers who are interested in writing +packages/scripts/heuristics and want to take advantage of clusterization. + +In order to make your scripts/packages “clusterized,” one must understand the +purpose of each of the cluster components (manager, workers, proxies and +logger) and how/where the data is generated and how to move data/information +across the different nodes in the cluster. + +* **Workers**: Workers are a good first choice for doing the brunt of any work. + They should be spending a lot of time parsing or analyzing incoming data from + packets. You might choose them to do a “first pass” analysis and then decide + how the results should be shared with other nodes in the cluster. + +* **Proxies**: Proxies serve as intermediaries for data storage and computation + offloading. Proxies help offload work or data in a scalable and distributed + way. Since any given worker is connected to all proxies and can agree on an + “arbitrary key -> proxy node” mapping (discussed later), you can partition + work or data amongst them in a uniform manner. You might choose to use + proxies as a method to share non-persistent state or as a “second pass” + analysis for any work that you don’t want interfering with the workers’ + capacity to keep up with capturing and parsing packets. The default scripts + that come with Zeek make minimal use of proxies. If you are migrating from a + previous ZeekControl deployment, you may want to implement a single proxy + node. If you have custom or community scripts that utilize proxies, + considering scaling up the number of proxies to meet demand. + +* **Manager**: A manager will make decisions that require a global view, as it + is in a centralized location and connected to everything. However, that + connectivity also makes it easy to overload it. Try to use a manager + sparingly and only for tasks that must be done in a centralized or + authoritative location. Optionally, for some deployments, the manager can + also serve as the sole logger. + +* **Loggers**: Loggers should simply write logs to disk. In the default cluster + configuration, log content gets distributed among available loggers in a + round-robin fashion, providing failover capability should any given logger + temporarily go offline. + +The Need to Move Data and Events Across Different Nodes +------------------------------------------------------- + +Imagine you have a list of IP addresses that you want to distribute across all +workers to keep in a watch list, such as the Intel framework. You may also want +to aggregate results across workers to see if that count crosses a threshold, +such as using scan detection. Finally, you might want to extract URLs from +emails and then redistribute the extracted URLs to all workers to be able to +find which of these extracted URLs got clicked on. All these examples tend to +introduce challenges in a Zeek cluster setup due to data centrality issues. In +other words, the very advantageous divide-and-conquer approach of +clusterization also introduces complexity in Zeek scripts. However, with the +introduction of the Broker communication framework and additional helper +functions, data centrality complexities can be addressed efficiently. One must +rely on clusterization techniques provided by Zeek scripting, the Broker API, +and clusterization components. + +When clustering your scripts, the fundamental work to move data or events in +the context of a cluster falls primarily on few high level abstractions of +communication patterns: + + 1. Manager-to-worker + 2. Worker-to-manager + 3. Worker-to-proxy + 4. Worker-to-manager-to-worker + 5. Manager-to-worker-to-manager + +All the communication between workers, proxies and manager is established by +Zeek via the Broker framework. The Broker framework provides basic facilities +for connecting Zeek instances to each other and exchanging messages, events or +data. + +Cluster Topics +-------------- + +All Broker-based messaging involves two components: the information you want to +send, such as an event with its arguments, along with an associated topic name +string. The topic strings are used as a filtering mechanism: Broker uses a +publish-subscribe communication pattern where peers advertise interest in topic +prefixes and only receive messages which match one of their prefix +subscriptions. Broker itself supports arbitrary topic strings. However, Zeek +generally follows certain conventions in choosing these topics to help avoid +conflicts and generally make them easier to remember. + +To communicate between workers, proxies and manager one needs to know the topic +name to which all workers, proxies and manager are subscribed to. These are: + + 1. :zeek:see:`Cluster::worker_topic` - to which all workers are subscribed + 2. :zeek:see:`Cluster::proxy_topic` - to which all proxies are subscribed + 3. :zeek:see:`Cluster::manager_topic` - to which manager is subscribed + + +The following table illustrates all the topics and communication events for +clusterization, along with potential use cases: + +.. list-table:: + :header-rows: 1 + + * - Event + - Topic + - Use cases + + * - Manager to worker + - :zeek:see:`Cluster::worker_topic` + - * Read input file on manager + * Distribute data and events from manager to workers + + * - Worker to manager + - :zeek:see:`Cluster::manager_topic` + - * Find characteristics of a “scan” eg. SYN-only pkts + * Send data to manager for aggregation + + * - Worker or manager to proxy + - :zeek:see:`Cluster::proxy_topic` + - * Run operation on all proxies + * Disseminate notice suppression + + * - Worker to manager to worker + - :zeek:see:`Cluster::manager_topic` + :zeek:see:`Cluster::worker_topic` + - * Find URLs in emails + * Send to manager + * Distribute to workers to check against HTTP GET requests + + * - Manager to worker to manager + - :zeek:see:`Cluster::worker_topic` + :zeek:see:`Cluster::manager_topic` + - * Read input file on manager + * Distribute data to workers + * Workers to report counts of connections to manager + * Aggregate the counts on manager + +Cluster Pools +------------- + +In addition to topics, Zeek nodes can join a :zeek:see:`Cluster::Pool`. +Using :zeek:see:`Cluster::publish_hrw` and :zeek:see:`Cluster::publish_rr`, +pools allow to publish events to individual proxies without prior knowledge +of a cluster's shape and size. + +A popular pool is the :zeek:see:`Cluster::proxy_pool`. It comprises all +the proxies of a cluster. Examples of its use are listed in the following table. + + +.. list-table:: + :header-rows: 1 + + * - Event + - Pool + - Use cases + + * - Workers to individual proxy processes + - :zeek:see:`Cluster::proxy_pool` + - * Aggregation based on Highest Random Weight (eg. DNS query types, see the :ref:`section below ` for details.) + * Aggregation of Software versions for a given host + * Offloading tasks in round-robin fashion across proxies + + +Publishing Events Across the Cluster +------------------------------------ + +Broker, as well as Zeek’s higher-level cluster framework, provide a set of +function to publish events, including: + +.. list-table:: + :header-rows: 1 + + * - Function + - Description + - Use + + * - :zeek:see:`Cluster::publish` + - Publishes an event at a given topic + - Standard function to send an event to all nodes subscribed to a given + topic. + + * - :zeek:see:`Cluster::publish_hrw` + - Publishes an event to a node within a pool according to + Highest Random Weight (HRW) hashing strategy; see details below + - Use this in cases of any aggregation needs - eg. scan detection or + anything that needs a counter going. + + * - :zeek:see:`Cluster::publish_rr` + - Publishes an event to a node within a pool according to Round-Robin + distribution strategy. + - Generally used inside Zeek for multiple logger nodes. + + * - :zeek:see:`Broker::publish` + - Publishes an event at a given topic + - Standard function to send an event to all nodes subscribed to a given + topic. + + Starting with Zeek 7.1, this function should only be used in + Broker-specific scripts. Use :zeek:see:`Cluster::publish` otherwise. + + +.. note:: + + The ``Cluster::publish`` function was added in Zeek 7.1. In contrast to + ``Broker:publish``, it publishes events even when a non-Broker cluster + backend is in use. Going forward, ``Cluster:publish`` should be preferred + over ``Broker::publish``, unless the script is specific to the Broker backend, + e.g. when interacting with an external application using native Python + bindings for Broker. + + +An example sending an event from worker to manager: + +.. code-block:: zeek + + event worker_to_manager(worker_name: string) + { + print "got event from worker", worker_name; + } + + event some_event_handled_on_worker() + { + Broker::publish(Cluster::manager_topic, worker_to_manager, + Cluster::node); + } + +More details and code snippets and documentation on Broker communication +frameworks are available at :ref:`broker-framework`. + + +.. _cluster-framework-proxies-uniform: + +Distributing Events Uniformly Across Proxies +-------------------------------------------- + +If you want to offload some data/work from a worker to your proxies, we can +make use of a `Highest Random Weight (HRW) hashing +`_ distribution strategy to +uniformly map an arbitrary key space across all available proxies through +:zeek:see:`Cluster::publish_hrw`. This function publishes an event to one node +within a pool according to a Highest Random Weight hashing strategy. By +assigning :zeek:see:`Cluster::proxy_pool` to this event, one can utilize +proxies to handle it. Note that :zeek:see:`Cluster::publish_hrw` requires a +unique key as an input to the hashing function to uniformly distribute keys +among available nodes. Often this key is a source or destination IP address. If +you are using :zeek:see:`Cluster::publish_hrw` for an aggregate function, such +as counts unique across the workers, make sure to appropriately select the +hashing key. + +The following example illustrates this issue. Assume that we are counting the +number of scanner IPs from each ``/24`` subnet. If the key were the source IP, +then depending on the hashing, different IP addresses from the same ``/24`` +might end up on different proxies for the aggregation function. In this case +one might instead want to use a more inclusive hashing key, such as the subnet +(``/24``) itself. To illustrate the issue, in the notice log below, you see +that 3 scanners each from ``52.100.165.0/24`` went to ``proxy-1`` and +``proxy-2``. Ideally we want a single count of 6 scanners instead. + +:: + + 1600212249.061779 Scan::Subnet 52.100.165.0/24 has 3 spf IPs originating from it 52.100.165.249 52.100.165.237 52.100.165.246 - 52.100.165.246 - - proxy-2 Notice::ACTION_LOG 3600.000000 F + + 1600212293.581745 Scan::Subnet 52.100.165.0/24 has 3 spf IPs originating from it 52.100.165.247 52.100.165.244 52.100.165.205 - 52.100.165.205 - - proxy-1 Notice::ACTION_LOG 3600.000000 + +Instead, we can ensure the hash key is ``52.100.165.0/24`` instead of the +original IP, as the hash for ``52.100.165.0/24`` will be the same for all +addresses belonging to this subnet. Then the data will reach only one proxy. +To that end, we can use the ``mask_address`` function to extract subnet +information for a given IP address to use as a key in the hash function: + +.. code-block:: zeek + + local spf = mask_address(orig); + + @if ( Cluster::is_enabled()) + Cluster::publish_hrw(Cluster::proxy_pool, spf, smtpsink::aggregate_stats, c) ; + @else + event smtpsink::aggregate_stats(c); + @endif + +Carefully select the key for :zeek:see:`Cluster::publish_hrw`. If done right, +this feature will bring tremendous benefits in code scalability, especially +when working with aggregate and threshold functions. + +.. note:: + + In scripting for clusterization, using the correct module names and + namespaces is crucial as both events and data are transmitted to different + systems. In order to make sure the contexts are correct, all functions, + events and datasets should be scoped within their respective namespaces and + modules. An easy rule of thumb is to always use the explicit module namespace + scoping. See :ref:`event-namespacing-pitfall` for further explanation and + examples. + +Clusterization of Zeek scripts can be an intimidating task for beginners. +However, with reliance on the new Broker framework, clusterization has become +simpler and straightforward. Consider the following: + +1. Communication overhead: Be sure not to generate unnecessary communication + overhead. For example, scan detection is one of the worst cases for + distributed analysis. One needs to count connections from a given IP address + across all workers and then aggregate them on a proxy or manager. All the + connections have to reach an aggregate function before Zeek can determine if + a given source is a scanner or not. This happens because each worker only + has a limited picture of the activity generated by a given remote IP. + +2. Communication optimizations: Once a given remote IP is identified as + desired, make sure a manager reports that to the worker, and workers stop + sending any further data for that IP to the manager. This is especially + useful in scan detection where it takes only a few connections to identify + scans, while a given scanner might send millions of probes eventually. If + done right, workers will only send the first N connections, and stop after + that, thus saving a lot of communication overheads. However, it makes sense + to stop workers from sending any further connection information + +3. Clusterization also requires timely state synchronization across the + workers, to make sure that all workers have a common view of a particular + heuristic. + +4. When writing scripts for clusterization make sure your detection runs in + both cluster and standalone setup. + +A Cluster Script Walkthrough +---------------------------- + +Let's say we want to count how many connections a remote IP is making to a host +in our network on port 3389 UDP. Due to the distributed nature of Zeek +clusters, connections are distributed across the workers based on a 5-tuple +hash (source IP, source port, destination IP, destination port, and protocol). +To get a central view of a connection between a given IP pair, one must deploy +a clusterized scripting approach. The following example highlights how to go +about doing so. + +In this use case, we intend to create an aggregation function. +:zeek:see:`Cluster::publish_hrw` appears to be the appropriate function, since +it allows offloading a lot of work to proxies, thus leaving workers and manager +to process traffic. + +In order to make sure all the connections between two hosts go to a single +specific proxy, we need to make sure the key for the hashing function +accommodates this constraint. We will use ``orig_h+resp_h`` as the key. We +create a new data-type called ``pair`` as seen in code below. This allows us +to use the ``orig+resp`` as a unique key across the code, including in the +candidate table. Further, we create a new data type called ``stats`` to keep +track of additional data associated with a connection pair. + +.. code-block:: zeek + + module DoS; + + export { + + redef enum Notice::Type += { + Threshold, + Victim_3389UDP, + }; + + type pair: record { + orig: addr; + resp: addr; + }; + + type stats: record { + orig: addr; + resp: addr ; + orig_bytes: count &default=0; + resp_bytes: count &default=0; + conns: count &default=0; + }; + + global dos_candidates: table [pair] of stats &create_expire=1 day; + + global DoS::aggregate_stats:event(s: stats); + } + +We choose the :zeek:see:`connection_state_remove` event as the primary event to +tap into. :zeek:see:`connection_state_remove` is generated when a connection’s +internal state is about to be removed from memory. It's appropriate for this +case, as all the information about the connection is now included in the +:zeek:see:`connection` record ``c``. One disadvantage of using +:zeek:see:`connection_state_remove` is that the event is fired at the very end +of the connection, after the expiration timeouts are over. Thus, there are +delays, and any operation which happens on the data is “after-the-fact” that +connection is over. While this could be a problem in approaches such as +proactive blocking and early detection heuristics, in this case of aggregation +it is not an issue. + +The thing to pay attention to in the code snippet below is the +:zeek:see:`@if`-:zeek:see:`@else`-:zeek:see:`@endif` directives which +differentiate between clusterized and standalone operation of the script. With +the :zeek:see:`@if` construct, the specified expression must evaluate to type +bool. If the value is true, then the following script lines (up to the next +:zeek:see:`@else` or :zeek:see:`@endif`) are available to be executed. In this +case we check if :zeek:see:`Cluster::is_enabled`. If so, we call +:zeek:see:`Cluster::publish_hrw` along with the key (``hash_pair``) and the +aggregate function followed by parameters, which is the stats record in this +case. If the cluster isn’t running that aggregate function, it is directly +called. + +.. code-block:: zeek + + event connection_state_remove(c: connection) + { + local service = c$id$resp_p; + local resp = c$id$resp_h; + + if ( service != 3389/udp ) + return; + + if ( resp !in Site::local_nets ) + return; + + local s: stats; + s$orig = c$id$orig_h; + s$resp = c$id$resp_h; + s$orig_bytes = c$conn$orig_ip_bytes; + s$resp_bytes = c$conn$resp_ip_bytes; + + local hash_pair: pair; + hash_pair$orig = c$id$orig_h; + hash_pair$resp = resp; + + @if ( Cluster::is_enabled() ) + Cluster::publish_hrw(Cluster::proxy_pool, hash_pair, DoS::aggregate_stats, s); + @else + event DoS::aggregate_stats(s); + @endif + } + +Since ``hash_pair`` makes the key unique, irrespective of what worker this +specific connection has gone to, it will end up on a one specific proxy only. + +.. code-block:: zeek + + event DoS::aggregate_stats(s: stats) + { + local p: pair ; + p$orig = s$orig; + p$resp = s$resp ; + + if ( p !in dos_candidates ) + { + local tmp_s: stats; + tmp_s$orig = s$orig; + tmp_s$resp = s$resp; + tmp_s$orig_bytes = 0; + tmp_s$resp_bytes= 0; + tmp_s$conns = 0; + + dos_candidates[p] = tmp_s; + } + + dos_candidates[p]$conns += 1; + dos_candidates[p]$orig_bytes += s$orig_bytes; + dos_candidates[p]$resp_bytes += s$resp_bytes; + + local n = dos_candidates[p]$conns; + + local thresh = check_ip_threshold(dos_threshold, ip_pair_threshold_idx, p, n); + + if ( thresh ) + { + local msg = fmt("%s pair has reached %s threshold %s", + p, n, dos_candidates[p]); + NOTICE([$note=DoS::Threshold, $src=p$orig, $msg=msg]); + + if ( dos_candidates[p]$resp_bytes > 0 ) + NOTICE([$note=DoS::Victim, $src=p$orig, $msg=msg, + $identifier=cat(p$resp), $suppress_for=1 hrs]); + } + } diff --git a/doc/frameworks/configuration.rst b/doc/frameworks/configuration.rst new file mode 100644 index 0000000000..91492b231b --- /dev/null +++ b/doc/frameworks/configuration.rst @@ -0,0 +1,356 @@ + +.. _framework-configuration: + +======================= +Configuration Framework +======================= + +Zeek includes a configuration framework that allows updating script options at +runtime. This functionality consists of an :zeek:see:`option` declaration in +the Zeek language, configuration files that enable changing the value of +options at runtime, option-change callbacks to process updates in your Zeek +scripts, a couple of script-level functions to manage config settings directly, +and a log file (:file:`config.log`) that contains information about every +option value change according to :zeek:see:`Config::Info`. + +Introduction +============ + +The configuration framework provides an alternative to using Zeek script +constants to store various Zeek settings. + +While traditional constants work well when a value is not expected to change at +runtime, they cannot be used for values that need to be modified occasionally. +While a :zeek:see:`redef` allows a re-definition of an already defined constant +in Zeek, these redefinitions can only be performed when Zeek first starts. +Afterwards, constants can no longer be modified. + +However, it is clearly desirable to be able to change at runtime many of the +configuration options that Zeek offers. Restarting Zeek can be time-consuming +and causes it to lose all connection state and knowledge that it accumulated. +Zeek’s configuration framework solves this problem. + +Declaring Options +================= + +The :zeek:see:`option` keyword allows variables to be declared as configuration +options: + +.. code-block:: zeek + + module Test; + + export { + option my_networks: set[subnet] = {}; + option enable_feature = F; + option hostname = "testsystem"; + option timeout_after = 1min; + option my_ports: vector of port = {}; + } + +Options combine aspects of global variables and constants. Like global +variables, options cannot be declared inside a function, hook, or event +handler. Like constants, options must be initialized when declared (the type +can often be inferred from the initializer but may need to be specified when +ambiguous). The value of an option can change at runtime, but options cannot be +assigned a new value using normal assignments. + +The initial value of an option can be redefined with a :zeek:see:`redef` +declaration just like for global variables and constants. However, there is no +need to specify the :zeek:see:`&redef` attribute in the declaration of an +option. For example, given the above option declarations, here are possible +redefs that work anyway: + +.. code-block:: zeek + + redef Test::enable_feature = T; + redef Test::my_networks += { 10.1.0.0/16, 10.2.0.0/16 }; + +Changing Options +================ + +The configuration framework facilitates reading in new option values from +external files at runtime. Configuration files contain a mapping between option +names and their values. Each line contains one option assignment, formatted as +follows:: + + [option name][tab/spaces][new value] + +Lines starting with ``#`` are comments and ignored. + +You register configuration files by adding them to +:zeek:see:`Config::config_files`, a set of filenames. Simply say something like +the following in :file:`local.zeek`: + +.. code-block:: zeek + + redef Config::config_files += { "/path/to/config.dat" }; + +Zeek will then monitor the specified file continuously for changes. For +example, editing a line containing:: + + Test::enable_feature T + +to the config file while Zeek is running will cause it to automatically update +the option’s value in the scripting layer. The next time your code accesses the +option, it will see the new value. + +.. note:: + + The config framework is clusterized. In a cluster configuration, only the + manager node watches the specified configuration files, and relays option + updates across the cluster. + +Config File Formatting +---------------------- + +The formatting of config option values in the config file is not the same as in +Zeek’s scripting language. Keep an eye on the :file:`reporter.log` for warnings +from the config reader in case of incorrectly formatted values, which it’ll +generally ignore when encountered. The following table summarizes supported +types and their value representations: + +.. list-table:: + :header-rows: 1 + + * - Data Type + - Sample Config File Entry + - Comments + + * - :zeek:see:`addr` + - ``1.2.3.4`` + - Plain IPv4 or IPv6 address, as in Zeek. No ``/32`` or similar netmasks. + + * - :zeek:see:`bool` + - ``T`` + - ``T`` or ``1`` for true, ``F`` or ``0`` for false + + * - :zeek:see:`count` + - ``42`` + - Plain, nonnegative integer. + + * - :zeek:see:`double` + - ``-42.5`` + - Plain double number. + + * - :zeek:see:`enum` + - ``Enum::FOO_A`` + - Plain enum string. + + * - :zeek:see:`int` + - ``-1`` + - Plain integer. + + * - :zeek:see:`interval` + - ``3600.0`` + - Always in epoch seconds, with optional fraction of seconds. Never + includes a time unit. + + * - :zeek:see:`pattern` + - ``/(foo|bar)/`` + - The regex pattern, within forward-slash characters. + + * - :zeek:see:`port` + - ``42/tcp`` + - Port number with protocol, as in Zeek. When the protocol part is missing, + Zeek interprets it as ``/unknown``. + + * - :zeek:see:`set` + - ``80/tcp,53/udp`` + - The set members, formatted as per their own type, separated by commas. + For an empty set, use an empty string: just follow the option name with + whitespace. + + Sets with multiple index types (e.g. ``set[addr,string]``) are currently + not supported in config files. + + * - :zeek:see:`string` + - ``Don’t bite, Zeek`` + - Plain string, no quotation marks. Given quotation marks become part of + the string. Everything after the whitespace separator delineating the + option name becomes the string. Saces and special characters are fine. + Backslash characters (e.g. ``\n``) have no special meaning. + + * - :zeek:see:`subnet` + - ``1.2.3.4/16`` + - Plain subnet, as in Zeek. + + * - :zeek:see:`time` + - ``1608164505.5`` + - Always in epoch seconds, with optional fraction of seconds. Never + includes a time unit. + + * - :zeek:see:`vector` + - ``1,2,3,4`` + - The set members, formatted as per their own type, separated by commas. + For an empty vector, use an empty string: just follow the option name + with whitespace. + +This leaves a few data types unsupported, notably tables and records. If you +require these, build up an instance of the corresponding type manually (perhaps +from a separate input framework file) and then call +:zeek:see:`Config::set_value` to update the option: + +.. code-block:: zeek + + module Test; + + export { + option host_port: table[addr] of port = {}; + } + + event zeek_init() { + local t: table[addr] of port = { [10.0.0.2] = 123/tcp }; + Config::set_value("Test::host_port", t); + } + + +Regardless of whether an option change is triggered by a config file or via +explicit :zeek:see:`Config::set_value` calls, Zeek always logs the change to +:file:`config.log`. A sample entry:: + + #fields ts id old_value new_value location + #types time string string string string + 1608167352.498872 Test::a_count 42 3 config.txt + +Mentioning options repeatedly in the config files leads to multiple update +events; the last entry “wins”. Mentioning options that do not correspond to +existing options in the script layer is safe, but triggers warnings in +:file:`reporter.log`:: + + warning: config.txt/Input::READER_CONFIG: Option 'an_unknown' does not exist. Ignoring line. + +Internally, the framework uses the Zeek input framework to learn about config +changes. If you inspect the configuration framework scripts, you will notice +that the scripts simply catch input framework events and call +:zeek:see:`Config::set_value` to set the relevant option to the new value. If +you want to change an option in your scripts at runtime, you can likewise call +:zeek:see:`Config::set_value` directly from a script (in a cluster +configuration, this only needs to happen on the manager, as the change will be +automatically sent to all other nodes in the cluster). + +.. note:: + + The input framework is usually very strict about the syntax of input files, but + that is not the case for configuration files. These require no header lines, + and both tabs and spaces are accepted as separators. A custom input reader, + specifically for reading config files, facilitates this. + +.. tip:: + + The gory details of option-parsing reside in ``Ascii::ParseValue()`` in + :file:`src/threading/formatters/Ascii.cc` and ``Value::ValueToVal`` in + :file:`src/threading/SerialTypes.cc` in the Zeek core. + +Change Handlers +=============== + +A change handler is a user-defined function that Zeek calls each time an option +value changes. This allows you to react programmatically to option changes. The +following example shows how to register a change handler for an option that has +a data type of :zeek:see:`addr` (for other data types, the return type and +second parameter data type must be adjusted accordingly): + +.. code-block:: zeek + + module Test; + + export { + option testaddr = 127.0.0.1; + } + + # Note: the data type of 2nd parameter and return type must match + function change_addr(id: string, new_value: addr): addr + { + print fmt("Value of %s changed from %s to %s", id, testaddr, new_value); + return new_value; + } + + event zeek_init() + { + Option::set_change_handler("Test::testaddr", change_addr); + } + +Immediately before Zeek changes the specified option value, it invokes any +registered change handlers. The value returned by the change handler is the +value Zeek assigns to the option. This allows, for example, checking of values +to reject invalid input (the original value can be returned to override the +change). + +.. note:: + + :zeek:see:`Option::set_change_handler` expects the name of the option to + invoke the change handler for, not the option itself. Also, that name + includes the module name, even when registering from within the module. + +It is possible to define multiple change handlers for a single option. In this +case, the change handlers are chained together: the value returned by the first +change handler is the “new value” seen by the next change handler, and so on. +The built-in function :zeek:see:`Option::set_change_handler` takes an optional +third argument that can specify a priority for the handlers. + +A change handler function can optionally have a third argument of type string. +When a config file triggers a change, then the third argument is the pathname +of the config file. When the :zeek:see:`Config::set_value` function triggers a +change, then the third argument of the change handler is the value passed to +the optional third argument of the :zeek:see:`Config::set_value` function. + +.. tip:: + + Change handlers are also used internally by the configuration framework. If + you look at the script-level source code of the config framework, you can see + that change handlers log the option changes to :file:`config.log`. + +When Change Handlers Trigger +---------------------------- + +Change handlers often implement logic that manages additional internal state. +For example, depending on a performance toggle option, you might initialize or +clean up a caching structure. In such scenarios you need to know exactly when +and whether a handler gets invoked. The following hold: + +* When no config files get registered in :zeek:see:`Config::config_files`, + change handlers do not run. +* When none of any registered config files exist on disk, change handlers do + not run. + +That is, change handlers are tied to config files, and don’t automatically run +with the option’s default values. + +* When a config file exists on disk at Zeek startup, change handlers run with + the file’s config values. +* When the config file contains the same value the option already defaults to, + its change handlers are invoked anyway. +* :zeek:see:`zeek_init` handlers run before any change handlers — i.e., they + run with the options’ default values. +* Since the config framework relies on the input framework, the input + framework’s inherent asynchrony applies: you can’t assume when exactly an + option change manifests in the code. + +If your change handler needs to run consistently at startup and when options +change, you can call the handler manually from :zeek:see:`zeek_init` when you +register it. That way, initialization code always runs for the option’s default +value, and also for any new values. + +.. code-block:: zeek + + module Test; + + export { + option use_cache = T; + } + + function use_cache_hdlr(id: string, new_value: bool): bool + { + if ( new_value ) { + # Ensure caching structures are set up properly + } + + return new_value; + } + + event zeek_init() + { + use_cache_hdlr("Test::use_cache", use_cache); + Option::set_change_handler("Test::use_cache", use_cache_hdlr); + } diff --git a/doc/frameworks/denylist.jsonl b/doc/frameworks/denylist.jsonl new file mode 100644 index 0000000000..1ea6a1851e --- /dev/null +++ b/doc/frameworks/denylist.jsonl @@ -0,0 +1,3 @@ +{"ip": "192.168.17.1", "timestamp": 1333252748, "reason": "Malware host"} +{"ip": "192.168.27.2", "timestamp": 1330235733, "reason": "Botnet server"} +{"ip": "192.168.250.3", "timestamp": 1333145108, "reason": "Virus detected"} diff --git a/doc/frameworks/file-analysis.rst b/doc/frameworks/file-analysis.rst new file mode 100644 index 0000000000..03f152af49 --- /dev/null +++ b/doc/frameworks/file-analysis.rst @@ -0,0 +1,283 @@ + +.. _file-analysis-framework: + +======================= +File Analysis Framework +======================= + +.. TODO: integrate BoZ revisions + +.. rst-class:: opening + + In the past, writing Zeek scripts with the intent of analyzing file + content could be cumbersome because of the fact that the content + would be presented in different ways, via events, at the + script-layer depending on which network protocol was involved in the + file transfer. Scripts written to analyze files over one protocol + would have to be copied and modified to fit other protocols. The + file analysis framework (FAF) instead provides a generalized + presentation of file-related information. The information regarding + the protocol involved in transporting a file over the network is + still available, but it no longer has to dictate how one organizes + their scripting logic to handle it. A goal of the FAF is to + provide analysis specifically for files that is analogous to the + analysis Zeek provides for network connections. + +Supported Protocols +=================== + +Zeek ships with file analysis for the following protocols: +:ref:`FTP `, +:ref:`HTTP `, +:ref:`IRC `, +:ref:`Kerberos `, +:ref:`MIME `, +:ref:`RDP `, +:ref:`SMTP `, and +:ref:`SSL/TLS/DTLS `. +Protocol analyzers are regular :ref:`Zeek plugins `, so users +are welcome to provide additional ones in separate Zeek packages. + +File Lifecycle Events +===================== + +The key events that may occur during the lifetime of a file are: +:zeek:see:`file_new`, :zeek:see:`file_over_new_connection`, +:zeek:see:`file_sniff`, :zeek:see:`file_timeout`, :zeek:see:`file_gap`, and +:zeek:see:`file_state_remove`. Handling any of these events provides +some information about the file such as which network +:zeek:see:`connection` and protocol are transporting the file, how many +bytes have been transferred so far, and its MIME type. + +Here's a simple example: + +.. literalinclude:: file_analysis_01.zeek + :caption: + :language: zeek + :linenos: + :tab-width: 4 + +.. code-block:: console + + $ zeek -r http/get.trace file_analysis_01.zeek + file_state_remove + FakNcS1Jfe01uljb3 + CHhAvVGS1DHFjwGM9 + [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] + HTTP + connection_state_remove + CHhAvVGS1DHFjwGM9 + [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] + HTTP + +This doesn't perform any interesting analysis yet, but does highlight +the similarity between analysis of connections and files. Connections +are identified by the usual 5-tuple or a convenient UID string while +files are identified just by a string of the same format as the +connection UID. So there's unique ways to identify both files and +connections and files hold references to a connection (or connections) +that transported it. + +File Type Identification +======================== + +Zeek ships with its own library of content signatures to determine the type of a +file, conveyed as MIME types in the :zeek:see:`file_sniff` event. You can find +those signatures in the Zeek distribution's ``scripts/base/frameworks/files/magic/`` +directory. (Despite the name, Zeek does *not* rely on libmagic for content analysis.) + +Adding Analysis +=============== + +Zeek supports customized file analysis via *file analyzers* that users can +attach to observed files. You can attach analyzers selectively to individual +files, or register them for auto-attachment under certain conditions. Once +attached, file analyzers start receiving the contents of files as Zeek parses +them from ongoing network connections. + +Zeek comes with the following built-in analyzers: + + * :ref:`plugin-zeek-filedataevent` to access file content via + events (as data streams or content chunks), + * :ref:`plugin-zeek-fileentropy` to compute various entropy for a file, + * :ref:`plugin-zeek-fileextract` to extract files to disk, + * :ref:`plugin-zeek-filehash` to produce common hash values for files, + * :ref:`plugin-zeek-pe` to parse executables in PE format, and + * :ref:`plugin-zeek-x509` to extract information about x509 certificates. + +Like protocol parsers, file analyzers are regular :ref:`Zeek plugins +`. Users are free to contribute additional ones via Zeek +packages. + +Per-file analyzer registration +------------------------------ + +To attach an analyzer to a specific file, call :zeek:see:`Files::add_analyzer` +with the analyzer's component tag (such as :zeek:see:`Files::ANALYZER_MD5`; +consult the above analyzers for details). Some file analyzers support parameters +that you can provide to this function via a :zeek:see:`Files::AnalyzerArgs` +record, while others introduce additional event types and tunable script-layer +settings. + +You can add multiple analyzers to a file, and add the same analyzer type +multiple times, assuming you use varying :zeek:see:`Files::AnalyzerArgs` +parameterization. You may remove these selectively from files via calls to +:zeek:see:`Files::remove_analyzer`. You may also enable and disable file +analyzers globally by calling :zeek:see:`Files::enable_analyzer` and +:zeek:see:`Files::disable_analyzer`, respectively. + +Generic analyzer registration +----------------------------- + +The framework provides mechanisms for automatically attaching analyzers to +files. For example, the :zeek:see:`Files::register_for_mime_types` function +ensures that Zeek automatically attaches a given analyzer to all files of a +given MIME type. For fully customized auto-attachment logic take a look at +:zeek:see:`Files::register_analyzer_add_callback`, and refer to +:doc:`base/frameworks/files/main.zeek ` +for additional APIs and data structures. + +Regardless of which file analyzers end up acting on a file, general +information about the file (e.g. size, time of last data transferred, +MIME type, etc.) is logged in :file:`files.log`. + +Protocol-specific state +----------------------- + +Some protocol analyzers redefine the ``fa_file`` record to add additional +state. For example, ``base/protocols/http/entities.zeek``, which Zeek loads by +default as part of the HTTP analyzer, makes the transaction's +:zeek:see:`HTTP::Info` record available via ``f$http`` to provide HTTP +context. As always, make sure to test the presence of optional fields via the +``a?$b`` :ref:`record field operator ` before accessing +them. + +Examples +-------- + +File hashing +^^^^^^^^^^^^ + +The following script uses the MD5 file analyzer to calculate the hashes of plain +text files: + +.. literalinclude:: file_analysis_02.zeek + :caption: + :language: zeek + :tab-width: 4 + +.. code-block:: console + + $ zeek -r http/get.trace file_analysis_02.zeek + new file, FakNcS1Jfe01uljb3 + file_hash, FakNcS1Jfe01uljb3, md5, 397168fd09991a0e712254df7bc639ac + +File extraction +^^^^^^^^^^^^^^^ + +The following example sets up extraction of observed files to disk: + +.. code-block:: zeek + + global idx: count = 0; + + event file_new(f: fa_file) + { + Files::add_analyzer(f, Files::ANALYZER_EXTRACT, + [$extract_filename=fmt("file-%04d", ++idx)]); + } + +The file extraction analyzer now writes the content of each observed file to a +separate file on disk. The output file name results from concatenating the +:zeek:see:`FileExtract::prefix` (normally ``./extract_files/``) and the +enumerated ``file-NNNN`` strings. + +In a production setting you'll likely want to include additional information in +the output, for example from state attached to the provided file record. The +Zeek distribution ships with a starting point for such approaches: the +``policy/frameworks/files/extract-all-files.zeek`` script. For additional +configurability, take a look at the `file-extraction +`_ Zeek package. + +Script-level content analysis +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``FileDataEvent`` analyzer provides script-layer access to file content for +customized analysis. Since observed files can be very large, Zeek cannot buffer +these files and provide their entire content to the script layer once +complete. Instead, the ``FileDataEvent`` analyzer reflects the incremental +nature of file content as Zeek observes it, and supports two types of events to +allow you to process it: user-provided *stream events* receive new file content +as supplied by connection-oriented protocols, while *chunk events* receive +observed data as provided by protocols that do not feature stream semantics. + +The following example manually computes the SHA256 hash of each observed file by +building up hash state and feeding streamed file content into the hash +computation. When Zeek removes a file's state (because it has fully observed it, +or perhaps because its state is timing out), it prints the resulting hash to the +console: + +.. code-block:: zeek + + global hashstate: table[string] of opaque of sha256; + + event file_stream(f: fa_file, data: string) + { + if ( f$id !in hashstate ) + hashstate[f$id] = sha256_hash_init(); + + sha256_hash_update(hashstate[f$id], data); + } + + event file_new(f: fa_file) + { + Files::add_analyzer(f, Files::ANALYZER_DATA_EVENT, [$stream_event=file_stream]); + } + + event file_state_remove(f: fa_file) + { + if ( f$id in hashstate ) + { + print(sha256_hash_finish(hashstate[f$id])); + delete hashstate[f$id]; + } + } + +Be careful with this approach, as it can quickly prove expensive to route all +file content through the script layer. Make sure to add the analyzer only for +relevant files, and consider removing it via :zeek:see:`Files::remove_analyzer` +when you no longer require content analysis. For performance-critical +applications a new file analyzer plugin could be a better approach. + +Input Framework Integration +=========================== + +The FAF comes with a simple way to integrate with the :doc:`Input +Framework `, so that Zeek can analyze files from external sources +in the same way it analyzes files that it sees coming over traffic from +a network interface it's monitoring. It only requires a call to +:zeek:see:`Input::add_analysis`: + +.. literalinclude:: file_analysis_03.zeek + :caption: + :language: zeek + :linenos: + :tab-width: 4 + +Note that the "source" field of :zeek:see:`fa_file` corresponds to the +"name" field of :zeek:see:`Input::AnalysisDescription` since that is what +the input framework uses to uniquely identify an input stream. + +Example output of the above script may be: + +.. code-block:: console + + $ echo "Hello world" > myfile + $ zeek file_analysis_03.zeek + new file, FZedLu4Ajcvge02jA8 + file_hash, FZedLu4Ajcvge02jA8, md5, f0ef7081e1539ac00ef5b761b4fb01b3 + file_state_remove + +Nothing that special, but it at least verifies the MD5 file analyzer +saw all the bytes of the input file and calculated the checksum +correctly! diff --git a/doc/frameworks/file_analysis_01.zeek b/doc/frameworks/file_analysis_01.zeek new file mode 100644 index 0000000000..a48f8184ad --- /dev/null +++ b/doc/frameworks/file_analysis_01.zeek @@ -0,0 +1,20 @@ +event connection_state_remove(c: connection) + { + print "connection_state_remove"; + print c$uid; + print c$id; + for ( s in c$service ) + print s; + } + +event file_state_remove(f: fa_file) + { + print "file_state_remove"; + print f$id; + for ( cid in f$conns ) + { + print f$conns[cid]$uid; + print cid; + } + print f$source; + } diff --git a/doc/frameworks/file_analysis_02.zeek b/doc/frameworks/file_analysis_02.zeek new file mode 100644 index 0000000000..fd4f0e775e --- /dev/null +++ b/doc/frameworks/file_analysis_02.zeek @@ -0,0 +1,12 @@ +event file_sniff(f: fa_file, meta: fa_metadata) + { + if ( ! meta?$mime_type ) return; + print "new file", f$id; + if ( meta$mime_type == "text/plain" ) + Files::add_analyzer(f, Files::ANALYZER_MD5); + } + +event file_hash(f: fa_file, kind: string, hash: string) + { + print "file_hash", f$id, kind, hash; + } diff --git a/doc/frameworks/file_analysis_03.zeek b/doc/frameworks/file_analysis_03.zeek new file mode 100644 index 0000000000..3f8aa35d31 --- /dev/null +++ b/doc/frameworks/file_analysis_03.zeek @@ -0,0 +1,25 @@ +redef exit_only_after_terminate = T; + +event file_new(f: fa_file) + { + print "new file", f$id; + Files::add_analyzer(f, Files::ANALYZER_MD5); + } + +event file_state_remove(f: fa_file) + { + print "file_state_remove"; + Input::remove(f$source); + terminate(); + } + +event file_hash(f: fa_file, kind: string, hash: string) + { + print "file_hash", f$id, kind, hash; + } + +event zeek_init() + { + local source: string = "./myfile"; + Input::add_analysis([$source=source, $name=source]); + } diff --git a/doc/frameworks/index.rst b/doc/frameworks/index.rst new file mode 100644 index 0000000000..a9ef76f884 --- /dev/null +++ b/doc/frameworks/index.rst @@ -0,0 +1,38 @@ + +========== +Frameworks +========== + +Zeek includes several software frameworks that provide commonly used +functionality to the scripting layer. Among other things, these frameworks +enhance Zeek’s ability to ingest data, structure and filter its outputs, adapt +settings at runtime, and interact with other components in your network. Most +frameworks include functionality implemented in Zeek’s core, with +corresponding data structures and APIs exposed to the script layer. + +Some frameworks target relatively specific use cases, while others run in +nearly every Zeek installation. The logging framework, for example, provides +the machinery behind all of the Zeek logs covered earlier. Frameworks also +build on each other, so it’s well worth knowing their capabilities. The next +sections cover them in detail. + +.. toctree:: + :maxdepth: 1 + + broker + cluster + configuration + file-analysis + input + intel + logging + management + netcontrol + notice + packet-analysis + signatures + storage + sumstats + supervisor + telemetry + tls-decryption diff --git a/doc/frameworks/input.rst b/doc/frameworks/input.rst new file mode 100644 index 0000000000..994eec01eb --- /dev/null +++ b/doc/frameworks/input.rst @@ -0,0 +1,640 @@ + +.. _framework-input: + +=============== +Input Framework +=============== + +Zeek features a flexible input framework that allows users to import arbitrary +data into Zeek. Data is either read into Zeek tables or directly converted to +events for scripts to handle as they see fit. A modular reader architecture +allows reading from files, databases, or other data sources. + +This chapter gives an overview of how to use the input framework, with +examples. For more complex scenarios take a look at the test cases in +:file:`testing/btest/scripts/base/frameworks/input/` in the Zeek distribution. + +.. note:: + + The input framework has no awareness of Zeek’s cluster architecture. Zeek + supports all of the mechanisms covered below on any cluster node. The config + and intelligence frameworks both leverage the input framework, adding logic + that applies the input framework on the manager node, distributing ingested + information across the cluster via events. + +Reading Data into Tables +======================== + +Probably the most interesting use-case of the input framework is to read data +into a Zeek table. By default, the input framework reads the data in the same +format as it is written by Zeek’s logging framework: a tab-separated ASCII +file. + +We will show the ways to read files into Zeek with a simple example. For this +example we assume that we want to import data from a denylist that contains +server IP addresses as well as the timestamp and the reason for the block. + +An example input file could look like this (note that all fields must be +tab-separated):: + + #fields ip timestamp reason + 192.168.17.1 1333252748 Malware host + 192.168.27.2 1330235733 Botnet server + 192.168.250.3 1333145108 Virus detected + +To read a file into a Zeek table, two record types have to be defined. One +contains the types and names of the columns that should constitute the table +keys, and the second contains the types and names of the columns that should +constitute the table values. + +In our case, we want to be able to look up IPs. Hence, our key record only +contains the server IP. All other elements should be stored as the table +content. + +.. code-block:: zeek + + type Idx: record { + ip: addr; + }; + + type Val: record { + timestamp: time; + reason: string; + }; + +Note that the names of the fields in the record definitions must correspond to +the column names listed in the ``#fields`` line of the input file, in this case +``ip``, ``timestamp``, and ``reason``. Also note that the ordering of the +columns does not matter, because each column is identified by name. + +The input file is read into the table with a call of the +:zeek:see:`Input::add_table` function: + +.. code-block:: zeek + + global denylist: table[addr] of Val = table(); + + event zeek_init() { + Input::add_table([$source="denylist.file", $name="denylist", + $idx=Idx, $val=Val, $destination=denylist]); + Input::remove("denylist"); + } + +With these three lines we first create an empty table that should receive the +denylist data and then instruct the input framework to open an input stream +named “denylist” to read the data into the table. The third line removes the +input stream again, because we do not need it any more after the data has been +read. + +Note that while the key and content records may use :zeek:attr:`&optional` +fields, omitting columns (usually via the "-" character) requires care. Since +the key record's columns expand into a list of values for indexing into the +receiving table (note how in the above example ``denylist`` is indexed via a +plain ``addr``) and all of those values must be present for indexing, you cannot +in practice omit these values. For content records, omitting is meaningful, but +only permitted for columns with the :zeek:attr:`&optional` attribute. The +framework skips offending input lines with a warning. + +.. note:: + + Prior to version 4.1 Zeek accepted such inputs, unsafely. When transitioning + from such versions to Zeek 4.1 or newer, users with omitted fields in their + input data may observe discrepancies in the loaded data sets. + +Asynchronous processing +----------------------- + +Since some data files might be rather large, the input framework works +asynchronously. A new thread is created for each new input stream. This thread +opens the input data file, converts the data into an internal format and sends +it back to the main Zeek thread. Because of this, the data is not immediately +accessible. Depending on the size of the data source it might take from a few +milliseconds up to a few seconds until all data is present in the table. Please +note that this means that when Zeek is running without an input source or on +very short captured files, it might terminate before the data is present in the +table (because Zeek already handled all packets before the import thread +finished). + +Subsequent calls to an input source are queued until the previous action has +been completed. Because of this it is, for example, possible to call +:zeek:see:`Input::add_table` and :zeek:see:`Input::remove` in two subsequent +lines: the remove action will remain queued until the first read has been +completed. + +Once the input framework finishes reading from a data source, it fires the +:zeek:see:`Input::end_of_data` event. Once this event has been received all +data from the input file is available in the table. + +.. code-block:: zeek + + event Input::end_of_data(name: string, source: string) { + # now all data is in the table + print denylist; + } + +The table can be used while the data is still being read — it just might not +contain all lines from the input file before the event has fired. After the +table has been populated it can be used like any other Zeek table and denylist +entries can easily be tested: + +.. code-block:: zeek + + if ( 192.168.18.12 in denylist ) + # take action + + +Sets instead of tables +---------------------- + +For some use cases the key/value notion that drives tabular data does not +apply, for example when the main purpose of the data is to test for membership +in a set. The input framework supports this approach by using sets as the +destination data type, and omitting ``$val`` in :zeek:see:`Input::add_table`: + +.. code-block:: zeek + + type Idx: record { + ip: addr; + }; + + global denylist: set[addr] = set(); + + event zeek_init() { + Input::add_table([$source="denylist.file", $name="denylist", + $idx=Idx, $destination=denylist]); + Input::remove("denylist"); + } + +Re-reading and streaming data +----------------------------- + +For some data sources (such as many denylists), the input data changes +continually. The input framework supports additional techniques to manage such +ever-changing input. + +The first, very basic method is an explicit refresh of an input stream. When an +input stream is open (meaning it has not yet been removed by a call to +:zeek:see:`Input::remove`), the function :zeek:see:`Input::force_update` can be +called. This will trigger a complete refresh of the table: any changed elements +from the file will be updated, new ones added, and any elements no longer in +the input data get removed. After the update is finished the +:zeek:see:`Input::end_of_data` event will be raised. + +In our example the call would look as follows: + +.. code-block:: zeek + + Input::force_update("denylist"); + +Alternatively, the input framework can automatically refresh the table contents +when it detects a change to the input file. To use this feature you need to +specify a non-default read mode by setting the mode option of the +:zeek:see:`Input::add_table` call. Valid values are :zeek:see:`Input::MANUAL` +(the default), :zeek:see:`Input::REREAD`, and :zeek:see:`Input::STREAM`. For +example, setting the value of the mode option in the previous example would +look like this: + +.. code-block:: zeek + + Input::add_table([$source="denylist.file", $name="denylist", + $idx=Idx, $val=Val, $destination=denylist, + $mode=Input::REREAD]); + +When using the reread mode (i.e., ``$mode=Input::REREAD``), Zeek continually +checks if the input file has been changed. If the file has been changed, it is +re-read and the data in the Zeek table is updated to reflect the current state. +Each time a change has been detected and all the new data has been read into +the table, the :zeek:see:`Input::end_of_data` event is raised. + +When using the streaming mode (i.e., ``$mode=Input::STREAM``), Zeek +assumes that the input is an append-only file to which new data is +continually appended. Zeek also checks to see if the file being +followed has been renamed or rotated. The file is closed and reopened +when tail detects that the filename being read from has a new inode +number. Zeek continually checks for new data at the end of the file +and will add the new data to the table. If newer lines in the file +have the same table index as previous lines, they will overwrite +the values in the output table. Because of the nature of streaming +reads (data is continually added to the table), the +:zeek:see:`Input::end_of_data` event is never raised when using +streaming reads. + +.. tip:: + + Change detection happens via periodic “heartbeat” events, defaulting to a + frequency of once per second as defined by the global + :zeek:see:`Threading::heartbeat_interval` constant. The reader considers the + input file changed when the file’s inode or modification time has changed + since the last check. + +Receiving change events +----------------------- + +When re-reading files, it might be interesting to know exactly which lines in +the source files have changed. For this reason, the input framework can raise +an event each time when a data item is added to, removed from, or changed in a +table. + +The event definition looks like this (note that you can change the name of this +event in your own Zeek script): + +.. code-block:: zeek + + event entry(description: Input::TableDescription, tpe: Input::Event, + left: Idx, right: Val) { + # do something here... + print fmt("%s = %s", left, right); + } + +The event must be specified in ``$ev`` in the :zeek:see:`Input::add_table` +call: + +.. code-block:: zeek + + Input::add_table([$source="denylist.file", $name="denylist", + $idx=Idx, $val=Val, $destination=denylist, + $mode=Input::REREAD, $ev=entry]); + +The description argument of the event contains the arguments that were +originally supplied to the :zeek:see:`Input::add_table` call. Hence, the name +of the stream can, for example, be accessed with ``description$name``. The +``tpe`` argument of the event is an enum containing the type of the change that +occurred. + +If a line that was not previously present in the table has been added, then the +value of ``tpe`` will be :zeek:see:`Input::EVENT_NEW`. In this case left +contains the index of the added table entry and right contains the values of +the added entry. + +If a table entry that already was present is altered during the re-reading or +streaming read of a file, then the value of ``tpe`` will be +:zeek:see:`Input::EVENT_CHANGED`. In this case ``left`` contains the index of +the changed table entry and ``right`` contains the values of the entry before +the change. The reason for this is that the table already has been updated when +the event is raised. The current value in the table can be ascertained by +looking up the current table value. Hence it is possible to compare the new and +the old values of the table. + +If a table element is removed because it was no longer present during a +re-read, then the value of ``tpe`` will be :zeek:see:`Input::EVENT_REMOVED`. In +this case ``left`` contains the index and ``right`` the values of the removed +element. + +Filtering data during import +---------------------------- + +The input framework also allows a user to filter the data during the import. To +this end, predicate functions are used. A predicate function is called before a +new element is added/changed/removed from a table. The predicate can either +accept or veto the change by returning true for an accepted change and false +for a rejected change. Furthermore, it can alter the data before it is written +to the table. + +The following example filter will reject adding entries to the table when they +were generated over a month ago. It will accept all changes and all removals of +values that are already present in the table. + +.. code-block:: zeek + + Input::add_table([$source="denylist.file", $name="denylist", + $idx=Idx, $val=Val, $destination=denylist, + $mode=Input::REREAD, + $pred(tpe: Input::Event, left: Idx, right: Val) = { + if ( tpe != Input::EVENT_NEW ) { + return T; + } + return (current_time() - right$timestamp) < 30day; + }]); + +To change elements while they are being imported, the predicate function can +manipulate ``left`` and ``right``. Note that predicate functions are called +before the change is committed to the table. Hence, when a table element is +changed (``tpe`` is :zeek:see:`Input::EVENT_CHANGED`), ``left`` and ``right`` +contain the new values, but the destination (``denylist`` in our example) still +contains the old values. This allows predicate functions to examine the changes +between the old and the new version before deciding if they should be allowed. + +Broken input data +----------------- + +The input framework notifies you of problems during data ingestion in two ways. +First, reporter messages, ending up in :file:`reporter.log`, indicate the type of +problem and the file in which the problem occurred:: + + #fields ts level message location + 0.000000 Reporter::WARNING denylist.file/Input::READER_ASCII: Did not find requested field ip in input data file denylist.file. (empty) + +Second, the :zeek:see:`Input::TableDescription` and +:zeek:see:`Input::EventDescription` records feature an ``$error_ev`` member to +trigger events indicating the same message and severity levels as shown above. +The use of these events mirrors that of change events. + +For both approaches, the framework suppresses repeated messages regarding the +same file, so mistakes in large data files do not trigger a message flood. + +Finally, the ASCII reader allows coarse control over the robustness in case of +problems during data ingestion. Concretely, the +:zeek:see:`InputAscii::fail_on_invalid_lines` and +:zeek:see:`InputAscii::fail_on_file_problem` flags indicate whether problems +should merely trigger warnings or lead to processing failure. Both default to +warnings. + +Reading Data to Events +====================== + +The second data ingestion mode of the input framework directly generates Zeek +events from ingested data instead of inserting them to a table. Event streams +work very similarly to the table streams discussed above, and most of the +features discussed (such as predicates for filtering) also work for event +streams. To read the denylist of the previous example into an event stream, we +use the :zeek:see:`Input::add_event` function: + +.. code-block:: zeek + + type Val: record { + ip: addr; + timestamp: time; + reason: string; + }; + + event denylistentry(description: Input::EventDescription, + tpe: Input::Event, data: Val) { + # do something here... + print "data:", data; + } + + event zeek_init() { + Input::add_event([$source="denylist.file", $name="denylist", + $fields=Val, $ev=denylistentry]); + } + +Event streams differ from table streams in two ways: + +* An event stream needs no separate index and value declarations — instead, all + source data types are provided in a single record definition. +* Since the framework perceives a continuous stream of events, it has no + concept of a data baseline (e.g. a table) to compare the incoming data to. + Therefore the change event type (an :zeek:see:`Input::Event` instance, + ``tpe`` in the above) is currently always :zeek:see:`Input::EVENT_NEW`. + +These aside, event streams work exactly the same as table streams and support +most of the options that are also supported for table streams. + +Data Readers +============ + +The input framework supports different kinds of readers for different kinds of +source data files. At the moment, the framework defaults to ingesting ASCII +files formatted in the Zeek log file format (tab-separated values with a +``#fields`` header line). Several other readers are included in Zeek, and Zeek +packages/plugins can provide additional ones. + +Reader selection proceeds as follows. The :zeek:see:`Input::default_reader` +variable defines the default reader: :zeek:see:`Input::READER_ASCII`. When you +call :zeek:see:`Input::add_table` or :zeek:see:`Input::add_event` this reader +gets used automatically. You can override the default by assigning the +``$reader`` member in the description record passed into these calls. See test +cases in :file:`testing/btest/scripts/base/frameworks/input/` for examples. + +The ASCII Reader +---------------- + +The ASCII reader, enabled by default or by selecting +:zeek:see:`Input::READER_ASCII`, understands Zeek’s TSV log format. It actually +understands the full set of directives in the preamble of those log files, e.g. +to define the column separator. This is rarely used, and most commonly input +files merely start with a tab-separated row that names the ``#fields`` in the +input file, as shown earlier. + +.. warning:: + + The ASCII reader has no notion of file locking, including UNIX’s advisory + locking. For large files, this means the framework might process a file + that’s still written to. The reader handles resulting errors robustly (e.g. + via the reporter log, as described earlier), but nevertheless will encounter + errors. In order to avoid these problems it’s best to produce a new input + file on the side, and then atomically rename it to the filename monitored by + the framework. + +There’s currently no JSON ingestion mode for this reader, but see the section +about using the :ref:`raw reader ` together with the +builtin :zeek:see:`from_json` function. + +The Benchmark Reader +-------------------- + +The benchmark reader, selected via :zeek:see:`Input::READER_BENCHMARK`, helps +the Zeek developers optimize the speed of the input framework. It can generate +arbitrary amounts of semi-random data in all Zeek data types supported by the +input framework. + +The Binary Reader +----------------- + +This reader, selected via :zeek:see:`Input::READER_BINARY`, is intended for +use with file analysis input streams to ingest file content (and is the default +type of reader for those streams). + +.. _input-raw-reader: + +The Raw Reader +-------------- + +The raw reader, selected via :zeek:see:`Input::READER_RAW`, reads a file that +is split by a specified record separator (newline by default). The contents are +returned line-by-line as strings; it can, for example, be used to read +configuration files and the like and is probably only useful in the event mode +and not for reading data to tables. + +Reading JSON Lines +~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 6.0 + + +While the ASCII reader does not currently support JSON natively, it is +possible to use the raw reader together with the builtin :zeek:see:`from_json` +function to read files in JSON lines format and instantiate Zeek record +values based on the input. + +The following example shows how this can be done, holding two state tables +in order to allow for removal updates of the read data. + +.. literalinclude:: denylist.jsonl + :caption: + :language: json + :linenos: + :tab-width: 4 + +.. literalinclude:: input_json_1.zeek + :caption: Loading denylist.jsonl, converting to Zeek types, populating a table. + :language: zeek + :linenos: + :tab-width: 4 + +If your input data is already in, or can be easily converted into, JSON Lines format +the above approach can be used to load it into Zeek. + +.. _input-sqlite-reader: + +The SQLite Reader +----------------- + +The SQLite input reader, selected via :zeek:see:`Input::READER_SQLITE`, +provides a way to access SQLite databases from Zeek. SQLite is a simple, +file-based, widely used SQL database system. Due to the transactional nature of +SQLite, databases can be used by several applications simultaneously. Hence +they can, for example, be used to make constantly evolving datasets available +to Zeek on a continuous basis. + +Reading Data from SQLite Databases +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Like with Zeek’s logging support, reading data from SQLite databases is built +into Zeek without any extra configuration needed. Just like text-based input +readers, the SQLite reader can read data — in this case the result of SQL +queries — into tables or events. + +Reading Data into Tables +************************ + +To read data from a SQLite database, we first have to provide Zeek with the +information how the resulting data will be structured. For this example, we +expect that we have a SQLite database, which contains host IP addresses and the +user accounts that are allowed to log into a specific machine. + +The SQLite commands to create the schema are as follows:: + + create table machines_to_users ( + host text unique not null, + users text not null); + + insert into machines_to_users values ( + '192.168.17.1', 'johanna,matthias,seth'); + insert into machines_to_users values ( + '192.168.17.2', 'johanna'); + insert into machines_to_users values ( + '192.168.17.3', 'seth,matthias'); + +After creating a file called hosts.sqlite with this content, we can read the +resulting table into Zeek: + +.. code-block:: zeek + + type Idx: record { + host: addr; + }; + + type Val: record { + users: set[string]; + }; + + global hostslist: table[addr] of Val = table(); + + event zeek_init() + { + Input::add_table([$source="/var/db/hosts", + $name="hosts", + $idx=Idx, + $val=Val, + $destination=hostslist, + $reader=Input::READER_SQLITE, + $config=table(["query"] = "select * from machines_to_users;") + ]); + + Input::remove("hosts"); + } + + event Input::end_of_data(name: string, source: string) + { + if ( name != "hosts" ) + return; + + # now all data is in the table + print "Hosts list has been successfully imported"; + + # List the users of one host. + print hostslist[192.168.17.1]$users; + } + +The ``hostslist`` table can now be used to check host logins against an +available user list. + +Turning Data into Events +************************ + +The second mode is to use the SQLite reader to output the input data as events. +Typically there are two reasons to do this. First, the structure of the input +data is too complicated for a direct table import. In this case, the data can +be read into an event which can then create the necessary data structures in +Zeek in scriptland. Second, the dataset is too big to hold in memory. In this +case, event-driven ingestion can perform checks on-demand. + +As an example, let’s consider a large database with malware hashes. Live +database queries allow us to cross-check sporadically occurring downloads +against this evolving database. The SQLite commands to create the schema are as +follows:: + + create table malware_hashes ( + hash text unique not null, + description text not null); + + insert into malware_hashes values ('86f7e437faa5a7fce15d1ddcb9eaeaea377667b8', 'malware a'); + insert into malware_hashes values ('e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98', 'malware b'); + insert into malware_hashes values ('84a516841ba77a5b4648de2cd0dfcb30ea46dbb4', 'malware c'); + insert into malware_hashes values ('3c363836cf4e16666669a25da280a1865c2d2874', 'malware d'); + insert into malware_hashes values ('58e6b3a414a1e090dfc6029add0f3555ccba127f', 'malware e'); + insert into malware_hashes values ('4a0a19218e082a343a1b17e5333409af9d98f0f5', 'malware f'); + insert into malware_hashes values ('54fd1711209fb1c0781092374132c66e79e2241b', 'malware g'); + insert into malware_hashes values ('27d5482eebd075de44389774fce28c69f45c8a75', 'malware h'); + insert into malware_hashes values ('73f45106968ff8dc51fba105fa91306af1ff6666', 'ftp-trace'); + +The following code uses the file-analysis framework to get the sha1 hashes of +files that are transmitted over the network. For each hash, a SQL-query runs +against SQLite. If the query returns a result, we output the matching hash. + +.. code-block:: zeek + + @load frameworks/files/hash-all-files + + type Val: record { + hash: string; + description: string; + }; + + event line(description: Input::EventDescription, tpe: Input::Event, r: Val) + { + print fmt("malware-hit with hash %s, description %s", r$hash, r$description); + } + + global malware_source = "/var/db/malware"; + + event file_hash(f: fa_file, kind: string, hash: string) + { + + # check all sha1 hashes + if ( kind=="sha1" ) + { + Input::add_event( + [ + $source=malware_source, + $name=hash, + $fields=Val, + $ev=line, + $want_record=T, + $config=table( + ["query"] = fmt("select * from malware_hashes where hash='%s';", hash) + ), + $reader=Input::READER_SQLITE + ]); + } + } + + event Input::end_of_data(name: string, source:string) + { + if ( source == malware_source ) + Input::remove(name); + } + +If you run this script against the trace in +:file:`testing/btest/Traces/ftp/ipv4.trace`, you will get one hit. diff --git a/doc/frameworks/input_json_1.zeek b/doc/frameworks/input_json_1.zeek new file mode 100644 index 0000000000..8a829e4375 --- /dev/null +++ b/doc/frameworks/input_json_1.zeek @@ -0,0 +1,56 @@ +## Read a denylist.jsonl file in JSON Lines format +module Denylist; + +type JsonLine: record { + s: string; +}; + +type Entry: record { + ip: addr; + timestamp: time; + reason: string; +}; + +global staged_denies: table[addr] of Entry; +global active_denies: table[addr] of Entry; + +event Input::end_of_data(name: string, source: string) + { + if ( name != "denylist" ) + return; + + # Switch active and staging tables when input file has been read. + active_denies = staged_denies; + staged_denies = table(); + + print network_time(), "end_of_data() active:", table_keys(active_denies); + } + + +event Denylist::json_line(description: Input::EventDescription, tpe: Input::Event, l: string) + { + local parse_result = from_json(l, Entry); + + # Parsing of JSON may fail, so ignore anything invalid. + if ( ! parse_result$valid ) + return; + + # Cast parsed value as Entry... + local entry = parse_result$v as Entry; + + # ...and populate staging table. + staged_denies[entry$ip] = entry; + } + +event zeek_init() + { + Input::add_event([ + $source="denylist.jsonl", + $name="denylist", + $reader=Input::READER_RAW, + $mode=Input::REREAD, + $fields=JsonLine, + $ev=Denylist::json_line, + $want_record=F, + ]); + } diff --git a/doc/frameworks/intel.rst b/doc/frameworks/intel.rst new file mode 100644 index 0000000000..dd7500f4c7 --- /dev/null +++ b/doc/frameworks/intel.rst @@ -0,0 +1,270 @@ + +====================== +Intelligence Framework +====================== + +Introduction +============ + +The goals of Zeek’s Intelligence Framework are to consume intelligence data, +make it available for matching, and provide infrastructure to improve +performance and memory utilization. + +Data in the Intelligence Framework is an atomic piece of intelligence such as +an IP address or an e-mail address. This atomic data will be packed with +metadata such as a freeform source field, a freeform descriptive field, and a +URL which might lead to more information about the specific item. The metadata +in the default scripts has been deliberately kept to a minimum. + +Quick Start +=========== + +First we need to define the intelligence data to match. Let's look for the +domain ``www.reddit.com``. For the details of the file format see the +:ref:`Loading Intelligence ` section below. + +:: + + #fields indicator indicator_type meta.source + www.reddit.com Intel::DOMAIN my_special_source + +Now we need to tell Zeek about the data. Add this line to your local.zeek to +load an intelligence file: + +.. code-block:: zeek + + redef Intel::read_files += { "/somewhere/yourdata.txt" }; + +In a cluster, the text files only need to reside on the manager. + +Add the following line to :file:`local.zeek` to load the scripts that send +“seen” data into the Intelligence Framework to be checked against the loaded +intelligence data: + +.. code-block:: zeek + + @load frameworks/intel/seen + +Intelligence data matches will be logged to the :file:`intel.log` file. A match +on ``www.reddit.com`` might look like this:: + + { + "ts":1320279566.452687, + "uid":"C4llPsinsviGyNY45", + "id.orig_h":"192.168.2.76", + "id.orig_p":52026, + "id.resp_h":"132.235.215.119", + "id.resp_p":80, + "seen.indicator":"www.reddit.com", + "seen.indicator_type":"Intel::DOMAIN", + "seen.where":"HTTP::IN_HOST_HEADER", + "seen.node":"zeek", + "matched":[ + "Intel::DOMAIN" + ], + "sources":[ + "my_special_source" + ]} + +You can explore this example on `try.zeek.org +`_. + +Architecture +============ + +The Intelligence Framework can be thought of as containing three separate +portions. The first part involves loading intelligence data. The second is a +mechanism for indicating to the intelligence framework that a piece of data +which needs to be checked has been seen. The third handles when a positive +match has been discovered. + +.. image:: /images/intel-architecture.png + :align: center + +The figure above depicts how these portions work together: loading intelligence +*inserts* the data into an in-memory data store that is managed by the +intelligence framework. During traffic analysis, scripts report the *seen* data +to the framework to check against the loaded items. + +.. _loading-intelligence: + +Loading Intelligence +-------------------- + +By default, intelligence data is loaded through plain text files using the +Input Framework. In clusters the manager is the only node that needs the +intelligence data. The intelligence framework has distribution mechanisms which +will push data out to all of the nodes that need it. + +Here is an example of the intelligence data format. All fields must be +separated by a single tab character and fields containing only a hyphen are +considered to be null values. Note that there may be additional fields +depending on the loaded extensions. One example is the +:doc:`/scripts/policy/frameworks/intel/do_notice.zeek` script as described +below. + +:: + + #fields indicator indicator_type meta.source meta.desc meta.url + 1.2.3.4 Intel::ADDR source1 Sending phishing email http://source1.com/badhosts/1.2.3.4 + a.b.com Intel::DOMAIN source2 Name used for data exfiltration - + +For a list of all built-in ``indicator_type`` values, please refer to the +documentation of :zeek:see:`Intel::Type`. + +To load the data once the files are created, add the following to your +``local.zeek`` to specify which intel files to load (with your own file names +of course): + +.. code-block:: zeek + + redef Intel::read_files += { + "/somewhere/feed1.txt", + "/somewhere/feed2.txt", + }; + +Remember, the files only need to be present on the file system of the manager +node on cluster deployments. + +The intel framework is very flexible so that intelligence matching can be +extended in numerous ways. For example, the +:doc:`/scripts/policy/frameworks/intel/do_notice.zeek` +script implements a +simple mechanism to raise a Zeek notice (of type :zeek:see:`Intel::Notice`) for +user-specified intelligence matches. To use this feature, add the following +line to ``local.zeek``: + +.. code-block:: zeek + + @load frameworks/intel/do_notice + +The script adds additional metadata fields. In particular, if the ``do_notice`` +field of type bool is set to ``T`` for an intelligence item, Zeek will create a +notice when the item is matched. + +Seen Data +--------- + +When some bit of data is extracted from network traffic (such as an email +address in the “From” header in a SMTP message), the Intelligence Framework +needs to be informed that this data was discovered so that its presence will be +checked within the loaded intelligence data. This is accomplished through the +:zeek:see:`Intel::seen` function. + +Zeek includes a default set of scripts that will send data to the intelligence +framework. To load all of the scripts included with Zeek for sending “seen” +data to the intelligence framework, just add this line to ``local.zeek``: + +.. code-block:: zeek + + @load frameworks/intel/seen + +Alternatively, specific scripts in that directory can be loaded. Keep in mind +that as more data is sent to the intelligence framework, the CPU load consumed +by Zeek will increase depending on how many times the :zeek:see:`Intel::seen` +function is being called. The effect of this condition depends on the nature +and volume of the traffic Zeek monitors. + +Zeek's intelligence framework can only match loaded items if corresponding +occurrences are reported as *seen*. For example, the scripts included with Zeek +will only report IP addresses from established TCP connections to the +intelligence framework. Thus, neither UDP traffic nor one-sided traffic will +trigger intelligence hits by default. However, it is easy to report additional +observations to the framework. The following will report the IPs of all +connections (including ICMP, UDP and one-sided traffic) to the intelligence +framework: + +.. code-block:: zeek + + event new_connection(c: connection) + { + Intel::seen([$host=c$id$orig_h, $conn=c, $where=Conn::IN_ORIG]); + Intel::seen([$host=c$id$resp_h, $conn=c, $where=Conn::IN_RESP]); + } + +Note that using the :zeek:see:`new_connection` event could have a significant +impact on the overall performance as much more data might be processed by the +intelligence framework. + +Intelligence Matches +-------------------- + +The Intelligence Framework provides an event that is generated whenever a match +is discovered. This event is named :zeek:see:`Intel::match` and receives two +arguments. First, a record of type :zeek:see:`Intel::Seen` that describes the +observation as reported to the framework. It contains information about what +was seen (e.g., the domain ``www.slideshare.net``), where it was seen (e.g. in +an X509 certificate) and further context (e.g., a connection or a file record) +if available. The second argument is a set of intelligence items that matched +the observation. A set is used because multiple items may match a given +observation. For example, assume you have ingested the IP ``1.2.3.4`` from +source A and from source B as well as the subnet ``1.2.3.0/24`` from source B. +If the IP ``1.2.3.4`` is seen in your traffic, the match event will receive all +three intelligence items. + +In a cluster setup, the match event is raised on the manager. This is important +to keep in mind when writing a script that handles the event. While the context +information about the match is available through the event parameters, the +handler itself is executed on the manager. Thus, one cannot access any state +that is local to the worker node that reported the observation in the first +place. Other interaction is also limited. For example, one cannot reliably +trigger file extraction based on an intelligence hit: Once the manager +processes the match event and comes to the conclusion that file extraction +would be desired, the worker that triggered the hit is most likely done +processing the corresponding data. Instead, one would need to start by +extracting all files that are potentially relevant, keep the ones that refer to +an intelligence hit and regularly discard the others. + +Intelligence matches are logged to the :file:`intel.log` file. For further +description of each field in that file, see the documentation for the +:zeek:see:`Intel::Info` record. + +The following are two matches from a sample :file:`intel.log`:: + + { + "ts": "2019-03-12T18:22:19.252191Z", + "uid": "Cpue7J1KNReqCodXHc", + "id.orig_h": "192.168.4.6", + "id.orig_p": 64738, + "id.resp_h": "13.107.18.13", + "id.resp_p": 443, + "seen.indicator": "www.slideshare.net", + "seen.indicator_type": "Intel::DOMAIN", + "seen.where": "X509::IN_CERT", + "seen.node": "so16-enp0s8-1", + "matched": [ + "Intel::DOMAIN" + ], + "sources": [ + "from http://hosts-file.net/fsa.txt via intel.criticalstack.com" + ], + "fuid": "FnRp0j1YMig5KhcMDg", + "file_mime_type": "application/x-x509-user-cert", + "file_desc": "13.107.18.13:443/tcp" + } + { + "ts": "2019-03-12T18:32:19.821962Z", + "uid": "CvusFJ2HdbTnCLxEUa", + "id.orig_h": "192.168.4.6", + "id.orig_p": 64826, + "id.resp_h": "13.107.42.14", + "id.resp_p": 443, + "seen.indicator": "www.slideshare.net", + "seen.indicator_type": "Intel::DOMAIN", + "seen.where": "X509::IN_CERT", + "seen.node": "so16-enp0s8-1", + "matched": [ + "Intel::DOMAIN" + ], + "sources": [ + "from http://hosts-file.net/fsa.txt via intel.criticalstack.com" + ], + "fuid": "FUrrLa45T7a8hjdRy", + "file_mime_type": "application/x-x509-user-cert", + "file_desc": "13.107.42.14:443/tcp" + } + +These examples show there were matches in a domain observed in a X509 +certificate. That domain was ``www.slideshare.net``. This is unusual as that +domain is used for legitimate purposes. This example demonstrates that analysts +must vet intelligence feeds for their local use and applicability. diff --git a/doc/frameworks/logging-input-sqlite.rst b/doc/frameworks/logging-input-sqlite.rst new file mode 100644 index 0000000000..a785e99b58 --- /dev/null +++ b/doc/frameworks/logging-input-sqlite.rst @@ -0,0 +1,8 @@ +:orphan: + +==================== +SQLite Input/Logging +==================== + +* :ref:`SQLite Input Reader ` +* :ref:`SQLite Log Writer ` diff --git a/doc/frameworks/logging.rst b/doc/frameworks/logging.rst new file mode 100644 index 0000000000..b764a730cc --- /dev/null +++ b/doc/frameworks/logging.rst @@ -0,0 +1,1087 @@ + +.. _framework-logging: + +================= +Logging Framework +================= + +Zeek comes with a flexible logging interface that allows fine-grained control of +what gets logged and how it is logged. This section explains how you can use +this framework to customize and extended your logs. + +Terminology +=========== + +Zeek’s logging interface is built around three main abstractions: + + Streams + A log stream corresponds to a single log. It defines the set of fields that + a log consists of with their names and types. Examples are the conn stream + for recording connection summaries, and the http stream for recording HTTP + activity. + + Filters + Each stream has a set of filters attached to it that determine what + information gets written out, and how. By default, each stream has one + default filter that just logs everything directly to disk. However, + additional filters can be added to record only a subset of the log records, + write to different outputs, or set a custom rotation interval. If all + filters are removed from a stream, then output is disabled for that stream. + + Writers + Each filter has a writer. A writer defines the actual output format for the + information being logged. The default writer is the ASCII writer, which + produces tab-separated ASCII files. Other writers are available, like for + binary output or direct logging into a database. + +There are several different ways to customize Zeek’s logging: you can create a +new log stream, you can extend an existing log with new fields, you can apply +filters to an existing log stream, or you can customize the output format by +setting log writer options. All of these approaches are described below. + +Streams +======= + +In order to log data to a new log stream, all of the following needs to be done: + +* A :zeek:see:`record` type must be defined which consists of all the fields + that will be logged (by convention, the name of this record type is usually + “Info”). +* A log stream ID (an :zeek:see:`enum` with type name :zeek:see:`Log::ID`) must + be defined that uniquely identifies the new log stream. +* A log stream must be created using the :zeek:see:`Log::create_stream` + function. +* When the data to be logged becomes available, the :zeek:see:`Log::write` + function must be called. + +In the following example, we create a new module, ``Foo``, which creates a new +log stream. + +.. code-block:: zeek + + module Foo; + + export { + # Create an ID for our new stream. By convention, this is + # called "LOG". + redef enum Log::ID += { LOG }; + + # Define the record type that will contain the data to log. + type Info: record { + ts: time &log; + id: conn_id &log; + service: string &log &optional; + missed_bytes: count &log &default=0; + }; + } + + # Optionally, we can add a new field to the connection record so that + # the data we are logging (our "Info" record) will be easily + # accessible in a variety of event handlers. + redef record connection += { + # By convention, the name of this new field is the lowercase name + # of the module. + foo: Info &optional; + }; + + # This event is handled at a priority higher than zero so that if + # users modify this stream in another script, they can do so at the + # default priority of zero. + event zeek_init() &priority=5 + { + # Create the stream. This adds a default filter automatically. + Log::create_stream(Foo::LOG, [$columns=Info, $path="foo"]); + } + +In the definition of the ``Info`` record above, notice that each field has the +:zeek:see:`&log` attribute. Without this attribute, a field will not appear in +the log output. Also notice one field has the :zeek:see:`&optional` attribute. +This indicates that the field might not be assigned any value before the log +record is written. Finally, a field with the :zeek:see:`&default` attribute +has a default value assigned to it automatically. + +At this point, the only thing missing is a call to the :zeek:see:`Log::write` +function to send data to the logging framework. The actual event handler where +this should take place will depend on where your data becomes available. In +this example, the :zeek:see:`connection_established` event provides our data, +and we also store a copy of the data being logged into the +:zeek:see:`connection` record: + +.. code-block:: zeek + + event connection_established(c: connection) + { + local rec: Foo::Info = [$ts=network_time(), $id=c$id]; + + # Store a copy of the data in the connection record so other + # event handlers can access it. + c$foo = rec; + + Log::write(Foo::LOG, rec); + } + +If you run Zeek with this script, a new log file :file:`foo.log` will be +created. Although we only specified four fields in the ``Info`` record above, +the log output will actually contain seven fields because one of the fields +(the one named ``id``) is itself a record type. Since a :zeek:see:`conn_id` +record has four fields, then each of these fields is a separate column in the +log output. Note that the way that such fields are named in the log output +differs slightly from the way we would refer to the same field in a Zeek script +(each dollar sign is replaced with a period). For example, to access the first +field of a :zeek:see:`conn_id` in a Zeek script we would use the notation +``id$orig_h``, but that field is named ``id.orig_h`` in the log output. + +When you are developing scripts that add data to the :zeek:see:`connection` +record, care must be given to when and how long data is stored. Normally data +saved to the connection record will remain there for the duration of the +connection and from a practical perspective it’s not uncommon to need to delete +that data before the end of the connection. + +Add Fields to a Log +------------------- + +You can add additional fields to a log by extending the record type that +defines its content, and setting a value for the new fields before each log +record is written. + +Let’s say we want to add a boolean field ``is_private`` to +:zeek:see:`Conn::Info` that indicates whether the originator IP address is part +of the :rfc:`1918` space: + +.. code-block:: zeek + + # Add a field to the connection log record. + redef record Conn::Info += { + ## Indicate if the originator of the connection is part of the + ## "private" address space defined in RFC1918. + is_private: bool &default=F &log; + }; + +As this example shows, when extending a log stream’s ``Info`` record, each new +field must always be declared either with a &default value or as +:zeek:see:`&optional`. Furthermore, you need to add the :zeek:see:`&log` +attribute or otherwise the field won’t appear in the log file. + +Now we need to set the field. Although the details vary depending on which log +is being extended, in general it is important to choose a suitable event in +which to set the additional fields because we need to make sure that the fields +are set before the log record is written. Sometimes the right choice is the +same event which writes the log record, but at a higher priority (in order to +ensure that the event handler that sets the additional fields is executed +before the event handler that writes the log record). + +In this example, since a connection’s summary is generated at the time its +state is removed from memory, we can add another handler at that time that sets +our field correctly: + +.. code-block:: zeek + + event connection_state_remove(c: connection) + { + if ( c$id$orig_h in Site::private_address_space ) + c$conn$is_private = T; + } + +Now :file:`conn.log` will show a new field ``is_private`` of type +:zeek:see:`bool`. If you look at the Zeek script which defines the connection +log stream :doc:`/scripts/base/protocols/conn/main.zeek`, you will see that +:zeek:see:`Log::write` gets called in an event handler for the same event as +used in this example to set the additional fields, but at a lower priority than +the one used in this example (i.e., the log record gets written after we assign +the ``is_private`` field). + +For extending logs this way, one needs a bit of knowledge about how the script +that creates the log stream is organizing its state keeping. Most of the +standard Zeek scripts attach their log state to the :zeek:see:`connection` +record where it can then be accessed, just like ``c$conn`` above. For example, +the HTTP analysis adds a field http of type :zeek:see:`HTTP::Info` to the +:zeek:see:`connection` record. + +Define a Logging Event +---------------------- + +Sometimes it is helpful to do additional analysis of the information being +logged. For these cases, a stream can specify an event that will be generated +every time a log record is written to it. To do this, we need to modify the +example module shown above to look something like this: + +.. code-block:: zeek + + module Foo; + + export { + redef enum Log::ID += { LOG }; + + type Info: record { + ts: time &log; + id: conn_id &log; + service: string &log &optional; + missed_bytes: count &log &default=0; + }; + + # Define a logging event. By convention, this is called + # "log_". + global log_foo: event(rec: Info); + } + + event zeek_init() &priority=5 + { + # Specify the "log_foo" event here in order for Zeek to raise it. + Log::create_stream(Foo::LOG, [$columns=Info, $ev=log_foo, + $path="foo"]); + } + +All of Zeek’s default log streams define such an event. For example, the +connection log stream raises the event :zeek:see:`Conn::log_conn`. You could +use that for example for flagging when a connection to a specific destination +exceeds a certain duration: + +.. code-block:: zeek + + redef enum Notice::Type += { + ## Indicates that a connection remained established longer + ## than 5 minutes. + Long_Conn_Found + }; + + event Conn::log_conn(rec: Conn::Info) + { + if ( rec?$duration && rec$duration > 5mins ) + NOTICE([$note=Long_Conn_Found, + $msg=fmt("unusually long conn to %s", rec$id$resp_h), + $id=rec$id]); + } + +Often, these events can be an alternative to post-processing Zeek logs +externally with Perl scripts. Much of what such an external script would do +later offline, one may instead do directly inside of Zeek in real-time. + +Disable a Stream +---------------- + +One way to “turn off” a log is to completely disable the stream. For example, +the following example will prevent the :file:`conn.log` from being written: + +.. code-block:: zeek + + event zeek_init() + { + Log::disable_stream(Conn::LOG); + } + +Note that this must run after the stream is created, so the priority of this +event handler must be lower than the priority of the event handler where the +stream was created. + + +Delaying Log Writes +------------------- + +.. versionadded:: 6.2 + +The logging framework allows delaying log writes using the +:zeek:see:`Log::delay` function. + +This functionality enables querying or waiting for additional data to attach to +an in-flight log record for which a :zeek:see:`Log::write` has happened. +Common examples are the execution of DNS reverse lookups for the addresses +of a connection, or - more generally - asynchronous queries to external systems. +Similarly, waiting a small duration for more data from an external process +pertaining to specific connections or events is another. For example, endpoint +agents may provide detailed process information for specific connections +logged by Zeek. + +Conceptually, the delay of a log record is placed after the execution of the +global :zeek:see:`Log::log_stream_policy` hook and before the execution of +:ref:`policy hooks attached to filters `. +At this point, calling :zeek:see:`Log::delay` is only valid for the currently +*active write* during the execution of the global :zeek:see:`Log::log_stream_policy` +hook. Calling :zeek:see:`Log::delay` in any other context or with the wrong +arguments results in runtime errors. + +.. note:: + + While this may appear very restrictive, it does make it explicit that it is + the action of a :zeek:see:`Log::write` for a given stream and log record + that is being delayed as well as providing a defined point where a delay starts. + + Prior ideas entertained the idea of an implicit and very lax interface, but + in the end was deemed too loose and provided too much flexibility that would + be hard to later restrict again or keep stable. The current interface might + be made more lax in the future if it turns out to be too rigid. + + +By default, log records are not delayed. That is, during the execution of +the :zeek:see:`Log::write` function, a serialized version of the given log +record is handed off to a remote logger or a local logging thread. +Modifications of the same log record after :zeek:see:`Log::write` has returned +have no effect. + +In contrast, when a log write is delayed using the :zeek:see:`Log::delay` +function, the record is enqueued into a per-stream record queue and the +:zeek:see:`Log::write` returns. Processing of the delayed write resumes once +it is released by using the :zeek:see:`Log::delay_finish` function or until +a maximum, per-stream configurable, delay duration expires. + +When processing of a log write is resumed, first, all post delay callbacks +given to :zeek:see:`Log::delay` are executed. Thereafter, as for non-delayed +writes, filter policy hooks are executed and the log record is serialized. + +Policy hooks attached to filters and the serialization step observe any +mutations done during the delay. Filter policy hooks may even use these +modifications for deciding on the verdict of the given log record. + +.. note:: + + Policy hooks attached to filters are often used to skip logging of + uninteresting log records. When combined with log write delaying, users + should consider lifting such filter logic up into the + :zeek:see:`Log::log_stream_policy` hook to avoid unnecessarily delaying + records when it is known that these will be discarded later on. + + +The :zeek:see:`Log::delay` and :zeek:see:`Log::delay_finish` functions increment +and decrement an internal reference count for a given write. To continue a +delayed write, :zeek:see:`Log::delay_finish` must be called as often as +:zeek:see:`Log::delay`. + + +Zeek delays a log record by a configurable interval defined for each log stream. +It defaults to the global :zeek:see:`Log::default_max_delay_interval`, and can be +adapted by calling :zeek:see:`Log::set_max_delay_interval` on the stream. +It is possible to explicitly extend the delay duration by providing a post +delay callback to :zeek:see:`Log::delay`. Calling :zeek:see:`Log::delay` from +within such a post delay callback re-delays the record, essentially putting +it at the end of the per-stream queue again. + +.. note:: + + While this puts additional burden on the script writer to realize per-record + specific longer delay intervals, it allows for a simpler internal implementation. + Additionally, the explicit re-delaying is also meant to make users aware of the + consequences when using such long delays either on purpose or by accident. + + For multiple second or even longer delays, it is suggested to consider resumable, + robust and non-ephemeral external post processing steps based on Zeek logs instead. + In the face of worker crashes or uncontrolled restarts of a Zeek cluster, all + delayed log records are inevitably lost. + + +The following example shows how to use the :ref:`when ` to asynchronously +lookup the DNS names of the originator and responder addresses to enrich an +in-flight :zeek:see:`Conn::Info` record. By default, a stream's maximum delay +interval is 200 milliseconds - the ``timeout 150msec`` part ensures a delayed +write resumes after 150 milliseconds already by explicitly calling +:zeek:see:`Log::delay_finish`. + + +.. literalinclude:: logging/delay1.zeek + :caption: Enriching conn.log with originator and responder names. + :language: zeek + :linenos: + :tab-width: 4 + + +Filters +======= + +A stream has one or more filters attached to it. A stream without any filters +will not produce any log output. Filters govern two aspects of log production: +they control which of the stream’s log entries get written out, and they define +how to actually implement the log writes. They do the latter by specifying a +log writer that implements the write operation, such as the ASCII writer (see +below) for text file output. When a stream is created, it automatically gets a +default filter attached to it. This default filter can be removed or replaced, +or other filters can be added to the stream. This is accomplished by using +either the :zeek:see:`Log::add_filter` or :zeek:see:`Log::remove_filter` +function. This section shows how to use filters to do such tasks as rename a +log file, split the output into multiple files, control which records are +written, and set a custom rotation interval. + +Each filter has a unique name, scoped to the stream it belongs to. That is, all +filters attached to a given stream have different names. Calling +:zeek:see:`Log::add_filter` to add a filter with a name that already exists for +the stream replaces the existing filter. + +Rename a Log File +----------------- + +Normally, the log filename for a given log stream is determined when the stream +is created, unless you explicitly specify a different one by adding a filter. + +The easiest way to change a log filename is to simply replace the default log +filter with a new filter that specifies a value for the ``path`` field. In this +example, :file:`conn.log` will be changed to :file:`myconn.log`: + +.. code-block:: zeek + + event zeek_init() + { + # Replace default filter for the Conn::LOG stream in order to + # change the log filename. + + local f = Log::get_filter(Conn::LOG, "default"); + f$path = "myconn"; + Log::add_filter(Conn::LOG, f); + } + +Keep in mind that the ``path`` field of a log filter never contains the +filename extension. The extension will be determined later by the log writer. + +Change the Logging Directory +---------------------------- + +By default, Zeek log files are created in the current working directory. +To write logs into a different directory, set :zeek:see:`Log::default_logdir`: + +.. code-block:: zeek + + redef Log::default_logdir = /path/to/output_log_directory + +The :zeek:see:`Log::default_logdir` option is honored by all file based +writes included with Zeek (ASCII and SQLite). + +Add an Additional Output File +----------------------------- + +Normally, a log stream writes to only one log file. However, you can add +filters so that the stream writes to multiple files. This is useful if you want +to restrict the set of fields being logged to the new file. + +In this example, a new filter is added to the :zeek:see:`Conn::LOG` stream that +writes two fields to a new log file: + +.. code-block:: zeek + + event zeek_init() + { + # Add a new filter to the Conn::LOG stream that logs only + # timestamp and originator address. + + local filter: Log::Filter = [$name="orig-only", $path="origs", + $include=set("ts", "id.orig_h")]; + Log::add_filter(Conn::LOG, filter); + } + +.. note:: + + When multiple filters added to a stream use the same path value, Zeek will + disambiguate the output file names by adding numeric suffixes to the name. If + we say ``$path="conn"`` in the above example, Zeek warns us about the fact that + it’ll write this filter’s log entries to a different file:: + + 1071580905.346457 warning: Write using filter 'orig-only' on path 'conn' changed to use new path 'conn-2' to avoid conflict with filter 'default' + + The same also happens when omitting a path value, in which case the filter + inherits the value of the stream’s path member. + +Notice how the ``include`` filter attribute specifies a set that limits the +fields to the ones given. The names correspond to those in the +:zeek:see:`Conn::Info` record (however, because the ``id`` field is itself a +record, we can specify an individual field of ``id`` by the dot notation shown +in the example). + +Using the code above, in addition to the regular :file:`conn.log`, you will now +also get a new log file :file:`origs.log` that looks like the regular +:file:`conn.log`, but will have only the fields specified in the ``include`` +filter attribute. + +If you want to skip only some fields but keep the rest, there is a +corresponding exclude filter attribute that you can use instead of include to +list only the ones you are not interested in. + +If you want to make this the only log file for the stream, you can remove the +default filter: + +.. code-block:: zeek + + event zeek_init() + { + # Remove the filter called "default". + Log::remove_filter(Conn::LOG, "default"); + } + +Determine Log Path Dynamically +------------------------------ + +Instead of using the ``path`` filter attribute, a filter can determine output +paths *dynamically* based on the record being logged. That allows, e.g., to +record local and remote connections into separate files. To do this, you define +a function that returns the desired path, and use the ``path_func`` filter +attribute: + +.. code-block:: zeek + + function myfunc(id: Log::ID, path: string, rec: Conn::Info) : string + { + # Return "conn-local" if originator is a local IP, otherwise + # return "conn-remote". + local r = Site::is_local_addr(rec$id$orig_h) ? "local" : "remote"; + return fmt("%s-%s", path, r); + } + + event zeek_init() + { + local filter: Log::Filter = [$name="conn-split", + $path_func=myfunc, $include=set("ts", "id.orig_h")]; + Log::add_filter(Conn::LOG, filter); + } + +Running this will now produce two new files, :file:`conn-local.log` and +:file:`conn-remote.log`, with the corresponding entries. For this example +to work, :zeek:see:`Site::local_nets` must specify your local network. +It defaults to IANA's standard private address space. One +could extend this further for example to log information by subnets or even by +IP address. Be careful, however, as it is easy to create many files very +quickly. + +The ``myfunc`` function has one drawback: it can be used only with the :zeek:see:`Conn::LOG` +stream as the record type is hardcoded into its argument list. However, Zeek +allows to do a more generic variant: + +.. code-block:: zeek + + function myfunc(id: Log::ID, path: string, + rec: record { id: conn_id; } ) : string + { + local r = Site::is_local_addr(rec$id$orig_h) ? "local" : "remote"; + return fmt("%s-%s", path, r); + } + +This function can be used with all log streams that have records containing an +``id: conn_id`` field. + +.. _logging-filtering-log-records: + +Filtering Log Records +--------------------- + +We just saw ways how to customize the logged columns. The logging framework also +lets you control which records Zeek writes out. It relies on Zeek’s :zeek:see:`hook` +mechanism to do this, as follows. The framework provides two levels of "policy" +hooks, a global one and a set of filter-level ones. The hook handlers can +implement additional processing of a log record, including vetoing the writing +of the record. Any handler that uses a :zeek:see:`break` statement to leave the +hook declares that a record shall not be written out. Anyone can attach handlers +to these hooks, which look as follows: + +.. code-block:: zeek + + type Log::StreamPolicyHook: hook(rec: any, id: ID); + type Log::PolicyHook: hook(rec: any, id: ID, filter: Filter); + +For both hook types, the ``rec`` argument contains the entry to be logged and is +an instance of the record type associated with the stream’s columns, and ``id`` +identifies the log stream. + +The logging framework defines one global hook policy hook: :zeek:see:`Log::log_stream_policy`. +For every log write, this hook gets invoked first. Any of its handlers may +decide to veto the log entry. The framework then iterates over the log stream's +filters. Each filter has a ``filter$policy`` hook of type :zeek:see:`Log::PolicyHook`. +Its handlers receive the log record, the ID of the log stream, and the filter +record itself. Each handler can veto the write. After the filter's hook has run, +any veto (by :zeek:see:`Log::log_stream_policy` or the filter's hook) aborts the +write via that filter. If no veto has occurred, the filter now steers the log +record to its output. + +You can pass arbitrary state through these hook handlers. For example, you can +extending streams or filters via a :zeek:see:`redef`, or pass key-value pairs +via the ``filter$config`` table.. + +Since you'll often want to use uniform handling for all writes on a given +stream, log streams offer a default hook, provided when constructing the stream, +that the stream's filters will use if they don't provide their own. To support +hooks on your log streams, you should always define a default hook when creating +new streams, as follows: + +.. code-block:: zeek + + module Foo; + + export { + ## The logging stream identifier. + redef enum Log::ID += { LOG }; + + ## A default logging policy hook for the stream. + global log_policy: Log::PolicyHook; + + # Define the record type that will contain the data to log. + type Info: record { + ts: time &log; + id: conn_id &log; + service: string &log &optional; + missed_bytes: count &log &default=0; + }; + } + + event zeek_init() &priority=5 + { + # Create the stream, adding the default policy hook: + Log::create_stream(Foo::LOG, [$columns=Info, $path="foo", $policy=log_policy]); + } + +With this hook in place, it’s now easy to add a filtering predicate for the ``Foo`` +log from anywhere: + +.. code-block:: zeek + + hook Foo::log_policy(rec: Foo::Info, id: Log::ID, filter: Log::Filter) + { + # Let's only log complete information: + if ( rec$missed_bytes > 0 ) + break; + } + +The Zeek distribution features default hooks for all of its streams. Here’s a +more realistic example, using HTTP: + +.. code-block:: zeek + + hook HTTP::log_policy(rec: HTTP::Info, id: Log::ID, filter: Log::Filter) + { + # Record only connections with successfully analyzed HTTP traffic + if ( ! rec?$service || rec$service != "http" ) + break; + } + +To override a hook selectively in a new filter, set the hook when adding the +filter to a stream: + +.. code-block:: zeek + + hook my_policy(rec: Foo::Info, id: Log::ID, filter: Log::Filter) + { + # Let's only log incomplete flows: + if ( rec$missed_bytes == 0 ) + break; + } + + event zeek_init() + { + local filter: Log::Filter = [$name="incomplete-only", + $path="foo-incomplete", + $policy=my_policy]; + Log::add_filter(Foo::LOG, filter); + } + +Note that this approach has subtle implications: the new filter does not use the +``Foo::log_policy`` hook, and that hook does not get invoked for writes to this +filter. Any vetoes or additional processing implemented in ``Foo::log_policy`` +handlers no longer happens for the new filter. Such hook replacement should +rarely be necessary; you may find it preferable to narrow the stream's default +handler to the filter in question: + +.. code-block:: zeek + + hook Foo::log_policy(rec: Foo::Info, id: Log::ID, filter: Log::Filter) + { + if ( filter$name != "incomplete-only" ) + return; + + # Let's only log incomplete flows: + if ( rec$missed_bytes == 0 ) + break; + } + +For tasks that need to run once per-write, not once per-write-and-filter, +use the :zeek:see:`Log::log_stream_policy` instead: + +.. code-block:: zeek + + hook Log::log_stream_policy(rec: Foo::Info, id: Log::ID) + { + # Called once per write + } + + hook Foo::log_policy(rec: Foo::Info, id: Log::ID, filter: Log::Filter) + { + # Called once for each of Foo's filters. + } + +To change an existing filter first retrieve it, then update it, and +re-establish it: + +.. code-block:: zeek + + hook my_policy(rec: Foo::Info, id: Log::ID, filter: Log::Filter) + { + # Let's only log incomplete flows: + if ( rec$missed_bytes == 0 ) + break; + } + + event zeek_init() + { + local f = Log::get_filter(Foo::LOG, "default"); + f$policy = my_policy; + Log::add_filter(Foo::LOG, f); + } + +.. note:: + + Policy hooks can also modify the log records, but with subtle implications. + The logging framework applies all of a stream’s log filters sequentially to + the same log record, so modifications made in a hook handler will persist + not only into subsequent handlers in the same hook, but also into any in + filters processed subsequently. In contrast to hook priorities, filters + provide no control over their processing order. + +Log Rotation and Post-Processing +-------------------------------- + +The logging framework provides fine-grained control over when and how to rotate +log files. Log rotation means that Zeek periodically renames an active log +file, such as :file:`conn.log`, in a manner configurable by the user (e.g., +renaming to :file:`conn_21-01-03_14-05-00.log` to timestamp it), and starts +over on a fresh :file:`conn.log` file. Post-processing means that Zeek can also +apply optional additional processing to the rotated file, such as compression +or file transfers. These mechanisms apply naturally to file-based log writers, +but are available to other writers as well for more generalized forms of +periodic additional processing of their outputs. + +Rotation Timing +~~~~~~~~~~~~~~~ + +The log rotation interval is globally controllable for all filters by +redefining the :zeek:see:`Log::default_rotation_interval` constant, or +specifically for certain :zeek:see:`Log::Filter` instances by setting their +``interv`` field. The default value, ``0secs``, disables rotation. + +.. note:: + + When using ZeekControl, this option is set automatically via the ZeekControl + configuration. + +Here’s an example of changing just the :zeek:see:`Conn::LOG` stream’s default +filter rotation: + +.. code-block:: zeek + + event zeek_init() + { + local f = Log::get_filter(Conn::LOG, "default"); + f$interv = 1 min; + Log::add_filter(Conn::LOG, f); + } + +Controlling File Naming +~~~~~~~~~~~~~~~~~~~~~~~ + +The redef’able :zeek:see:`Log::rotation_format_func` determines the naming of +the rotated-to file. The logging framework invokes the function with sufficient +context (a :zeek:see:`Log::RotationFmtInfo` record), from which it determines +the output name in two parts: the output directory, and the output file’s base +name, meaning its name without a suffix. It returns these two components via a +:zeek:see:`Log::RotationPath` record. The output directory defaults to +:zeek:see:`Log::default_rotation_dir` (a config option) and incorporates a +timestamp in the base name, as specified by +:zeek:see:`Log::default_rotation_date_format`. + +When :zeek:see:`Log::default_logdir` is in use and :zeek:see:`Log::rotation_format_func` +does not set an output directory (e.g. when :zeek:see:`Log::default_rotation_dir` is not set), +:zeek:see:`Log::default_logdir` is used as the default output directory. + +For examples of customized log rotation, take a look at the +`relevant `_ +`test `_ +`cases `_. + +Post-Processing of Rotated Logs +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Post-processing can proceed via defaults configured across all log filters, or +with per-filter customizations. Zeek provides helpful default infrastructure to +simplify running shell commands on rotated logs, but you’re free to define your +own post-processing infrastructure from scratch. + +By default, the :zeek:see:`Log::default_rotation_postprocessor_cmd`, if +defined, runs on every rotated log. The wrapper function making the actual +command invocation is :zeek:see:`Log::run_rotation_postprocessor_cmd`. It +passes six additional arguments to the configured shell command: + +* The rotated-to file name (e.g. ``conn_21-01-03_14-05-00.log``) +* The original base name (e.g. ``conn``) +* The timestamp at which the original log file got created (e.g. ``21-01-03_14.04.00``) +* The timestamp at which the original log file got rotated (e.g. ``21-01-03_15.05.00``) +* ``1`` if Zeek is terminating, ``0`` otherwise +* The name of the writer (e.g. ``ascii`` for the ASCII writer) + +.. warning:: + + Zeek ignores failures (non-zero exit codes) of this shell command: the + default rotation postprocessor command returns ``T`` regardless. Be careful + if you implement your own postprocessor function: returning ``F`` from it + will cause the corresponding log writer instance to shut down, therefore do + so only when the writer really won’t be able to continue. + +Zeek ships with ready-to-use postprocessors for file transfer via :doc:`SCP +` and +:doc:`SFTP `. The +Zeek project also provides an external tool, `zeek-archiver +`_, that performs log compression +outside of the Zeek process for robustness. + +Other Features +-------------- + +Log Extension Fields +~~~~~~~~~~~~~~~~~~~~ + +The logging framework provides rudimentary support for adding additional +columns to an already defined log format, globally for all logs or for +individual log filters only. Records returned by the +:zeek:see:`Log::default_ext_func` function get added to every log, and the +``ext_func`` member of :zeek:see:`Log::Filter` in filter records allows local +overrides. + +You can configure a prefix string separately for either of these options — this +string ensures that the resulting fields don’t collide with already existing +log fields. The prefix defaults to an underscore, via +:zeek:see:`Log::default_ext_prefix`. The ``ext_prefix`` field in filter +records overrides as needed. + +The following example, taken straight from a Zeek testcase, adds three extra +columns to all logs: + +.. code-block:: zeek + + type Extension: record { + write_ts: time &log; + stream: string &log; + system_name: string &log; + }; + + function add_extension(path: string): Extension + { + return Extension($write_ts = network_time(), + $stream = path, + $system_name = peer_description); + } + + redef Log::default_ext_func = add_extension; + +A resulting :file:`conn.log`:: + + #fields _write_ts _stream _system_name ts uid … + #types time string string time string … + 1071580905.346457 conn zeek 1071580904.891921 Cod6Wj3YeJFHgkaO8j … + +.. note:: + + Extension fields remain separate from the original log record. They remain + invisible to filters, policy hooks, and log events. *After* filter processing + determines that an entry is to be logged, the framework simply tucks the + extension's members onto the list of fields to write out. + +Field Name Mapping +~~~~~~~~~~~~~~~~~~ + +On occasion it can be handy to rewrite column names as they appear in a Zeek +log. A typical use case for this would be to ensure that column naming complies +with the requirements of your log ingestion system. To achieve this, you can +provide name translation maps, and here too you can do this either globally or +per-filter. The maps are simple string tables with the keys being Zeek’s field +names and the values being the ones to actually write out. Field names not +present in the maps remain unchanged. The global variant is the (normally +empty) :zeek:see:`Log::default_field_name_map`, and the corresponding +filter-local equivalent is the filter’s ``field_name_map`` member. + +For example, the following name map gets rid of the dots in the usual naming of +connection IDs: + +.. code-block:: zeek + + redef Log::default_field_name_map = { + ["id.orig_h"] = "id_orig_h", + ["id.orig_p"] = "id_orig_p", + ["id.resp_h"] = "id_resp_h", + ["id.resp_p"] = "id_resp_p" + }; + +With it, all logs rendering a connection identifier tuple now use ... + +:: + + #fields ts uid id_orig_h id_orig_p id_resp_h id_resp_p ... + +… instead of the default names: + +:: + + #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p ... + +If you’d prefer this change only for a given log filter, make the change to the +filter record directly. The following changes the naming only for +:file:`conn.log`: + +.. code-block:: zeek + + event zeek_init() + { + local f = Log::get_filter(Conn::LOG, "default"); + f$field_name_map = table( + ["id.orig_h"] = "id_orig_h", + ["id.orig_p"] = "id_orig_p", + ["id.resp_h"] = "id_resp_h", + ["id.resp_p"] = "id_resp_p"); + Log::add_filter(Conn::LOG, f); + } + +Printing to Log Messages +~~~~~~~~~~~~~~~~~~~~~~~~ + +Zeek’s :zeek:see:`print` statement normally writes to ``stdout`` or a specific +output file. By adjusting the :zeek:see:`Log::print_to_log` enum value you can +redirect such statements to instead go directly into a Zeek log. Possible +values include: + +* :zeek:see:`Log::REDIRECT_NONE`: the default, which doesn’t involve Zeek logs +* :zeek:see:`Log::REDIRECT_STDOUT`: prints that would normally go to stdout go + to a log +* :zeek:see:`Log::REDIRECT_ALL`: any prints end up in a log instead of stdout + or other files + +The :zeek:see:`Log::print_log_path` defines the name of the log file, +:zeek:see:`Log::PrintLogInfo` its columns, and :zeek:see:`Log::log_print` +events allow you to process logged messages via event handlers. + +Local vs Remote Logging +~~~~~~~~~~~~~~~~~~~~~~~ + +In its log processing, Zeek considers whether log writes should happen locally +to a Zeek node or remotely on another node, after forwarding log entries to it. +Single-node Zeek setups default to local logging, whereas cluster setups enable +local logging only on logger nodes, and log remotely on all but the logger +nodes. You normally don’t need to go near these settings, but you can do so by +``redef``’ing the :zeek:see:`Log::enable_local_logging` and +:zeek:see:`Log::enable_remote_logging` booleans, respectively. + +Writers +======= + +Each filter has a writer. If you do not specify a writer when adding a filter +to a stream, then the ASCII writer is the default. + +There are two ways to specify a non-default writer. To change the default +writer for all log filters, just redefine the :zeek:see:`Log::default_writer` +option. Alternatively, you can specify the writer to use on a per-filter basis +by setting a value for the filter’s ``writer`` field. Consult the documentation +of the writer to use to see if there are other options that are needed. + +ASCII Writer +------------ + +By default, the ASCII writer outputs log files that begin with several lines of +metadata, followed by the actual log output. The metadata describes the format +of the log file, the ``path`` of the log (i.e., the log filename without file +extension), and also specifies the time that the log was created and the time +when Zeek finished writing to it. The ASCII writer has a number of options for +customizing the format of its output, see +:doc:`/scripts/base/frameworks/logging/writers/ascii.zeek`. If you change the +output format options, then be careful to check whether your post-processing +scripts can still recognize your log files. + +Some writer options are global (i.e., they affect all log filters using that +log writer). For example, to change the output format of all ASCII logs to JSON +format: + +.. code-block:: zeek + + redef LogAscii::use_json = T; + + +Some writer options are filter-specific (i.e., they affect only the filters +that explicitly specify the option). For example, to change the output format +of the :file:`conn.log` only: + +.. code-block:: zeek + + event zeek_init() + { + local f = Log::get_filter(Conn::LOG, "default"); + # Use tab-separated-value mode + f$config = table(["tsv"] = "T"); + Log::add_filter(Conn::LOG, f); + } + +.. _logging-sqlite-writer: + +SQLite Writer +------------- + +SQLite is a simple, file-based, widely used SQL database system. Using SQLite +allows Zeek to write and access data in a format that is easy to use in +interchange with other applications. Due to the transactional nature of SQLite, +databases can be used by several applications simultaneously. Zeek’s input +framework supports a :ref:`SQLite reader `. + +Logging support for SQLite is available in all Zeek installations. There is no +need to load any additional scripts or for any compile-time configurations. +Sending data from existing logging streams to SQLite is rather straightforward. +Most likely you’ll want SQLite output only for select log filters, so you have +to configure one to use the SQLite writer. The following example code adds +SQLite as a filter for the connection log: + +.. code-block:: zeek + + event zeek_init() + { + local filter: Log::Filter = + [ + $name="sqlite", + $path="/var/db/conn", + $config=table(["tablename"] = "conn"), + $writer=Log::WRITER_SQLITE + ]; + + Log::add_filter(Conn::LOG, filter); + } + +Zeek will create the database file :file:`/var/db/conn.sqlite` if it does not +already exist. It will also create a table with the name ``conn`` (if it does +not exist) and start appending connection information to the table. + +Zeek does not currently support rotating SQLite databases as it does for ASCII +logs. You have to take care to create them in adequate locations. + +If you examine the resulting SQLite database, the schema will contain the same +fields that are present in the ASCII log files: + +.. code-block:: console + + sqlite3 /var/db/conn.sqlite + +:: + + SQLite version 3.8.0.2 2013-09-03 17:11:13 + Enter ".help" for instructions + Enter SQL statements terminated with a ";" + sqlite> .schema + CREATE TABLE conn ( + 'ts' double precision, + 'uid' text, + 'id.orig_h' text, + 'id.orig_p' integer, + ... + +Note that with the above code the ASCII :file:`conn.log` will still be created, +because it adds an additional log filter alongside the default, ASCII-logging +one. To prevent this you can remove the default filter: + +.. code-block:: zeek + + Log::remove_filter(Conn::LOG, "default"); + +To create a custom SQLite log file, you have to create a new log stream that +contains just the information you want to commit to the database. See the above +documentation on how to create custom log streams. + +None Writer +----------- + +The ``None`` writer, selected via :zeek:see:`Log::WRITER_NONE`, is largely a +troubleshooting and development aide. It discards all log entries it receives, +but behaves like a proper writer to the rest of the logging framework, +including, for example, pretended log rotation. If you enable its debugging +mode by setting :zeek:see:`LogNone::debug` to ``T``, Zeek reports operational +details about the writer’s activity to ``stdout``. diff --git a/doc/frameworks/logging/delay1.zeek b/doc/frameworks/logging/delay1.zeek new file mode 100644 index 0000000000..ae75bd76ca --- /dev/null +++ b/doc/frameworks/logging/delay1.zeek @@ -0,0 +1,37 @@ +@load base/protocols/conn + +redef record Conn::Info += { + orig_name: string &log &optional; + resp_name: string &log &optional; +}; + +hook Log::log_stream_policy(rec: Conn::Info, id: Log::ID) + { + if ( id != Conn::LOG ) + return; + + local token1 = Log::delay(id, rec); + local token2 = Log::delay(id, rec); + + when [id, rec, token1] ( local orig_name = lookup_addr(rec$id$orig_h) ) + { + rec$orig_name = orig_name; + Log::delay_finish(id, rec, token1); + } + timeout 150msec + { + Reporter::warning(fmt("lookup_addr timeout for %s", rec$id$orig_h)); + Log::delay_finish(id, rec, token1); + } + + when [id, rec, token2] ( local resp_name = lookup_addr(rec$id$resp_h) ) + { + rec$resp_name = resp_name; + Log::delay_finish(id, rec, token2); + } + timeout 150msec + { + Reporter::warning(fmt("lookup_addr timeout for %s", rec$id$resp_h)); + Log::delay_finish(id, rec, token2); + } + } diff --git a/doc/frameworks/management.rst b/doc/frameworks/management.rst new file mode 100644 index 0000000000..f8d7499339 --- /dev/null +++ b/doc/frameworks/management.rst @@ -0,0 +1,867 @@ +.. _framework-management: + +==================== +Management Framework +==================== + +.. rst-class:: opening + + The management framework provides a Zeek-based, service-oriented architecture + and event-driven APIs to manage a Zeek cluster that monitors live traffic. It + provides a central, stateful *controller* that relays and orchestrates + cluster management tasks across connected *agents*. Each agent manages Zeek + processes in its local *instance*, the Zeek process tree controlled by the + local Zeek :ref:`Supervisor `. A management *client* + lets the user interact with the controller to initiate cluster management + tasks, such as deployment of cluster configurations, monitoring of + operational aspects, or to restart cluster nodes. The default client is + ``zeek-client``, included in the Zeek distribution. + +.. _framework-management-quickstart: + +Quickstart +========== + +Run the following (as root) to launch an all-in-one management instance on your +system: + +.. code-block:: console + + # zeek -C -j policy/frameworks/management/controller policy/frameworks/management/agent + +The above will stay in the foreground. In a new shell, save the following +content to a file ``cluster.cfg`` and adapt the workers' sniffing interfaces to +your system: + +.. literalinclude:: management/mini-config.ini + :language: ini + +Run the following command (as any user) to deploy the configuration: + +.. literalinclude:: management/mini-deployment.console + :language: console + +You are now running a Zeek cluster on your system. Try ``zeek-client get-nodes`` +to see more details about the cluster's current status. (In the above, "testbox" +is the system's hostname.) + +Architecture and Terminology +============================ + +Controller +---------- + +The controller forms the central hub of cluster management. It exists once in +every installation and runs as a Zeek process solely dedicated to management +tasks. It awaits instructions from a management client and communicates with one +or more agents to manage their cluster nodes. + +All controller communication happens via :ref:`Broker `-based +Zeek event exchange, usually in the form of request-response event pairs tagged +with a request ID to provide context. The controller is stateful and persists +cluster configurations to disk. In a multi-system setup, the controller runs +inside a separate, dedicated Zeek instance. In a single-system setup, the +controller can run as an additional process in the local instance. + +The controller's API resides in the :zeek:see:`Management::Controller::API` module. +Additional code documentation is :doc:`here `. + +Instance +-------- + +A Zeek instance comprises the set of processes managed by a Zeek +:ref:`Supervisor `. The management framework builds +heavily on the Supervisor framework and cannot run without it. Typically, a +single instance includes all Zeek processes on the local system (a physical +machine, a container, etc), but running multiple instances on a system is +possible. + +Agent +----- + +Management agents implement instance-level cluster management tasks. Every +instance participating in cluster management runs an agent. Agents peer with the +controller to receive instructions (a node restart, say), carry them out, and +respond with the outcome. The direction of connection establishment for the +peering depends on configuration and can go either way (more on this below); by +default, agents connect to the controller. + +The agent's API resides in the :zeek:see:`Management::Agent::API` module. +Additional code documentation is :doc:`here `. + +Agents add script-layer code to both the Supervisor (details :doc:`here +`) and Zeek cluster +nodes (details :doc:`here `) +to enable management tasks (e.g. to tap into node stdout/stderr output) and to +receive confirmation of successful node startup. + +Cluster nodes +------------- + +The Zeek processes involved in traffic analysis and log output make up the Zeek +*cluster*, via the :ref:`cluster framework `. The management +framework does not change the cluster framework, and all of its concepts (the +manager, logger(s), workers, etc) apply as before. Cluster *nodes* refer to +individual Zeek processes in the cluster, as managed by the Supervisor. + +Client +------ + +The management client provides the user's interface to cluster management. It +allows configuration and deployment of the Zeek cluster, insight into the +running cluster, the ability to restart nodes, etc. The client uses the +controller's event API to communicate and is the only component in the framework +not (necessarily) implemented in Zeek's script layer. The Zeek distribution +ships with ``zeek-client``, a command-line client implemented in Python, to +provide management functionality. Users are welcome to implement other clients. + +.. _framework-management-visual-example: + +A Visual Example +================ + +Consider the following setup, consisting of a single instance, controller, and a +connected ``zeek-client``, all running on different machines: + +.. image:: /images/management.png + :align: center + +The cluster system runs a single management instance, with an agent listening on +TCP port 2151, the default. Since the agent needs to communicate with the +Supervisor for node management tasks and the two run in separate processes, the +Supervisor listens for Broker peerings, on TCP port 9999 (again, the default), +and the two communicate events over topic ``zeek/supervisor``. As shown, the +agent has launched a 4-node Zeek cluster consisting of two workers, a logger, +and a manager, communicating internally as usual. + +The controller system is more straightforward, consisting merely of a +Supervisor-governed management controller. This controller has connected to and +peered with the agent on the cluster system, to relay commands received by the +client via the agent's API and receive responses over Broker topic +``zeek/management/agent``. Since the controller doesn't need to interact with +the Supervisor, the latter doesn't listen on any ports. Standalone controllers, +as running here, still require a Supervisor, to simplify co-located deployment +of agent and controller in a single instance. + +Finally, the admin system doesn't run Zeek, but has it installed to provide +``zeek-client``, the CLI for issuing cluster management requests. This client +connects to and peers with the controller, exchanging controller API events over +topic ``zeek/management/controller``. For more details on ``zeek-client``, see +:ref:`below `. + +In practice you can simplify the deployment by running ``zeek-client`` directly +on the controller machine, or by running agent and controller jointly on a +single system. We cover this in :ref:`more detail +`. + +Goals and Relationship to ZeekControl +===================================== + +The management framework first shipped in usable form in Zeek 5.0. It will +replace the aging :ref:`ZeekControl ` over the course of +the coming releases. The framework is not compatible with ZeekControl's approach +to cluster management: use one or the other, not both. + +The framework currently targets single-instance deployments, i.e., setups in +which traffic monitoring happens on a single system. While the management +framework technically supports clusters spanning multiple monitoring systems, +much of the infrastructure users know from ``zeekctl`` (such as the ability to +deploy Zeek scripts and additional configuration) is not yet available in the +management framework. + +ZeekControl remains included in the Zeek distribution, and remains the +recommended solution for multi-system clusters and those needing rich management +capabilities. + +.. _framework-management-running: + +Running Controller and Agent +============================ + +.. _joint-launch: + +Joint launch +------------ + +The easiest approach is to run a single Zeek instance in which the Supervisor +launches both an agent and the controller. The framework comes pre-configured for +this use-case. Its invocation looks as follows: + +.. code-block:: console + + # zeek -j policy/frameworks/management/controller policy/frameworks/management/agent + +The ``-j`` flag enables the Supervisor and is required for successful launch of +the framework. (Without it, the above command will simply return.) + +.. note:: + + If you're planning to monitor the machine's own traffic, add the ``-C`` flag + to avoid checksum errors, which commonly happen in local monitoring due to + offload of the checksum computation to the NIC. + +The following illustrates this setup: + +.. image:: /images/management-all-in-one.png + :align: center + :scale: 75% + +Separate controller and agent instances +--------------------------------------- + +You can also separate the agent and controller instances. For this, you'd say + +.. code-block:: console + + # zeek -j policy/frameworks/management/agent + +for the agent, and + +.. code-block:: console + + # zeek -j policy/frameworks/management/controller + +for the controller. You can run the latter as a regular user, assuming the user +has write access to the installation's spool and log directories (more on this +below). While technically not required to operate a stand-alone controller, the +Supervisor is currently also required in this scenario, so don't omit the +``-j``. + +This looks as follows: + +.. image:: /images/management-all-in-one-two-zeeks.png + :align: center + + +Controller and agent instances on separate systems +-------------------------------------------------- + +You can also separate the two across different systems, though that approach +will only really start to make sense when the framework fully supports running +multiple traffic-sniffing instances. To do this, you either need to configure +the agent to find the controller, or tell the controller where to find the +agent. For the former, redefine the corresponding config setting, for example by +saying + +.. code-block:: zeek + + redef Management::Agent::controller = [$address="1.2.3.4", $bound_port=21500/tcp]; + +in ``local.zeek`` and then launching + +.. code-block:: console + + # zeek -j policy/frameworks/management/agent local + +The result looks as already covered :ref:`earlier `: + +.. image:: /images/management.png + :align: center + +To make the controller connect to remote agents, deploy configurations that +include the location of such agents in the configuration. More on this below. + +Multiple instances +------------------ + +You can run multiple instances on a single system, but it requires some +care. Doing so requires specifying a different listening port for each agent, +and additionally providing a different listening port for each instance's +Supervisor. Since agents communicate with their Supervisor to facilitate node +management, the Supervisor needs to listen (though only locally). Furthermore, +you need to ensure this agent runs with a unique name (see the next section for +more on naming). + +Assuming you already have an instance running, a launch of an additional agent +might look as follows: + +.. code-block:: console + + # zeek -j policy/frameworks/management/agent \ + Management::Agent::default_port=2152/tcp \ + Management::Agent::name=agent-standby \ + Broker::default_port=10001/tcp + +Finally, as already mentioned, you can spread multiple instances across multiple +systems to explore distributed cluster management. This simplifies the +individual launch invocations, but for practical distributed cluster use you may +find the framework's current cluster management features lacking when compared +to ZeekControl. + +Controller and agent naming +--------------------------- + +The management framework identifies all nodes in the system by name, and all +nodes (agent(s), controller, and Zeek cluster nodes) must have unique names. By +default, the framework chooses ``agent-`` and +``controller-`` for agent and controller, respectively. To reconfigure +naming, set the ``ZEEK_AGENT_NAME`` / ``ZEEK_CONTROLLER_NAME`` environment +variables, or redefine the following: + +.. code-block:: zeek + + redef Management::Controller::name = "controller1"; + redef Management::Agent::name = "agent1"; + +Firewalling and encryption +-------------------------- + +By default, the controller listens for clients and agents on ports ``2149/tcp`` and +``2150/tcp``. The former port supports Broker's WebSocket data format, the latter its +traditional one. +Unless you run all components, including the client, on a single system, you'll +want to open up these ports on the controller's system. The agent's default port +is ``2151/tcp``. It always listens; this allows cluster nodes to connect to it +to send status reports. If the agents connect to the controller, your firewall +may block the agent's port since host-local connectivity from cluster nodes to +the agent process suffices. + +To switch agent and/or controller to different ports, set environment variables +``ZEEK_CONTROLLER_PORT`` / ``ZEEK_CONTROLLER_WEBSOCKET_PORT`` / ``ZEEK_AGENT_PORT``, +or use the following: + +.. code-block:: zeek + + redef Management::Controller::default_port_websocket = 21490/tcp; + redef Management::Controller::default_port = 21500/tcp; + redef Management::Agent::default_port = 21510/tcp; + +By default, agent and controller listen globally. To make them listen on a +specific interface, set environment variables ``ZEEK_CONTROLLER_ADDR`` / +``ZEEK_CONTROLLER_WEBSOCKET_ADDR`` / ``ZEEK_AGENT_ADDR``, +or redefine the framework's fallback default address: + +.. code-block:: zeek + + redef Management::default_address = "127.0.0.1"; + +The framework inherits Broker's TLS capabilities and defaults. For details, +please refer to the :doc:`Broker config settings +`. + +.. note:: + + ``zeek-client`` currently doesn't support client-side certificates. + +Additional framework configuration +---------------------------------- + +The framework features a number of additional settings that we cover as needed +in the remainder of this chapter. Refer to the following to browse them all: + +* :doc:`General settings ` +* :doc:`Controller ` +* :doc:`Agents ` +* :doc:`Cluster nodes ` +* :doc:`Supervisor ` + +Node Operation and Outputs +========================== + +The framework places every Supervisor-created node into its own working +directory, located in ``$(zeek-config --prefix)/var/lib/nodes/``. You can +reconfigure this by setting the ``ZEEK_MANAGEMENT_STATE_DIR`` or redefining +:zeek:see:`Management::state_dir`. Doing either will change the toplevel +directory (i.e., replacing the path up to and including ``var/lib`` in the +above); the framework will still create the ``nodes/`` directory structure +within it. + +Outputs in the resulting directory include: + +* Two separate ad-hoc logs (not structured by Zeek's logging framework) + capturing the node's stdout and stderr streams. Their naming is configurable, + defaulting simply to ``stdout`` and ``stderr``. + +* Zeek log files prior to log rotation. + +* Persisted Zeek state, such as Broker-backed tables. + + +Log Management +============== + +The framework configures log rotation and archival via Zeek's included +`zeek-archiver tool `_, as follows: + +* The :zeek:see:`Log::default_rotation_interval` is one hour, with both local + and remote logging enabled. You are free to adjust it as needed. + +* The log rotation directory defaults to ``$(zeek-config --prefix)/spool/log-queue``. + To adjust this, redefine :zeek:see:`Log::default_rotation_dir` as usual. + You can also relocate the spool by setting the ``ZEEK_MANAGEMENT_SPOOL_DIR`` + environment variable or redefining :zeek:see:`Management::spool_dir`. The + framework will place ``log-queue`` into that new destination. + +* The log rotation callback rotates node-local logs into the log queue, with + naming suitable for ``zeek-archiver``. An example: + + .. code-block:: console + + conn__2022-06-20-10-00-00__2022-06-20-11-00-00__.log + + For details, take a look at the implementation in + ``scripts/policy/frameworks/management/persistence.zeek``. + +* Once per log rotation interval, the agent launches log archival to archive + rotated logs into the installation's log directory (``$(zeek-config + --root)/logs``). By default this invokes ``zeek-archiver``, which establishes + a datestamp directory in the ``logs`` directory and places the compressed logs + into it: + + .. code-block:: console + + # cd $(zeek-config --root)/logs + # ls -l + total 4 + drwx------. 2 root root 4096 Jun 20 21:17 2022-06-20 + # cd 2022-06-20 + # ls -l + total 712 + -rw-r--r--. 1 root root 280 Jun 20 20:17 broker.19:00:00-20:00:00.log.gz + -rw-r--r--. 1 root root 24803 Jun 20 20:17 conn.19:00:00-20:00:00.log.gz + -rw-r--r--. 1 root root 26036 Jun 20 21:17 conn.20:00:00-21:00:00.log.gz + -rw-r--r--. 1 root root 350 Jun 20 20:17 dhcp.19:00:00-20:00:00.log.gz + -rw-r--r--. 1 root root 400 Jun 20 21:17 dhcp.20:00:00-21:00:00.log.gz + ... + +You can adapt the log archival configuration via the following settings: + +* Redefine :zeek:see:`Management::Agent::archive_logs` to ``F`` to disable + archival entirely. + +* Redefine :zeek:see:`Management::Agent::archive_interval` for an interval other + than the log rotation one. + +* Redefine :zeek:see:`Management::Agent::archive_dir` to change the + destination directory. + +* Redefine :zeek:see:`Management::Agent::archive_cmd` to invoke an executable + other than the included ``zeek-archiver``. The replacement should accept the + same argument structure: `` -1 ``. The + ``-1`` here refers to ``zeek-archiver``'s one-shot processing mode. + +.. _framework-management-zeek-client: + +The zeek-client CLI +=================== + +Zeek ships with a command-line client for the Management framework: +``zeek-client``, installed alongside the other executables in the +distribution. It looks as follows: + +.. literalinclude:: management/zeek-client-help.console + :language: console + +Run commands with ``--help`` for additional details. + +The majority of ``zeek-client``'s commands send off a request to the controller, +wait for it to act on it, retrieve the response, and render it to the +console. The output is typically in JSON format, though a few commands also +support ``.ini`` output. + +Looking at the :zeek:see:`Management::Controller::API` module, you'll notice +that the structure of response event arguments is fairly rigid, consisting of +one or more :zeek:see:`Management::Result` records. ``zeek-client`` does not +render these directly to JSON. Instead, it translates the responses to a more +convenient JSON format reflecting specific types of requests. Several commands +share a common output format. + +.. _zeek-client-installation: + +Standalone installation +----------------------- + +As mentioned above, Zeek ships with ``zeek-client`` by default. Since users will +often want to use the client from machines not otherwise running Zeek, the +client is also available as a standalone Python package via ``pip``: + +.. code-block:: console + + $ pip install zeek-client + +Users with custom Zeek builds who don't require a Zeek-bundled ``zeek-client`` +can skip its installation by configuring their build with +``--disable-zeek-client``. + +.. _zeek-client-compatibility: + +Compatibility +------------- + +Zeek 5.2 switched client/controller communication from Broker's native wire +format to the newer `WebSocket data transport +`_, with +``zeek-client`` 1.2.0 being the first version to exclusively use WebSockets. +This has a few implications: + +* Since Broker dedicates separate ports to the respective wire formats, the + controller listens on TCP port 2149 for WebSocket connections, while + TCP port 2150 remains available for connections by native-Broker clients, as well + as by management agents connecting to the controller. + +* ``zeek-client`` 1.2.0 and newer default to connecting to port 2149. + +* Controllers running Zeek older than 5.2 need tweaking to listen on a WebSocket + port, for example by saying: + + .. code-block:: console + + event zeek_init() + { + Broker::listen_websocket("0.0.0.0", 2149/tcp); + } + +* Older clients continue to work with Zeek 5.2 and newer. + +.. _zeek-client-configuration: + +Configuration +------------- + +The client features a handful of configuration settings, reported when running +``zeek-client show-settings``: + +.. literalinclude:: management/zeek-client-show-settings.console + :language: console + +You can override these via a configuration file, the environment variable +``ZEEK_CLIENT_CONFIG_SETTINGS``, and the ``--set`` command-line argument, in +order of increasing precedence. To identify a setting, use +``
    .``, as shown by your client. For example, in order to +specify a controller's location on the network, you could: + +* Put the following in a config file, either at its default location shown in + the help output (usually ``$(zeek-config --prefix)/etc/zeek-client.cfg``) + or one that you provide via ``-c``/``--configfile``: + + .. code-block:: ini + + [controller] + host = mycontroller + port = 21490 + +* Set the environment: + + .. code-block:: console + + ZEEK_CLIENT_CONFIG_SETTINGS="controller.host=mycontroller controller.port=21490" + +* Use the ``--set`` option, possibly repeatedly: + + .. code-block:: console + + $ zeek-client --set controller.host=mycontroller --set controller.port=21490 ... + +Other than the controller coordinates, the settings should rarely require +changing. If you're curious about their meaning, please consult the `source code +`_. + +Auto-complete +------------- + +On systems with an installed `argcomplete `_ +package, ``zeek-client`` features command-line auto-completion. For example: + +.. code-block:: console + + $ zeek-client --set controller. + controller.host=127.0.0.1 controller.port=2149 + +Common cluster management tasks +=============================== + +With a running controller and agent, it's time start using ``zeek-client`` for +actual cluster management tasks. By default, the client will connect to a +controller running on the local system. If that doesn't match your setting, +instruct the client to contact the controller via one of the approaches shown +:ref:`earlier `. + +Checking connected agents +------------------------- + +Use ``zeek-client get-instances`` to get a summary of agents currently peered +with the controller: + +.. code-block:: console + + $ zeek-client get-instances + { + "agent-testbox": { + "host": "127.0.0.1" + } + } + +For agents connecting to the controller you'll see the above output; for agents +the controller connected to you'll also see those agent's listening ports. + +Defining a cluster configuration +-------------------------------- + +For ``zeek-client``, cluster configurations are simple ``.ini`` files with two +types of sections: the special ``instances`` section defines the instances +involved in the cluster, represented by their agents. All other sections in the +file name individual cluster nodes and describe their roles and properties. + +Here's a full-featured configuration describing the available options, assuming +a single agent running on a machine "testbox" with default settings: + +.. literalinclude:: management/full-config.ini + :language: ini + +.. _simplification-instance-local: + +Simplification for instance-local deployment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In practice you can omit many of the settings. We already saw in the +:ref:`Quickstart ` section that a configuration +deployed locally in a :ref:`joint agent-controller setup ` need +not specify any instances at all. In that case, use of the local instance +``agent-`` is implied. If you use other agent naming or more complex +setups, every node needs to specify its instance. + +Simplification for agent-to-controller connectivity +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In setups where agents connect to the controller, you may omit the ``instances`` +section if it would merely repeat the list of instances claimed by the nodes. + +Simplification for port selection +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +All but the worker nodes in a Zeek cluster require a listening port, and you can +specify one for each node as shown in the above configuration. If you'd rather +not pick ports, the controller can auto-enumerate ports for you, as follows: + +* The :zeek:see:`Management::Controller::auto_assign_broker_ports` Boolean, which defaults to + ``T``, controls whether port auto-enumeration is active. Redefining to ``F`` + disables the feature. + +* :zeek:see:`Management::Controller::auto_assign_broker_start_port` defines the starting point + for port enumeration. This defaults to ``2200/tcp``. + +* Any nodes with explicitly configured ports will keep them. + +* For other nodes, the controller will assign ports first to the manager, then + logger(s), then proxies. Within each of those groups, it first groups nodes + in the same instance (to obtain locally sequential ports), and orders these + alphabetically by name before enumerating. It also avoids conflicts with + configured agent and controller ports. + +* The controller does not verify that selected ports are in fact unclaimed. + It's up to you to ensure an available pool of unclaimed listening ports from + the start port onward. + +By retrieving the deployed configuration from the controller (see the next two +sections) you can examine which ports the controller selected. + +Configuration of the Telemetry framework +---------------------------------------- + +By default, the framework will enable Prometheus metrics exposition ports, +including a service discovery endpoint on the manager (refer to the +:ref:`Telemetry Framework ` for details), and +auto-assign them for you. Specifically, the controller will enumerate ports +starting from +:zeek:see:`Management::Controller::auto_assign_metrics_start_port`, which +defaults to ``9000/tcp``. Any ports you define manually will be preserved. To +disable metrics port auto-assignment, redefine +:zeek:see:`Management::Controller::auto_assign_metrics_ports` to ``F``. + +Staging and deploying configurations +------------------------------------ + +The framework structures deployment of a cluster configuration into two +phases: + +#. First, the cluster configuration is *staged*: the client uploads it to the + controller, which validates its content, and --- upon successful validation + --- persists this configuration to disk. Restarting the controller at this + point will preserve this configuration in its staged state. Validation checks + the configuration for consistency and structural errors, such as doubly + defined nodes, port collisions, or inconsistent instance use. The controller + only ever stores a single staged configuration. + +#. Then, *deployment* applies needed finalization to the configuration (e.g. to + auto-enumerate ports) and, assuming all needed instances have peered, + distributes the configuration to their agents. Deployment replaces any + preexisting Zeek cluster, shutting down the existing node processes. The + controller also persists the deployed configuration to disk, alongside the + staged one. Deployment does *not* need to be successful to preserve a + deployed configuration: it's the attempt to deploy that matters. + +Internally, configurations bear an identifier string to allow tracking. The +client selects this identifier, which comes with no further assurances --- for +example, identical configurations need not bear the same identifier. + +To stage a configuration, use the following: + +.. code-block:: console + + $ zeek-client stage-config cluster.cfg + { + "errors": [], + "results": { + "id": "5e90197a-f850-11ec-a77f-7c10c94416bb" + } + } + +The ``errors`` array contains textual description of any validation problems +encountered, causing the client to exit with error. The reported ``id`` is the +configuration's identifier, as set by the client. + +Then, trigger deployment of the staged configuration: + +.. code-block:: console + + $ zeek-client deploy + { + "errors": [], + "results": { + "id": "5e90197a-f850-11ec-a77f-7c10c94416bb" + "nodes": { + "logger": { + "instance": "agent-testbox4", + "success": true + }, + "manager": { + "instance": "agent-testbox4", + "success": true + }, + "worker-01": { + "instance": "agent-testbox4", + "success": true + }, + "worker-02": { + "instance": "agent-testbox4", + "success": true + } + } + } + } + +Success! Note the matching identifiers. The errors array covers any internal +problems, and per-node summaries report the deployment outcome. In case of +launch errors in individual nodes, stdout/stderr is captured and hopefully +provides clues. Revisiting the quickstart example, let's introduce an error in +``cluster.cfg``: + +.. literalinclude:: management/mini-config-with-error.ini + :language: ini + +Since staging and deployment will frequently go hand-in-hand, the client +provides the ``deploy-config`` command to combine them into one. Let's use it: + +.. literalinclude:: management/mini-deployment-error.console + :language: console + +The client exits with error, the timeout error refers to the fact that one of +the launch commands timed out, and ``worker-02``'s stderr shows the problem. The +Supervisor will continue to try to launch the node with ever-increasing +reattempt delays, and keep failing. + +Retrieving configurations +------------------------- + +The client's ``get-config`` command lets you retrieve staged and deployed +configurations from the controller, in JSON or :file:`.ini` form. This is helpful for +examining the differences between the two. Following the successful deployment +shown above, we have: + +.. literalinclude:: management/mini-deployment-get-config-staged.console + :language: console + +You can see here how the client's :ref:`instance-local simplification +` filled in instances under the hood. + +The ``.ini`` output is reusable as deployable configuration. The same +configuration is available in JSON, showing more detail: + +.. literalinclude:: management/mini-deployment-get-config-staged-json.console + :language: console + +Finally, you can also retrieve the deployed configuration (in either format): + +.. literalinclude:: management/mini-deployment-get-config-deployed.console + :language: console + +Note the manager's and logger's auto-enumerated ports in this one. + +Showing the current instance nodes +---------------------------------- + +To see the current node status as visible to the Supervisors in each agent's +instance, use the ``get-nodes`` command: + +.. literalinclude:: management/mini-deployment-get-nodes.console + :language: console + +This groups nodes by instances, while also showing agents and controllers, so +``agent-testbox`` shows up twice in the above. Nodes can be in two states, +``PENDING`` upon launch and before the new node has checked in with the agent, +and ``RUNNING`` once that has happened. Nodes also have a role either in cluster +management (as ``AGENT`` or ``CONTROLLER``), or in the Zeek cluster. The +information shown per node essentially reflects the framework's +:zeek:see:`Management::NodeStatus` record. + +Showing current global identifier values +---------------------------------------- + +For troubleshooting scripts in production it can be very handy to verify the +contents of global variables in specific nodes. The client supports this via the +``get-id-value`` command. To use it, specify the name of a global identifier, as +well as any node names from which you'd like to retrieve it. The framework +renders the value to JSON directly in the queried cluster node, side-stepping +potential serialization issues for complex types, and integrates the result in +the response: + +.. literalinclude:: management/get-id-value-simple.console + :language: console + +.. literalinclude:: management/get-id-value-complex.console + :language: console + +Restarting cluster nodes +------------------------ + +The ``restart`` command allows you to restart specific cluster nodes, or the +entire cluster. Note that this refers only to the operational cluster (manager, +workers, etc) --- this will not restart any agents or a co-located controller. + +Here's the current manager: + +.. code-block:: console + + $ zeek-client get-nodes | jq '.results."agent-testbox".manager' + { + "cluster_role": "MANAGER", + "mgmt_role": null, + "pid": 54073, + "port": 2200, + "state": "RUNNING" + } + +Let's restart it: + +.. code-block:: console + + $ zeek-client restart manager + { + "errors": [], + "results": { + "manager": true + } + } + +It's back up and running (note the PID change): + +.. code-block:: console + + $ zeek-client get-nodes | jq '.results."agent-testbox".manager' + { + "cluster_role": "MANAGER", + "mgmt_role": null, + "pid": 68752, + "port": 2200, + "state": "RUNNING" + } diff --git a/doc/frameworks/management/full-config.ini b/doc/frameworks/management/full-config.ini new file mode 100644 index 0000000000..c4c3912d5e --- /dev/null +++ b/doc/frameworks/management/full-config.ini @@ -0,0 +1,88 @@ +# The instances section describes where you run Management agents +# and whether these agents connect to the controller, or the controller +# connects to them. Each instance (or, equivalently, the agent running +# on it) is identified by a unique name. The names in this configuration +# must match the names the agents use in the Zeek configuration. Without +# customization, that name is "agent-". +[instances] +# A value-less entry means this agent connects to the controller: +agent-testbox + +# An entry with a value of the form ":" means the controller will +# connect to them. +# +# agent-testbox = 12.34.56.78:1234 + +# All other sections identify Zeek cluster nodes. The section name sets +# the name of the node: +[manager] + +# Nodes typically state which instance they run on: +instance = agent-testbox + +# Every node needs to define its role. Possible values are "manager", +# "logger", "proxy", and "worker". +role = manager + +# For nodes that require a listening port (all roles but workers), +# you can choose to define a port. If you omit it, the framework will +# define ports for you. Only give a number; TCP is implied. +# +# port = 1234 + +# You can optionally specify explicit metrics exposition ports for each +# node. If you omit these, the framework (specifically, the controller) +# will define ports for you. Only give a number; TCP is implied. +# +# metrics_port = 9090 + +# You can provide additional scripts that a node should run. These scripts +# must be available on the instance. Space-separate multiple scripts. +# +# scripts = policy/tuning/json-logs policy/misc/loaded-scripts + +# You can define environment variables for the node. List them as =, +# space-separated if you provide multiple. If the value has whitespace, say +# ="' +# +# env = FOO=BAR + +# For workers, specify a sniffing interface as follows: +# +# interface = + +# To express CPU affinity, use the following: +# +# cpu_affinity = + +[logger] +instance = agent-testbox +role = logger + +[proxy1] +instance = agent-testbox +role = proxy + +[proxy2] +instance = agent-testbox +role = proxy + +[worker1] +instance = agent-testbox +role = worker +interface = eth0 + +[worker2] +instance = agent-testbox +role = worker +interface = eth1 + +[worker3] +instance = agent-testbox +role = worker +interface = eth2 + +[worker4] +instance = agent-testbox +role = worker +interface = eth3 diff --git a/doc/frameworks/management/get-id-value-complex.console b/doc/frameworks/management/get-id-value-complex.console new file mode 100644 index 0000000000..34657be5dc --- /dev/null +++ b/doc/frameworks/management/get-id-value-complex.console @@ -0,0 +1,31 @@ +$ zeek-client get-id-value Log::all_streams worker-01 +{ + "errors": [], + "results": { + "worker-01": { + "Broker::LOG": { + "columns": null, + "path": "broker", + "policy": "Broker::log_policy" + }, + "Cluster::LOG": { + "columns": null, + "path": "cluster", + "policy": "Cluster::log_policy" + }, + ... + "X509::LOG": { + "columns": null, + "ev": "X509::log_x509", + "path": "x509", + "policy": "X509::log_policy" + }, + "mysql::LOG": { + "columns": null, + "ev": "MySQL::log_mysql", + "path": "mysql", + "policy": "MySQL::log_policy" + } + } + } +} diff --git a/doc/frameworks/management/get-id-value-simple.console b/doc/frameworks/management/get-id-value-simple.console new file mode 100644 index 0000000000..e8ce2a69f1 --- /dev/null +++ b/doc/frameworks/management/get-id-value-simple.console @@ -0,0 +1,10 @@ +$ zeek-client get-id-value LogAscii::use_json +{ + "errors": [], + "results": { + "logger": false, + "manager": false, + "worker-01": false, + "worker-02": false + } +} diff --git a/doc/frameworks/management/mini-config-with-error.ini b/doc/frameworks/management/mini-config-with-error.ini new file mode 100644 index 0000000000..9f5f27c109 --- /dev/null +++ b/doc/frameworks/management/mini-config-with-error.ini @@ -0,0 +1,13 @@ +[manager] +role = manager + +[logger] +role = logger + +[worker-01] +role = worker +interface = lo + +[worker-02] +role = worker +interface = not-a-valid-interface diff --git a/doc/frameworks/management/mini-config.ini b/doc/frameworks/management/mini-config.ini new file mode 100644 index 0000000000..51bd5ad8d7 --- /dev/null +++ b/doc/frameworks/management/mini-config.ini @@ -0,0 +1,13 @@ +[manager] +role = manager + +[logger] +role = logger + +[worker-01] +role = worker +interface = lo + +[worker-02] +role = worker +interface = eth0 diff --git a/doc/frameworks/management/mini-deployment-error.console b/doc/frameworks/management/mini-deployment-error.console new file mode 100644 index 0000000000..6ced5e61ea --- /dev/null +++ b/doc/frameworks/management/mini-deployment-error.console @@ -0,0 +1,29 @@ +$ zeek-client deploy-config cluster.cfg +{ + "errors": [ + "request timed out" + ], + "results": { + "id": "eed87b02-f851-11ec-80e7-7c10c94416bb", + "nodes": { + "logger": { + "instance": "agent-testbox", + "success": true + }, + "manager": { + "instance": "agent-testbox", + "success": true + }, + "worker-01": { + "instance": "agent-testbox", + "success": true + }, + "worker-02": { + "instance": "agent-testbox", + "stderr": "fatal error: problem with interface not-a-valid-interface (pcap_error: No such device exists (pcap_activate))", + "stdout": "", + "success": false + } + } + } +} diff --git a/doc/frameworks/management/mini-deployment-get-config-deployed.console b/doc/frameworks/management/mini-deployment-get-config-deployed.console new file mode 100644 index 0000000000..4850b73dba --- /dev/null +++ b/doc/frameworks/management/mini-deployment-get-config-deployed.console @@ -0,0 +1,23 @@ +$ zeek-client get-config --deployed +[instances] +agent-testbox + +[logger] +instance = agent-testbox +role = LOGGER +port = 2201 + +[manager] +instance = agent-testbox +role = MANAGER +port = 2200 + +[worker-01] +instance = agent-testbox +role = WORKER +interface = lo + +[worker-02] +instance = agent-testbox +role = WORKER +interface = eth0 diff --git a/doc/frameworks/management/mini-deployment-get-config-staged-json.console b/doc/frameworks/management/mini-deployment-get-config-staged-json.console new file mode 100644 index 0000000000..8ce19a1853 --- /dev/null +++ b/doc/frameworks/management/mini-deployment-get-config-staged-json.console @@ -0,0 +1,55 @@ +$ zeek-client get-config --as-json +{ + "id": "5e90197a-f850-11ec-a77f-7c10c94416bb", + "instances": [ + { + "name": "agent-testbox" + } + ], + "nodes": [ + { + "cpu_affinity": null, + "env": {}, + "instance": "agent-testbox", + "interface": null, + "name": "logger", + "options": null, + "port": null, + "role": "LOGGER", + "scripts": null + }, + { + "cpu_affinity": null, + "env": {}, + "instance": "agent-testbox", + "interface": null, + "name": "manager", + "options": null, + "port": null, + "role": "MANAGER", + "scripts": null + }, + { + "cpu_affinity": null, + "env": {}, + "instance": "agent-testbox", + "interface": "lo", + "name": "worker-01", + "options": null, + "port": null, + "role": "WORKER", + "scripts": null + }, + { + "cpu_affinity": null, + "env": {}, + "instance": "agent-testbox", + "interface": "eth0", + "name": "worker-02", + "options": null, + "port": null, + "role": "WORKER", + "scripts": null + } + ] +} diff --git a/doc/frameworks/management/mini-deployment-get-config-staged.console b/doc/frameworks/management/mini-deployment-get-config-staged.console new file mode 100644 index 0000000000..f1f2595961 --- /dev/null +++ b/doc/frameworks/management/mini-deployment-get-config-staged.console @@ -0,0 +1,21 @@ +$ zeek-client get-config +[instances] +agent-testbox + +[logger] +instance = agent-testbox +role = LOGGER + +[manager] +instance = agent-testbox +role = MANAGER + +[worker-01] +instance = agent-testbox +role = WORKER +interface = lo + +[worker-02] +instance = agent-testbox +role = WORKER +interface = eth0 diff --git a/doc/frameworks/management/mini-deployment-get-nodes.console b/doc/frameworks/management/mini-deployment-get-nodes.console new file mode 100644 index 0000000000..cdd1776529 --- /dev/null +++ b/doc/frameworks/management/mini-deployment-get-nodes.console @@ -0,0 +1,47 @@ +$ zeek-client get-nodes +{ + "errors": [], + "results": { + "agent-testbox": { + "agent-testbox": { + "cluster_role": null, + "mgmt_role": "AGENT", + "pid": 52076, + "state": "RUNNING" + }, + "controller-testbox": { + "cluster_role": null, + "mgmt_role": "CONTROLLER", + "pid": 52075, + "port": 2151, + "state": "RUNNING" + }, + "logger": { + "cluster_role": "LOGGER", + "mgmt_role": null, + "pid": 54075, + "port": 2201, + "state": "RUNNING" + }, + "manager": { + "cluster_role": "MANAGER", + "mgmt_role": null, + "pid": 54073, + "port": 2200, + "state": "RUNNING" + }, + "worker-01": { + "cluster_role": "WORKER", + "mgmt_role": null, + "pid": 54074, + "state": "RUNNING" + }, + "worker-02": { + "cluster_role": "WORKER", + "mgmt_role": null, + "pid": 54072, + "state": "RUNNING" + } + } + } +} diff --git a/doc/frameworks/management/mini-deployment.console b/doc/frameworks/management/mini-deployment.console new file mode 100644 index 0000000000..90ea8f2490 --- /dev/null +++ b/doc/frameworks/management/mini-deployment.console @@ -0,0 +1,25 @@ +$ zeek-client deploy-config cluster.cfg +{ + "errors": [], + "results": { + "id": "9befc56c-f7e8-11ec-8626-7c10c94416bb", + "nodes": { + "logger": { + "instance": "agent-testbox", + "success": true + }, + "manager": { + "instance": "agent-testbox", + "success": true + }, + "worker-01": { + "instance": "agent-testbox", + "success": true + }, + "worker-02": { + "instance": "agent-testbox", + "success": true + } + } + } +} diff --git a/doc/frameworks/management/zeek-client-help.console b/doc/frameworks/management/zeek-client-help.console new file mode 100644 index 0000000000..cd3d56fd60 --- /dev/null +++ b/doc/frameworks/management/zeek-client-help.console @@ -0,0 +1,45 @@ +$ zeek-client --help +usage: zeek-client [-h] [-c FILE] [--controller HOST:PORT] + [--set SECTION.KEY=VAL] [--quiet | --verbose] [--version] + {deploy,deploy-config,get-config,get-id-value,get-instances,get-nodes,monitor,restart,stage-config,show-settings,test-timeout} + ... + +A Zeek management client + +options: + -h, --help show this help message and exit + -c FILE, --configfile FILE + Path to zeek-client config file. (Default: + /usr/local/zeek/etc/zeek-client.cfg) + --controller HOST:PORT + Address and port of the controller, either of which + may be omitted (default: 127.0.0.1:2149) + --set SECTION.KEY=VAL + Adjust a configuration setting. Can use repeatedly. + See show-settings. + --quiet, -q Suppress informational output to stderr. + --verbose, -v Increase informational output to stderr. Repeat for + more output (e.g. -vvv). + --version Show version number and exit. + +commands: + {deploy,deploy-config,get-config,get-id-value,get-instances,get-nodes,monitor,restart,stage-config,show-settings,test-timeout} + See `zeek-client -h` for per-command usage + info. + deploy Deploy a staged cluster configuration. + deploy-config Upload a cluster configuration and deploy it. + get-config Retrieve staged or deployed cluster configuration. + get-id-value Show the value of a given identifier in Zeek cluster + nodes. + get-instances Show instances connected to the controller. + get-nodes Show active Zeek nodes at each instance. + monitor For troubleshooting: do nothing, just report events. + restart Restart cluster nodes. + stage-config Upload a cluster configuration for later deployment. + show-settings Show zeek-client's own configuration. + test-timeout Send timeout test event. + +environment variables: + + ZEEK_CLIENT_CONFIG_FILE: Same as `--configfile` argument, but lower precedence. + ZEEK_CLIENT_CONFIG_SETTINGS: Same as a space-separated series of `--set` arguments, but lower precedence. diff --git a/doc/frameworks/management/zeek-client-show-settings.console b/doc/frameworks/management/zeek-client-show-settings.console new file mode 100644 index 0000000000..e87a42ada3 --- /dev/null +++ b/doc/frameworks/management/zeek-client-show-settings.console @@ -0,0 +1,14 @@ +$ zeek-client show-settings +[client] +request_timeout_secs = 20 +peer_retry_secs = 1 +peering_status_attempts = 10 +peering_status_retry_delay_secs = 0.5 +rich_logging_format = False +pretty_json = True +verbosity = 0 + +[controller] +host = 127.0.0.1 +port = 2150 + diff --git a/doc/frameworks/netcontrol-1-drop-with-debug.zeek b/doc/frameworks/netcontrol-1-drop-with-debug.zeek new file mode 100644 index 0000000000..d4bbcde042 --- /dev/null +++ b/doc/frameworks/netcontrol-1-drop-with-debug.zeek @@ -0,0 +1,10 @@ +event NetControl::init() + { + local debug_plugin = NetControl::create_debug(T); + NetControl::activate(debug_plugin, 0); + } + +event connection_established(c: connection) + { + NetControl::drop_connection(c$id, 20 secs); + } diff --git a/doc/frameworks/netcontrol-10-use-skeleton.zeek b/doc/frameworks/netcontrol-10-use-skeleton.zeek new file mode 100644 index 0000000000..4f683dfd83 --- /dev/null +++ b/doc/frameworks/netcontrol-10-use-skeleton.zeek @@ -0,0 +1,10 @@ +event NetControl::init() + { + local skeleton_plugin = NetControl::create_skeleton(""); + NetControl::activate(skeleton_plugin, 0); + } + +event connection_established(c: connection) + { + NetControl::drop_connection(c$id, 20 secs); + } diff --git a/doc/frameworks/netcontrol-2-ssh-guesser.zeek b/doc/frameworks/netcontrol-2-ssh-guesser.zeek new file mode 100644 index 0000000000..2b8757cdea --- /dev/null +++ b/doc/frameworks/netcontrol-2-ssh-guesser.zeek @@ -0,0 +1,16 @@ + +@load protocols/ssh/detect-bruteforcing + +redef SSH::password_guesses_limit=10; + +event NetControl::init() + { + local debug_plugin = NetControl::create_debug(T); + NetControl::activate(debug_plugin, 0); + } + +hook Notice::policy(n: Notice::Info) + { + if ( n$note == SSH::Password_Guessing ) + NetControl::drop_address(n$src, 60min); + } diff --git a/doc/frameworks/netcontrol-3-ssh-guesser.zeek b/doc/frameworks/netcontrol-3-ssh-guesser.zeek new file mode 100644 index 0000000000..6f6065350c --- /dev/null +++ b/doc/frameworks/netcontrol-3-ssh-guesser.zeek @@ -0,0 +1,16 @@ + +@load protocols/ssh/detect-bruteforcing + +redef SSH::password_guesses_limit=10; + +event NetControl::init() + { + local debug_plugin = NetControl::create_debug(T); + NetControl::activate(debug_plugin, 0); + } + +hook Notice::policy(n: Notice::Info) + { + if ( n$note == SSH::Password_Guessing ) + add n$actions[Notice::ACTION_DROP]; + } diff --git a/doc/frameworks/netcontrol-4-drop.zeek b/doc/frameworks/netcontrol-4-drop.zeek new file mode 100644 index 0000000000..b95822736d --- /dev/null +++ b/doc/frameworks/netcontrol-4-drop.zeek @@ -0,0 +1,26 @@ +function our_drop_connection(c: conn_id, t: interval) + { + # As a first step, create the NetControl::Entity that we want to block + local e = NetControl::Entity($ty=NetControl::CONNECTION, $conn=c); + # Then, use the entity to create the rule to drop the entity in the forward path + local r = NetControl::Rule($ty=NetControl::DROP, + $target=NetControl::FORWARD, $entity=e, $expire=t); + + # Add the rule + local id = NetControl::add_rule(r); + + if ( id == "" ) + print "Error while dropping"; + } + +event NetControl::init() + { + local debug_plugin = NetControl::create_debug(T); + NetControl::activate(debug_plugin, 0); + } + +event connection_established(c: connection) + { + our_drop_connection(c$id, 20 secs); + } + diff --git a/doc/frameworks/netcontrol-5-hook.zeek b/doc/frameworks/netcontrol-5-hook.zeek new file mode 100644 index 0000000000..dee8d5547a --- /dev/null +++ b/doc/frameworks/netcontrol-5-hook.zeek @@ -0,0 +1,22 @@ +hook NetControl::rule_policy(r: NetControl::Rule) + { + if ( r$ty == NetControl::DROP && + r$entity$ty == NetControl::CONNECTION && + r$entity$conn$orig_h in 192.168.0.0/16 ) + { + print "Ignored connection from", r$entity$conn$orig_h; + break; + } + } + +event NetControl::init() + { + local debug_plugin = NetControl::create_debug(T); + NetControl::activate(debug_plugin, 0); + } + +event connection_established(c: connection) + { + NetControl::drop_connection(c$id, 20 secs); + } + diff --git a/doc/frameworks/netcontrol-6-find.zeek b/doc/frameworks/netcontrol-6-find.zeek new file mode 100644 index 0000000000..9e7677dfac --- /dev/null +++ b/doc/frameworks/netcontrol-6-find.zeek @@ -0,0 +1,17 @@ +event NetControl::init() + { + local netcontrol_debug = NetControl::create_debug(T); + NetControl::activate(netcontrol_debug, 0); + } + +event connection_established(c: connection) + { + if ( |NetControl::find_rules_addr(c$id$orig_h)| > 0 ) + { + print "Rule already exists"; + return; + } + + NetControl::drop_connection(c$id, 20 secs); + print "Rule added"; + } diff --git a/doc/frameworks/netcontrol-7-catch-release.zeek b/doc/frameworks/netcontrol-7-catch-release.zeek new file mode 100644 index 0000000000..189a89e68d --- /dev/null +++ b/doc/frameworks/netcontrol-7-catch-release.zeek @@ -0,0 +1,10 @@ +event NetControl::init() + { + local debug_plugin = NetControl::create_debug(T); + NetControl::activate(debug_plugin, 0); + } + +event connection_established(c: connection) + { + NetControl::drop_address_catch_release(c$id$orig_h); + } diff --git a/doc/frameworks/netcontrol-8-multiple.zeek b/doc/frameworks/netcontrol-8-multiple.zeek new file mode 100644 index 0000000000..4d134a577c --- /dev/null +++ b/doc/frameworks/netcontrol-8-multiple.zeek @@ -0,0 +1,29 @@ +function our_openflow_check(p: NetControl::PluginState, r: NetControl::Rule): bool + { + if ( r$ty == NetControl::DROP && + r$entity$ty == NetControl::ADDRESS && + subnet_width(r$entity$ip) == 32 && + subnet_to_addr(r$entity$ip) in 192.168.17.0/24 ) + return F; + + return T; + } + +event NetControl::init() + { + # Add debug plugin with low priority + local debug_plugin = NetControl::create_debug(T); + NetControl::activate(debug_plugin, 0); + + # Instantiate OpenFlow debug plugin with higher priority + local of_controller = OpenFlow::log_new(42); + local netcontrol_of = NetControl::create_openflow(of_controller, [$check_pred=our_openflow_check]); + NetControl::activate(netcontrol_of, 10); + } + +event NetControl::init_done() + { + NetControl::drop_address(10.0.0.1, 1min); + NetControl::drop_address(192.168.17.2, 1min); + NetControl::drop_address(192.168.18.2, 1min); + } diff --git a/doc/frameworks/netcontrol-9-skeleton.zeek b/doc/frameworks/netcontrol-9-skeleton.zeek new file mode 100644 index 0000000000..f768ba8b0e --- /dev/null +++ b/doc/frameworks/netcontrol-9-skeleton.zeek @@ -0,0 +1,39 @@ +module NetControl; + +export { + ## Instantiates the plugin. + global create_skeleton: function(argument: string) : PluginState; +} + +function skeleton_name(p: PluginState) : string + { + return "NetControl skeleton plugin"; + } + +function skeleton_add_rule_fun(p: PluginState, r: Rule) : bool + { + print "add", r; + event NetControl::rule_added(r, p); + return T; + } + +function skeleton_remove_rule_fun(p: PluginState, r: Rule, reason: string &default="") : bool + { + print "remove", r; + event NetControl::rule_removed(r, p); + return T; + } + +global skeleton_plugin = Plugin( + $name = skeleton_name, + $can_expire = F, + $add_rule = skeleton_add_rule_fun, + $remove_rule = skeleton_remove_rule_fun + ); + +function create_skeleton(argument: string) : PluginState + { + local p = PluginState($plugin=skeleton_plugin); + + return p; + } diff --git a/doc/frameworks/netcontrol-architecture.graffle b/doc/frameworks/netcontrol-architecture.graffle new file mode 100644 index 0000000000..136cbdb2b0 Binary files /dev/null and b/doc/frameworks/netcontrol-architecture.graffle differ diff --git a/doc/frameworks/netcontrol-architecture.png b/doc/frameworks/netcontrol-architecture.png new file mode 100644 index 0000000000..4f7b585c66 Binary files /dev/null and b/doc/frameworks/netcontrol-architecture.png differ diff --git a/doc/frameworks/netcontrol-openflow.graffle b/doc/frameworks/netcontrol-openflow.graffle new file mode 100644 index 0000000000..09a418f716 Binary files /dev/null and b/doc/frameworks/netcontrol-openflow.graffle differ diff --git a/doc/frameworks/netcontrol-openflow.png b/doc/frameworks/netcontrol-openflow.png new file mode 100644 index 0000000000..e760c9b074 Binary files /dev/null and b/doc/frameworks/netcontrol-openflow.png differ diff --git a/doc/frameworks/netcontrol-rules.png b/doc/frameworks/netcontrol-rules.png new file mode 100644 index 0000000000..5141c81ceb Binary files /dev/null and b/doc/frameworks/netcontrol-rules.png differ diff --git a/doc/frameworks/netcontrol.rst b/doc/frameworks/netcontrol.rst new file mode 100644 index 0000000000..70e08174c5 --- /dev/null +++ b/doc/frameworks/netcontrol.rst @@ -0,0 +1,812 @@ + +.. _framework-netcontrol: + +==================== +NetControl Framework +==================== + +.. TODO: integrate BoZ revisions + +.. rst-class:: opening + + Zeek can connect with network devices like, for example, switches + or soft- and hardware firewalls using the NetControl framework. The + NetControl framework provides a flexible, unified interface for active + response and hides the complexity of heterogeneous network equipment + behind a simple task-oriented API, which is easily usable via Zeek + scripts. This section gives an overview of how to use the NetControl + framework in different scenarios; to get a better understanding of how + it can be used in practice, it might be worthwhile to take a look at + the integration tests. + +NetControl Architecture +======================= + +.. figure:: netcontrol-architecture.png + :width: 600 + :align: center + :alt: NetControl framework architecture + :target: ../_images/netcontrol-architecture.png + + NetControl architecture (click to enlarge). + +The basic architecture of the NetControl framework is shown in the figure above. +Conceptually, the NetControl framework sits between the user provided scripts +(which use the Zeek event engine) and the network device (which can either be a +hardware or software device), that is used to implement the commands. + +The NetControl framework supports a number of high-level calls, like the +:zeek:see:`NetControl::drop_address` function, or a lower level rule +syntax. After a rule has been added to the NetControl framework, NetControl +sends the rule to one or several of its *backends*. Each backend is responsible +to communicate with a single hard- or software device. The NetControl framework +tracks rules throughout their entire lifecycle and reports the status (like +success, failure and timeouts) back to the user scripts. + +The backends are implemented as Zeek scripts using a plugin based API; an example +for this is :doc:`/scripts/base/frameworks/netcontrol/plugins/broker.zeek`. We'll +show how to write plugins in :ref:`framework-netcontrol-plugins`. + +NetControl API +============== + +High-level NetControl API +------------------------- + +In this section, we will introduce the high level NetControl API. As mentioned +above, NetControl uses *backends* to communicate with the external devices that +will implement the rules. You will need at least one active backend before you +can use NetControl. For our examples, we will just use the debug plugin to +create a backend. This plugin outputs all actions that are taken to the standard +output. + +Backends should be initialized in the :zeek:see:`NetControl::init` event, calling +the :zeek:see:`NetControl::activate` function after the plugin instance has been +initialized. The debug plugin can be initialized as follows: + +.. code-block:: zeek + + event NetControl::init() + { + local debug_plugin = NetControl::create_debug(T); + NetControl::activate(debug_plugin, 0); + } + +After at least one backend has been added to the NetControl framework, the +framework can be used and will send added rules to the added backend. + +The NetControl framework contains several high level functions that allow users +to drop connections of certain addresses and networks, shunt network traffic, +etc. The following table shows and describes all of the currently available +high-level functions. + +.. list-table:: + :widths: 32 40 + :header-rows: 1 + + * - Function + - Description + + * - :zeek:see:`NetControl::drop_address` + - Calling this function causes NetControl to block all packets involving + an IP address from being forwarded. + + * - :zeek:see:`NetControl::drop_connection` + - Calling this function stops all packets of a specific connection + (identified by its 5-tuple) from being forwarded. + + * - :zeek:see:`NetControl::drop_address_catch_release` + - Calling this function causes all packets of a specific source IP to be + blocked. This function uses catch-and-release functionality and the IP + address is only dropped for a short amount of time to conserve rule + space in the network hardware. It is immediately re-dropped when it is + seen again in traffic. See :ref:`framework-netcontrol-catchrelease` for + more information. + + * - :zeek:see:`NetControl::shunt_flow` + - Calling this function causes NetControl to stop forwarding a + uni-directional flow of packets to Zeek. This allows Zeek to conserve + resources by shunting flows that have been identified as being benign. + + * - :zeek:see:`NetControl::redirect_flow` + - Calling this function causes NetControl to redirect a uni-directional + flow to another port of the networking hardware. + + * - :zeek:see:`NetControl::quarantine_host` + - Calling this function allows Zeek to quarantine a host by sending DNS + traffic to a host with a special DNS server, which resolves all queries + as pointing to itself. The quarantined host is only allowed between the + special server, which will serve a warning message detailing the next + steps for the user. + + * - :zeek:see:`NetControl::whitelist_address` + - Calling this function causes NetControl to push a whitelist entry for an + IP address to the networking hardware. + + * - :zeek:see:`NetControl::whitelist_subnet` + - Calling this function causes NetControl to push a whitelist entry for a + subnet to the networking hardware. + +After adding a backend, all of these functions can immediately be used and will +start sending the rules to the added backend(s). To give a very simple example, +the following script will simply block the traffic of all connections that it +sees being established: + +.. literalinclude:: netcontrol-1-drop-with-debug.zeek + :caption: + :language: zeek + :linenos: + :tab-width: 4 + +Running this script on a file containing one connection will cause the debug +plugin to print one line to the standard output, which contains information +about the rule that was added. It will also cause creation of :file:`netcontrol.log`, +which contains information about all actions that are taken by NetControl: + +.. code-block:: console + + $ zeek -C -r tls/ecdhe.pcap netcontrol-1-drop-with-debug.zeek + netcontrol debug (Debug-All): init + netcontrol debug (Debug-All): add_rule: [ty=NetControl::DROP, target=NetControl::FORWARD, entity=[ty=NetControl::CONNECTION, conn=[orig_h=192.168.18.50, orig_p=56981/tcp, resp_h=74.125.239.97, resp_p=443/tcp], flow=, ip=, mac=], expire=20.0 secs, priority=0, location=, out_port=, mod=, id=2, cid=2, _plugin_ids={\x0a\x0a}, _active_plugin_ids={\x0a\x0a}, _no_expire_plugins={\x0a\x0a}, _added=F] + + $ cat netcontrol.log + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path netcontrol + #open 2018-12-14-18-50-53 + #fields ts rule_id category cmd state action target entity_type entity mod msg priority expire location plugin + #types time string enum string enum string enum string string string string int interval string string + 0.000000 - NetControl::MESSAGE - - - - - - - activating plugin with priority 0 - - - Debug-All + 0.000000 - NetControl::MESSAGE - - - - - - - activation finished - - - Debug-All + 0.000000 - NetControl::MESSAGE - - - - - - - plugin initialization done - - - - + 1398529018.678276 2 NetControl::RULE ADD NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::CONNECTION 192.168.18.50/56981<->74.125.239.97/443 - - 0 20.000000 - Debug-All + 1398529018.678276 2 NetControl::RULE ADD NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::CONNECTION 192.168.18.50/56981<->74.125.239.97/443 - - 0 20.000000 - Debug-All + #close 2018-12-14-18-50-53 + +In our case, :file:`netcontrol.log` contains several :zeek:see:`NetControl::MESSAGE` +entries, which show that the debug plugin has been initialized and added. +Afterwards, there are two :zeek:see:`NetControl::RULE` entries; the first shows +that the addition of a rule has been requested (state is +:zeek:see:`NetControl::REQUESTED`). The following line shows that the rule was +successfully added (the state is :zeek:see:`NetControl::SUCCEEDED`). The +remainder of the log line gives more information about the added rule, which in +our case applies to a specific 5-tuple. + +In addition to the :file:`netcontrol.log`, the drop commands also create a second, +additional log called :file:`netcontrol_drop.log`. This log file is much more succinct and +only contains information that is specific to drops that are enacted by +NetControl: + +.. code-block:: console + + $ cat netcontrol_drop.log + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path netcontrol_drop + #open 2018-12-14-18-50-53 + #fields ts rule_id orig_h orig_p resp_h resp_p expire location + #types time string addr port addr port interval string + 1398529018.678276 2 192.168.18.50 56981 74.125.239.97 443 20.000000 - + #close 2018-12-14-18-50-53 + +While this example of blocking all connections is usually not very useful, the +high-level API gives an easy way to take action, for example when a host is +identified doing some harmful activity. To give a more realistic example, the +following code automatically blocks a recognized SSH guesser: + +.. literalinclude:: netcontrol-2-ssh-guesser.zeek + :caption: + :language: zeek + :linenos: + :tab-width: 4 + +.. code-block:: console + + $ zeek -C -r ssh/sshguess.pcap netcontrol-2-ssh-guesser.zeek + netcontrol debug (Debug-All): init + netcontrol debug (Debug-All): add_rule: [ty=NetControl::DROP, target=NetControl::FORWARD, entity=[ty=NetControl::ADDRESS, conn=, flow=, ip=192.168.56.1/32, mac=], expire=1.0 hr, priority=0, location=, out_port=, mod=, id=2, cid=2, _plugin_ids={\x0a\x0a}, _active_plugin_ids={\x0a\x0a}, _no_expire_plugins={\x0a\x0a}, _added=F] + + $ cat netcontrol.log + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path netcontrol + #open 2018-12-14-18-50-54 + #fields ts rule_id category cmd state action target entity_type entity mod msg priority expire location plugin + #types time string enum string enum string enum string string string string int interval string string + 0.000000 - NetControl::MESSAGE - - - - - - - activating plugin with priority 0 - - - Debug-All + 0.000000 - NetControl::MESSAGE - - - - - - - activation finished - - - Debug-All + 0.000000 - NetControl::MESSAGE - - - - - - - plugin initialization done - - - - + 1427726759.303199 2 NetControl::RULE ADD NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.56.1/32 - - 0 3600.000000 - Debug-All + 1427726759.303199 2 NetControl::RULE ADD NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.56.1/32 - - 0 3600.000000 - Debug-All + #close 2018-12-14-18-50-54 + +Note that in this case, instead of calling NetControl directly, we also can use +the :zeek:see:`Notice::ACTION_DROP` action of the notice framework: + +.. literalinclude:: netcontrol-3-ssh-guesser.zeek + :caption: + :language: zeek + :linenos: + :tab-width: 4 + +.. code-block:: console + + $ zeek -C -r ssh/sshguess.pcap netcontrol-3-ssh-guesser.zeek + netcontrol debug (Debug-All): init + netcontrol debug (Debug-All): add_rule: [ty=NetControl::DROP, target=NetControl::FORWARD, entity=[ty=NetControl::ADDRESS, conn=, flow=, ip=192.168.56.1/32, mac=], expire=10.0 mins, priority=0, location=ACTION_DROP: T, out_port=, mod=, id=2, cid=2, _plugin_ids={\x0a\x0a}, _active_plugin_ids={\x0a\x0a}, _no_expire_plugins={\x0a\x0a}, _added=F] + + $ cat netcontrol.log + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path netcontrol + #open 2018-12-14-18-50-55 + #fields ts rule_id category cmd state action target entity_type entity mod msg priority expire location plugin + #types time string enum string enum string enum string string string string int interval string string + 0.000000 - NetControl::MESSAGE - - - - - - - activating plugin with priority 0 - - - Debug-All + 0.000000 - NetControl::MESSAGE - - - - - - - activation finished - - - Debug-All + 0.000000 - NetControl::MESSAGE - - - - - - - plugin initialization done - - - - + 1427726759.303199 2 NetControl::RULE ADD NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.56.1/32 - - 0 600.000000 ACTION_DROP: T Debug-All + 1427726759.303199 2 NetControl::RULE ADD NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.56.1/32 - - 0 600.000000 ACTION_DROP: T Debug-All + #close 2018-12-14-18-50-55 + +Using the :zeek:see:`Notice::ACTION_DROP` action of the notice framework also +will cause the ``dropped`` column in :file:`notice.log` to be set to true each time that +the NetControl framework enacts a block: + +.. code-block:: console + + $ cat notice.log + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path notice + #open 2018-12-14-18-50-55 + #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude + #types time string addr port addr port string string string enum enum string string addr addr port count string set[enum] interval bool string string string double double + 1427726759.303199 - - - - - - - - - SSH::Password_Guessing 192.168.56.1 appears to be guessing SSH passwords (seen in 10 connections). Sampled servers: 192.168.56.103, 192.168.56.103, 192.168.56.103, 192.168.56.103, 192.168.56.103 192.168.56.1 - - - - Notice::ACTION_DROP,Notice::ACTION_LOG 3600.000000 F - - - - - + #close 2018-12-14-18-50-55 + +Rule API +-------- + +As already mentioned in the last section, in addition to the high-level API, the +NetControl framework also supports a Rule based API which allows greater +flexibility while adding rules. Actually, all the high-level functions are +implemented using this lower-level rule API; the high-level functions simply +convert their arguments into the lower-level rules and then add the rules +directly to the NetControl framework (by calling :zeek:see:`NetControl::add_rule`). + +The following figure shows the main components of NetControl rules: + +.. figure:: netcontrol-rules.png + :width: 600 + :align: center + :alt: NetControl rule overview + :target: ../_images/netcontrol-rules.png + + NetControl Rule overview (click to enlarge). + +The types that are used to make up a rule are defined in +:doc:`/scripts/base/frameworks/netcontrol/types.zeek`. + +Rules are defined as a :zeek:see:`NetControl::Rule` record. Rules have a *type*, +which specifies what kind of action is taken. The possible actions are to +**drop** packets, to **modify** them, to **redirect** or to **whitelist** them. +The *target* of a rule specifies if the rule is applied in the *forward path*, +and affects packets as they are forwarded through the network, or if it affects +the *monitor path* and only affects the packets that are sent to Zeek, but not +the packets that traverse the network. The *entity* specifies the address, +connection, etc. that the rule applies to. In addition, each rule has a +*timeout* (which can be left empty), a *priority* (with higher priority rules +overriding lower priority rules). Furthermore, a *location* string with more +text information about each rule can be provided. + +There are a couple more fields that are only needed for some rule types. For +example, when you insert a redirect rule, you have to specify the port that +packets should be redirected to. All these fields are shown in the +:zeek:see:`NetControl::Rule` documentation. + +To give an example on how to construct your own rule, we are going to write +our own version of the :zeek:see:`NetControl::drop_connection` function. The only +difference between our function and the one provided by NetControl is the fact +that the NetControl function has additional functionality, e.g. for logging. + +Once again, we are going to test our function with a simple example that simply +drops all connections on the network: + +.. literalinclude:: netcontrol-4-drop.zeek + :caption: + :language: zeek + :linenos: + :tab-width: 4 + +.. code-block:: console + + $ zeek -C -r tls/ecdhe.pcap netcontrol-4-drop.zeek + netcontrol debug (Debug-All): init + netcontrol debug (Debug-All): add_rule: [ty=NetControl::DROP, target=NetControl::FORWARD, entity=[ty=NetControl::CONNECTION, conn=[orig_h=192.168.18.50, orig_p=56981/tcp, resp_h=74.125.239.97, resp_p=443/tcp], flow=, ip=, mac=], expire=20.0 secs, priority=0, location=, out_port=, mod=, id=2, cid=2, _plugin_ids={\x0a\x0a}, _active_plugin_ids={\x0a\x0a}, _no_expire_plugins={\x0a\x0a}, _added=F] + + $ cat netcontrol.log + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path netcontrol + #open 2018-12-14-18-50-55 + #fields ts rule_id category cmd state action target entity_type entity mod msg priority expire location plugin + #types time string enum string enum string enum string string string string int interval string string + 0.000000 - NetControl::MESSAGE - - - - - - - activating plugin with priority 0 - - - Debug-All + 0.000000 - NetControl::MESSAGE - - - - - - - activation finished - - - Debug-All + 0.000000 - NetControl::MESSAGE - - - - - - - plugin initialization done - - - - + 1398529018.678276 2 NetControl::RULE ADD NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::CONNECTION 192.168.18.50/56981<->74.125.239.97/443 - - 0 20.000000 - Debug-All + 1398529018.678276 2 NetControl::RULE ADD NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::CONNECTION 192.168.18.50/56981<->74.125.239.97/443 - - 0 20.000000 - Debug-All + #close 2018-12-14-18-50-55 + +The last example shows that :zeek:see:`NetControl::add_rule` returns a string +identifier that is unique for each rule (uniqueness is not preserved across +restarts of Zeek). This rule id can be used to later remove rules manually using +:zeek:see:`NetControl::remove_rule`. + +Similar to :zeek:see:`NetControl::add_rule`, all the high-level functions also +return their rule IDs, which can be removed in the same way. + +Interacting with Rules +---------------------- + +The NetControl framework offers a number of different ways to interact with +rules. Before a rule is applied by the framework, a number of different hooks +allow you to either modify or discard rules before they are added. Furthermore, +a number of events can be used to track the lifecycle of a rule while it is +being managed by the NetControl framework. It is also possible to query and +access the current set of active rules. + +Rule Policy +*********** + +The hook :zeek:see:`NetControl::rule_policy` provides the mechanism for modifying +or discarding a rule before it is sent onwards to the backends. Hooks can be +thought of as multi-bodied functions and using them looks very similar to +handling events. In contrast to events, they are processed immediately. Like +events, hooks can have priorities to sort the order in which they are applied. +Hooks can use the ``break`` keyword to show that processing should be aborted; +if any :zeek:see:`NetControl::rule_policy` hook uses ``break``, the rule will be +discarded before further processing. + +Here is a simple example which tells Zeek to discard all rules for connections +originating from the 192.168.* network: + +.. literalinclude:: netcontrol-5-hook.zeek + :caption: + :language: zeek + :linenos: + :tab-width: 4 + +.. code-block:: console + + $ zeek -C -r tls/ecdhe.pcap netcontrol-5-hook.zeek + netcontrol debug (Debug-All): init + Ignored connection from, 192.168.18.50 + +NetControl Events +***************** + +In addition to the hooks, the NetControl framework offers a variety of events +that are raised by the framework to allow users to track rules, as well as the +state of the framework. + +We already encountered and used one event of the NetControl framework, +:zeek:see:`NetControl::init`, which is used to initialize the framework. After +the framework has finished initialization and will start accepting rules, the +:zeek:see:`NetControl::init_done` event will be raised. + +When rules are added to the framework, the following events will be called in +this order: + +.. list-table:: + :widths: 20 80 + :header-rows: 1 + + * - Event + - Description + + * - :zeek:see:`NetControl::rule_new` + - Signals that a new rule is created by the NetControl framework due to + :zeek:see:`NetControl::add_rule`. At this point, the rule has not + yet been added to any backend. + + * - :zeek:see:`NetControl::rule_added` + - Signals that a new rule has successfully been added by a backend. + + * - :zeek:see:`NetControl::rule_exists` + - This event is raised instead of :zeek:see:`NetControl::rule_added` when a + backend reports that a rule was already existing. + + * - :zeek:see:`NetControl::rule_timeout` + - Signals that a rule timeout was reached. If the hardware does not support + automatic timeouts, the NetControl framework will automatically call + :zeek:see:`NetControl::remove_rule`. + + * - :zeek:see:`NetControl::rule_removed` + - Signals that a new rule has successfully been removed a backend. + + * - :zeek:see:`NetControl::rule_destroyed` + - This event is the pendant to :zeek:see:`NetControl::rule_added`, and + reports that a rule is no longer being tracked by the NetControl framework. + This happens, for example, when a rule was removed from all backends. + + * - :zeek:see:`NetControl::rule_error` + - This event is raised whenever an error occurs during any rule operation. + +Finding active rules +******************** + +The NetControl framework provides two functions for finding currently active +rules: :zeek:see:`NetControl::find_rules_addr` finds all rules that affect a +certain IP address and :zeek:see:`NetControl::find_rules_subnet` finds all rules +that affect a specified subnet. + +Consider, for example, the case where a Zeek instance monitors the traffic at the +border, before any firewall or switch rules were applied. In this case, Zeek will +still be able to see connection attempts of already blocked IP addresses. In this +case, :zeek:see:`NetControl::find_rules_addr` could be used to check if an +address already was blocked in the past. + +Here is a simple example, which uses a trace that contains two connections from +the same IP address. After the first connection, the script recognizes that the +address is already blocked in the second connection. + +.. literalinclude:: netcontrol-6-find.zeek + :caption: + :language: zeek + :linenos: + :tab-width: 4 + +.. code-block:: console + + $ zeek -C -r tls/google-duplicate.trace netcontrol-6-find.zeek + netcontrol debug (Debug-All): init + netcontrol debug (Debug-All): add_rule: [ty=NetControl::DROP, target=NetControl::FORWARD, entity=[ty=NetControl::CONNECTION, conn=[orig_h=192.168.4.149, orig_p=60623/tcp, resp_h=74.125.239.129, resp_p=443/tcp], flow=, ip=, mac=], expire=20.0 secs, priority=0, location=, out_port=, mod=, id=2, cid=2, _plugin_ids={\x0a\x0a}, _active_plugin_ids={\x0a\x0a}, _no_expire_plugins={\x0a\x0a}, _added=F] + Rule added + Rule already exists + +Notice that the functions return vectors because it is possible that several +rules exist simultaneously that affect one IP; either there could be +rules with different priorities, or rules for the subnet that an IP address is +part of. + + + +.. _framework-netcontrol-catchrelease: + +Catch and Release +----------------- + +We already mentioned earlier that in addition to the +:zeek:see:`NetControl::drop_connection` and :zeek:see:`NetControl::drop_address` +functions, which drop a connection or address for a specified amount of time, +NetControl also comes with a blocking function that uses an approach called +*catch and release*. + +Catch and release is a blocking scheme that conserves valuable rule space in +your hardware. Instead of using long-lasting blocks, catch and release first +only installs blocks for a short amount of time (typically a few minutes). After +these minutes pass, the block is lifted, but the IP address is added to a +watchlist and the IP address will immediately be re-blocked again (for a longer +amount of time), if it is seen reappearing in any traffic, no matter if the new +traffic triggers any alert or not. + +This makes catch and release blocks similar to normal, longer duration blocks, +while only requiring a small amount of space for the currently active rules. IP +addresses that only are seen once for a short time are only blocked for a few +minutes, monitored for a while and then forgotten. IP addresses that keep +appearing will get re-blocked for longer amounts of time. + +In contrast to the other high-level functions that we have documented so far, the +catch and release functionality is much more complex and adds a number of +different specialized functions to NetControl. The documentation for catch and +release is contained in the file +:doc:`/scripts/policy/frameworks/netcontrol/catch-and-release.zeek`. + +Using catch and release in your scripts is easy; just use +:zeek:see:`NetControl::drop_address_catch_release` like in this example: + +.. literalinclude:: netcontrol-7-catch-release.zeek + :caption: + :language: zeek + :linenos: + :tab-width: 4 + +.. code-block:: console + + $ zeek -C -r tls/ecdhe.pcap netcontrol-7-catch-release.zeek + netcontrol debug (Debug-All): init + netcontrol debug (Debug-All): add_rule: [ty=NetControl::DROP, target=NetControl::FORWARD, entity=[ty=NetControl::ADDRESS, conn=, flow=, ip=192.168.18.50/32, mac=], expire=10.0 mins, priority=0, location=, out_port=, mod=, id=2, cid=2, _plugin_ids={\x0a\x0a}, _active_plugin_ids={\x0a\x0a}, _no_expire_plugins={\x0a\x0a}, _added=F] + +Note that you do not have to provide the block time for catch and release; +instead, catch and release uses the time intervals specified in +:zeek:see:`NetControl::catch_release_intervals` (by default 10 minutes, 1 hour, +24 hours, 7 days). That means when an address is first blocked, it is blocked +for 10 minutes and monitored for 1 hour. If the address reappears after the +first 10 minutes, it is blocked for 1 hour and then monitored for 24 hours, etc. + +Catch and release adds its own new logfile in addition to the already existing +ones (:file:`netcontrol_catch_release.log`): + +.. code-block:: console + + $ cat netcontrol_catch_release.log + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path netcontrol_catch_release + #open 2018-12-14-18-50-58 + #fields ts rule_id ip action block_interval watch_interval blocked_until watched_until num_blocked location message + #types time string addr enum interval interval time time count string string + 1398529018.678276 2 192.168.18.50 NetControl::DROP 600.000000 3600.000000 1398529618.678276 1398532618.678276 1 - - + 1398529018.678276 2 192.168.18.50 NetControl::DROPPED 600.000000 3600.000000 1398529618.678276 1398532618.678276 1 - - + #close 2018-12-14-18-50-58 + +In addition to the blocking function, catch and release comes with the +:zeek:see:`NetControl::get_catch_release_info` function to +check if an address is already blocked by catch and release (and get information +about the block). The :zeek:see:`NetControl::unblock_address_catch_release` +function can be used to unblock addresses from catch and release. + +.. note:: + + Since catch and release does its own connection tracking in addition to the + tracking used by the NetControl framework, it is not sufficient to remove + rules that were added by catch and release using :zeek:see:`NetControl::remove_rule`. + You have to use :zeek:see:`NetControl::unblock_address_catch_release` in this + case. + +.. _framework-netcontrol-plugins: + +NetControl Plugins +================== + +Using the existing plugins +-------------------------- + +In the API part of the documentation, we exclusively used the debug plugin, +which simply outputs its actions to the screen. In addition to this debugging +plugin, Zeek ships with a small number of plugins that can be used to interface +the NetControl framework with your networking hard- and software. + +The plugins that currently ship with NetControl are: + +.. list-table:: + :widths: 15 55 + :header-rows: 1 + + * - Plugin name + - Description + + * - OpenFlow plugin + - This is the most fully featured plugin which allows the NetControl + framework to be interfaced with OpenFlow switches. The source of this + plugin is contained in :doc:`/scripts/base/frameworks/netcontrol/plugins/openflow.zeek`. + + * - Broker plugin + - This plugin provides a generic way to send NetControl commands using the + new Zeek communication library (Broker). External programs can receive + the rules and take action; we provide an example script that calls + command-line programs triggered by NetControl. The source of this + plugin is contained in :doc:`/scripts/base/frameworks/netcontrol/plugins/broker.zeek`. + + * - acld plugin + - This plugin adds support for the acld daemon, which can interface with + several switches and routers. The current version of acld is available + from the `LBL ftp server `_. The source of this + plugin is contained in :doc:`/scripts/base/frameworks/netcontrol/plugins/acld.zeek`. + + * - PacketFilter plugin + - This plugin uses the Zeek process-level packet filter (see + :zeek:see:`install_src_net_filter` and + :zeek:see:`install_dst_net_filter`). Since the functionality of the + PacketFilter is limited, this plugin is mostly for demonstration purposes. The source of this + plugin is contained in :doc:`/scripts/base/frameworks/netcontrol/plugins/packetfilter.zeek`. + + * - Debug plugin + - The debug plugin simply outputs its action to the standard output. The source of this + plugin is contained in :doc:`/scripts/base/frameworks/netcontrol/plugins/debug.zeek`. + +Activating plugins +****************** + +In the API reference we already used the debug plugin. +We first had to instantiate it by calling +:zeek:see:`NetControl::create_debug` and then add it to NetControl by +calling :zeek:see:`NetControl::activate`. + +As we already hinted before, NetControl supports having several plugins that are +active at the same time. The second argument to the :zeek:see:`NetControl::activate` +function is the priority of the backend that was just added. Each rule is sent +to all plugins in order, from highest priority to lowest priority. The backend +can then choose if it accepts the rule and pushes it out to the hardware that it +manages. Or, it can opt to reject the rule. In this case, the NetControl +framework will try to apply the rule to the backend with the next lower +priority. If no backend accepts a rule, the rule insertion is marked as failed. + +The choice if a rule is accepted or rejected stays completely with each plugin. +The debug plugin we used so far just accepts all rules. However, for other +plugins you can specify what rules they will accept. Consider, for example, a +network with two OpenFlow switches. The first switch forwards packets from the +network to the external world, the second switch sits in front of your Zeek +cluster to provide packet shunting. In this case, you can add two OpenFlow +backends to NetControl. When you create the instances using +:zeek:see:`NetControl::create_openflow`, you set the ``monitor`` and ``forward`` +attributes of the configuration in :zeek:see:`NetControl::OfConfig` +appropriately. Afterwards, one of the backends will only accept rules for the +monitor path; the other backend will only accept rules for the forward path. + +Commonly, plugins also support predicate functions, that allow the user to +specify restrictions on the rules that they will accept. This can for example be +used if you have a network where certain switches are responsible for specified +subnets. The predicate can examine the subnet of the rule and only accept the +rule if the rule matches the subnet that the specific switch is responsible for. + +To give an example, the following script adds two backends to NetControl. One +backend is the NetControl debug backend, which just outputs the rules to the +console. The second backend is an OpenFlow backend, which uses the OpenFlow +debug mode that outputs the openflow rules to :file:`openflow.log`. The OpenFlow +backend uses a predicate function to only accept rules with a source address in +the 192.168.17.0/24 network; all other rules will be passed on to the debug +plugin. We manually block a few addresses in the +:zeek:see:`NetControl::init_done` event to verify the correct functionality. + +.. literalinclude:: netcontrol-8-multiple.zeek + :caption: + :language: zeek + :linenos: + :tab-width: 4 + +.. code-block:: console + + $ zeek netcontrol-8-multiple.zeek + netcontrol debug (Debug-All): init + netcontrol debug (Debug-All): add_rule: [ty=NetControl::DROP, target=NetControl::FORWARD, entity=[ty=NetControl::ADDRESS, conn=, flow=, ip=192.168.17.2/32, mac=], expire=1.0 min, priority=0, location=, out_port=, mod=, id=3, cid=3, _plugin_ids={\x0a\x0a}, _active_plugin_ids={\x0a\x0a}, _no_expire_plugins={\x0a\x0a}, _added=F] + +As you can see, only the single block affecting the 192.168.17.0/24 network is +output to the command line. The other two lines are handled by the OpenFlow +plugin. We can verify this by looking at :file:`netcontrol.log`. The plugin column shows +which plugin handled a rule and reveals that two rules were handled by OpenFlow: + +.. code-block:: console + + $ cat netcontrol.log + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path netcontrol + #open 2018-12-14-18-50-58 + #fields ts rule_id category cmd state action target entity_type entity mod msg priority expire location plugin + #types time string enum string enum string enum string string string string int interval string string + 1544813458.913148 - NetControl::MESSAGE - - - - - - - activating plugin with priority 0 - - - Debug-All + 1544813458.913148 - NetControl::MESSAGE - - - - - - - activation finished - - - Debug-All + 1544813458.913148 - NetControl::MESSAGE - - - - - - - activating plugin with priority 10 - - - Openflow-Log-42 + 1544813458.913148 - NetControl::MESSAGE - - - - - - - activation finished - - - Openflow-Log-42 + 1544813458.913148 - NetControl::MESSAGE - - - - - - - plugin initialization done - - - - + 1544813458.913148 2 NetControl::RULE ADD NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 10.0.0.1/32 - - 0 60.000000 - Openflow-Log-42 + 1544813458.913148 3 NetControl::RULE ADD NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.17.2/32 - - 0 60.000000 - Debug-All + 1544813458.913148 4 NetControl::RULE ADD NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.18.2/32 - - 0 60.000000 - Openflow-Log-42 + 1544813458.913148 3 NetControl::RULE ADD NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.17.2/32 - - 0 60.000000 - Debug-All + 1544813458.913148 2 NetControl::RULE ADD NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 10.0.0.1/32 - - 0 60.000000 - Openflow-Log-42 + 1544813458.913148 4 NetControl::RULE ADD NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.18.2/32 - - 0 60.000000 - Openflow-Log-42 + #close 2018-12-14-18-50-58 + +Furthermore, :file:`openflow.log` also shows the two added rules, converted to OpenFlow +flow mods: + +.. code-block:: console + + $ cat openflow.log + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path openflow + #open 2018-12-14-18-50-58 + #fields ts dpid match.in_port match.dl_src match.dl_dst match.dl_vlan match.dl_vlan_pcp match.dl_type match.nw_tos match.nw_proto match.nw_src match.nw_dst match.tp_src match.tp_dst flow_mod.cookie flow_mod.table_id flow_mod.command flow_mod.idle_timeout flow_mod.hard_timeout flow_mod.priority flow_mod.out_port flow_mod.out_group flow_mod.flags flow_mod.actions.out_ports flow_mod.actions.vlan_vid flow_mod.actions.vlan_pcp flow_mod.actions.vlan_strip flow_mod.actions.dl_src flow_mod.actions.dl_dst flow_mod.actions.nw_tos flow_mod.actions.nw_src flow_mod.actions.nw_dst flow_mod.actions.tp_src flow_mod.actions.tp_dst + #types time count count string string count count count count count subnet subnet count count count count enum count count count count count count vector[count] count count bool string string count addr addr count count + 1544813458.913148 42 - - - - - 2048 - - 10.0.0.1/32 - - - 4398046511108 - OpenFlow::OFPFC_ADD 0 60 0 - - 1 (empty) - - F - - - - - - - + 1544813458.913148 42 - - - - - 2048 - - - 10.0.0.1/32 - - 4398046511109 - OpenFlow::OFPFC_ADD 0 60 0 - - 1 (empty) - - F - - - - - - - + 1544813458.913148 42 - - - - - 2048 - - 192.168.18.2/32 - - - 4398046511112 - OpenFlow::OFPFC_ADD 0 60 0 - - 1 (empty) - - F - - - - - - - + 1544813458.913148 42 - - - - - 2048 - - - 192.168.18.2/32 - - 4398046511113 - OpenFlow::OFPFC_ADD 0 60 0 - - 1 (empty) - - F - - - - - - - + #close 2018-12-14-18-50-58 + +.. note:: + + You might have asked yourself what happens when you add two or more with the + same priority. In this case, the rule is sent to all the backends + simultaneously. This can be useful, for example when you have redundant + switches that should keep the same rule state. + +Interfacing with external hardware +********************************** + +Now that we know which plugins exist, and how they can be added to NetControl, +it is time to discuss how we can interface Zeek with actual hardware. The typical +way to accomplish this is to use the Zeek communication library (Broker), which +can be used to exchange Zeek events with external programs and scripts. The +NetControl plugins can use Broker to send events to external programs, which can +then take action depending on these events. + +The following figure shows this architecture with the example of the OpenFlow +plugin. The OpenFlow plugin uses Broker to send events to an external Python +script, which uses the `Ryu SDN controller `_ to +communicate with the Switch. + +.. figure:: netcontrol-openflow.png + :width: 600 + :align: center + :alt: NetControl and OpenFlow architecture. + :target: ../_images/netcontrol-openflow.png + + NetControl and OpenFlow architecture (click to enlarge). + +The Python scripts that are used to interface with the available NetControl +plugins are contained in the ``zeek-netcontrol`` repository (`github link `_). +The repository contains scripts for the OpenFlow as well as the acld plugin. +Furthermore, it contains a script for the broker plugin which can be used to +call configurable command-line programs when used with the broker plugin. + +The repository also contains documentation on how to install these connectors. +The ``netcontrol`` directory contains an API that allows you to write your own +connectors to the broker plugin. + +Writing plugins +--------------- + +In addition to using the plugins that are part of NetControl, you can write your +own plugins to interface with hard- or software that we currently do not support +out of the box. + +Creating your own plugin is easy; besides a bit of boilerplate, you only need to +create two functions: one that is called when a rule is added, and one that is +called when a rule is removed. The following script creates a minimal plugin +that just outputs a rule when it is added or removed. Note that you have to +raise the :zeek:see:`NetControl::rule_added` and +:zeek:see:`NetControl::rule_removed` events in your plugin to let NetControl know +when a rule was added and removed successfully. + +.. literalinclude:: netcontrol-9-skeleton.zeek + :caption: + :language: zeek + :linenos: + :tab-width: 4 + +This example is already fully functional and we can use it with a script similar +to our very first example: + +.. literalinclude:: netcontrol-10-use-skeleton.zeek + :caption: + :language: zeek + :linenos: + :tab-width: 4 + +.. code-block:: console + + $ zeek -C -r tls/ecdhe.pcap netcontrol-10-use-skeleton.zeek + add, [ty=NetControl::DROP, target=NetControl::FORWARD, entity=[ty=NetControl::CONNECTION, conn=[orig_h=192.168.18.50, orig_p=56981/tcp, resp_h=74.125.239.97, resp_p=443/tcp], flow=, ip=, mac=], expire=20.0 secs, priority=0, location=, out_port=, mod=, id=2, cid=2, _plugin_ids={ + + }, _active_plugin_ids={ + + }, _no_expire_plugins={ + + }, _added=F] + +If you want to write your own plugins, it will be worthwhile to look at the +plugins that ship with the NetControl framework to see how they define the +predicates and interact with Broker. diff --git a/doc/frameworks/notice.rst b/doc/frameworks/notice.rst new file mode 100644 index 0000000000..cd8bc6e73f --- /dev/null +++ b/doc/frameworks/notice.rst @@ -0,0 +1,434 @@ + +.. _notice-framework: + +================ +Notice Framework +================ + +One of the easiest ways to customize Zeek is writing a local notice policy. +Zeek can detect a large number of potentially interesting situations, and the +notice policy hook identifies which of them the user wants to be acted upon in +some manner. In particular, the notice policy can specify actions to be taken, +such as sending an email or compiling regular alarm emails. This page gives an +introduction into writing such a notice policy. + +Overview +======== + +Let’s start with a little bit of background on Zeek’s philosophy on reporting +things. Zeek ships with a large number of policy scripts which perform a wide +variety of analyses. Most of these scripts monitor for activity which might be +of interest for the user. However, none of these scripts determines the +importance of what it finds itself. Instead, the scripts only flag situations +as *potentially* interesting, leaving it to the local configuration to define +which of them are in fact actionable. This decoupling of detection and +reporting allows Zeek to address the different needs that different sites have. +Definitions of what constitutes an attack or even a compromise differ quite a +bit between environments, and activity deemed malicious at one site might be +fully acceptable at another. + +Whenever one of Zeek’s analysis scripts sees something potentially interesting +it flags the situation by calling the :zeek:see:`NOTICE` function and giving it +a single :zeek:see:`Notice::Info` record. A Notice has a +:zeek:see:`Notice::Type`, which reflects the kind of activity that has been +seen, and it is usually also augmented with further context about the +situation. + +More information about raising notices can be found in the :ref:`Raising +Notices ` section. + +Once a notice is raised, it can have any number of actions applied to it by +writing :zeek:see:`Notice::policy` hooks which are described in the +:ref:`Notice Policy ` section below. Such actions can for +example send email to configured address(es), or simply ignore the +notice. Currently, the following actions are defined: + +.. list-table:: + :header-rows: 1 + + * - Action + - Description + + * - :zeek:see:`Notice::ACTION_LOG` + - Write the notice to the :zeek:see:`Notice::LOG` logging stream. + + * - :zeek:see:`Notice::ACTION_ALARM` + - Log into the :zeek:see:`Notice::ALARM_LOG` stream which will rotate + hourly and email the contents to the email address or addresses in the + :zeek:field:`Notice::Info$email_dest` field of that notice's :zeek:see:`Notice::Info` record. + + * - :zeek:see:`Notice::ACTION_EMAIL` + - Send the notice in an email to the email address or addresses in the + :zeek:field:`Notice::Info$email_dest` field of that notice's :zeek:see:`Notice::Info` record. + + * - :zeek:see:`Notice::ACTION_PAGE` + - Send an email to the email address or addresses in the + :zeek:field:`Notice::Info$email_dest` field of that notice's :zeek:see:`Notice::Info` record. + +How these notice actions are applied to notices is discussed in the +:ref:`Notice Policy ` and :ref:`Notice Policy Shortcuts +` sections. + +Processing Notices +================== + +.. _notice-policy: + +Notice Policy +------------- + +The hook :zeek:see:`Notice::policy` provides the mechanism for applying actions +and generally modifying the notice before it’s sent onward to the action +plugins. Hooks can be thought of as multi-bodied functions and using them +looks very similar to handling events. The difference is that they don’t go +through the event queue like events. Users can alter notice processing by +directly modifying fields in the :zeek:see:`Notice::Info` record given as the +argument to the hook. + +Here’s a simple example which tells Zeek to send an email for all notices of +type :zeek:see:`SSH::Password_Guessing` if the guesser attempted to log in to +the server at ``192.168.56.103``: + +.. code-block:: zeek + :caption: notice_ssh_guesser.zeek + + @load protocols/ssh/detect-bruteforcing + + redef SSH::password_guesses_limit=10; + + hook Notice::policy(n: Notice::Info) + { + if ( n$note == SSH::Password_Guessing && /192\.168\.56\.103/ in n$sub ) + { + add n$actions[Notice::ACTION_EMAIL]; + n$email_dest = "ssh_alerts@example.net"; + } + } + +.. code-block:: console + + $ zeek -C -r ssh/sshguess.pcap notice_ssh_guesser.zeek + $ cat notice.log + +:: + + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path notice + #open 2018-12-13-22-56-35 + #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc proto note msg sub src dst p n peer_descr actions email-dest suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude + #types time string addr port addr port string string string enum enum string string addr addr port count string set[enum] set[string] interval bool string string string double double + 1427726759.303199 - - - - - - - - - SSH::Password_Guessing 192.168.56.1 appears to be guessing SSH passwords (seen in 10 connections). Sampled servers: 192.168.56.103, 192.168.56.103, 192.168.56.103, 192.168.56.103, 192.168.56.103 192.168.56.1 - - - - Notice::ACTION_EMAIL,Notice::ACTION_LOG ssh_alerts@example.net 3600.000000 F - - - - - + #close 2018-12-13-22-56-35 + +.. note:: + + Keep in mind that the semantics of the :zeek:see:`SSH::Password_Guessing` + notice are such that it is only raised when Zeek heuristically detects a + failed login. + +Hooks can also have priorities applied to order their execution like events +with a default priority of 0. Greater values are executed first. Setting a hook +body to run before default hook bodies might look like this: + +.. code-block:: zeek + + hook Notice::policy(n: Notice::Info) &priority=5 + { + # Insert your code here. + } + +Hooks can also abort later hook bodies with the :zeek:see:`break` keyword. This +is primarily useful if one wants to completely preempt processing by lower +priority :zeek:see:`Notice::policy` hooks. + +.. _notice-policy-shortcuts: + +Notice Policy Shortcuts +----------------------- + +Although the notice framework provides a great deal of flexibility and +configurability there are many times that the full expressiveness isn’t needed +and actually becomes a hindrance to achieving results. The framework provides a +default :zeek:see:`Notice::policy` hook body as a way of giving users the +shortcuts to easily apply many common actions to notices. + +These are implemented as sets and tables indexed with a +:zeek:see:`Notice::Type` enum value. The following table shows and describes +all of the variables available for shortcut configuration of the notice +framework. + +.. list-table:: + :header-rows: 1 + + * - Variable name + - Description + + * - :zeek:see:`Notice::ignored_types` + - Adding a :zeek:see:`Notice::Type` to this set results in the notice being + ignored. It won’t have any other action applied to it, not even + :zeek:see:`Notice::ACTION_LOG`. + + * - :zeek:see:`Notice::emailed_types` + - Adding a :zeek:see:`Notice::Type` to this set results in + :zeek:see:`Notice::ACTION_EMAIL` being applied to the notices of that + type. + + * - :zeek:see:`Notice::alarmed_types` + - Adding a :zeek:see:`Notice::Type` to this set results in + :zeek:see:`Notice::ACTION_ALARM` being applied to the notices of that + type. + + * - :zeek:see:`Notice::not_suppressed_types` + - Adding a :zeek:see:`Notice::Type` to this set results in that notice no + longer undergoing the normal notice suppression that would take place. Be + careful when using this in production it could result in a dramatic + increase in the number of notices being processed. + + * - :zeek:see:`Notice::type_suppression_intervals` + - This is a table indexed on :zeek:see:`Notice::Type` and yielding an + interval. It can be used as an easy way to extend the default suppression + interval for an entire :zeek:see:`Notice::Type` without having to create + a whole :zeek:see:`Notice::policy` entry and setting the + ``$suppress_for`` field. + +.. _raising-notices: + +Raising Notices +=============== + +A script should raise a notice for any occurrence that a user may want to be +notified about or take action on. For example, whenever the base SSH analysis +script sees enough failed logins to a given host, it raises a notice of the +type :zeek:see:`SSH::Password_Guessing`. The code in the base SSH analysis +script which raises the notice looks like this: + +.. code-block:: zeek + + NOTICE([$note=Password_Guessing, + $msg=fmt("%s appears to be guessing SSH passwords (seen in %d connections).", key$host, r$num), + $src=key$host, + $identifier=cat(key$host)]); + +:zeek:see:`NOTICE` is a normal function in the global namespace which wraps a +function within the Notice namespace. It takes a single argument of the +:zeek:see:`Notice::Info` record type. The most common fields used when raising +notices are described in the following table: + +.. list-table:: + :header-rows: 1 + + * - Field name + - Description + + * - :zeek:field:`note` + - This field is required and is an enum value which represents the notice + type. + + * - :zeek:field:`msg` + - This is a human readable message which is meant to provide more + information about this particular instance of the notice type. + + * - :zeek:field:`sub` + - This is a sub-message meant for human readability but will frequently + also be used to contain data meant to be matched with the + :zeek:see:`Notice::policy`. + + * - :zeek:field:`conn` + - If a connection record is available when the notice is being raised and + the notice represents some attribute of the connection, then the + connection record can be given here. Other fields such as :zeek:field:`id` and :zeek:field:`src` + will automatically be populated from this value. + + * - :zeek:field:`id` + - If a :zeek:see:`conn_id` record is available when the notice is being + raised and the notice represents some attribute of the connection, then + the connection can be given here. Other fields such as :zeek:field:`src` will + automatically be populated from this value. + + * - :zeek:field:`src` + - If the notice represents an attribute of a single host then it’s possible + that only this field should be filled out to represent the host that is + being “noticed”. + + * - :zeek:field:`n` + - This normally represents a number if the notice has to do with some + number. It’s most frequently used for numeric tests in the + :zeek:see:`Notice::policy` for making policy decisions. + + * - :zeek:field:`identifier` + - This represents a unique identifier for this notice. This field is + described in more detail in the :ref:`Automated Suppression + ` section. + + * - :zeek:field:`suppress_for` + - This field can be set if there is a natural suppression interval for the + notice that may be different than the default value. The value set to + this field can also be modified by a user’s :zeek:see:`Notice::policy` so + the value is not set permanently and unchangeably. + +When writing Zeek scripts that raise notices, some thought should be given to +what the notice represents and what data should be provided to give a consumer +of the notice the best information about the notice. If the notice is +representative of many connections and is an attribute of a host (e.g., a +scanning host) it probably makes most sense to fill out the :zeek:field:`src` field and +not give a connection or :zeek:see:`conn_id`. If a notice is representative of +a connection attribute (e.g. an apparent SSH login) then it makes sense to fill +out either :zeek:field:`Notice::Info$conn` or :zeek:field:`Notice::Info$id` +based on the data that is available when the notice is raised. + +Using care when inserting data into a notice will make later analysis easier +when only the data to fully represent the occurrence that raised the notice is +available. If complete connection information is included when an SSL server +certificate is expiring, for example, the logs will be very confusing because +the connection that the certificate was detected on is a side topic to the fact +that an expired certificate was detected. It’s possible in many cases that two +or more separate notices may need to be generated. As an example, one could be +for the detection of the expired SSL certificate and another could be for if +the client decided to go ahead with the connection neglecting the expired +certificate. + +.. _automated-notice-suppression: + +Automated Suppression +===================== + +The notice framework supports suppression for notices if the author of the +script that is generating the notice has indicated to the notice framework how +to identify notices that are intrinsically the same. Identification of these +“intrinsically duplicate” notices is implemented with an optional field in +:zeek:see:`Notice::Info` records named :zeek:field:`Notice::Info$identifier` +which is a simple string. If the :zeek:field:`Notice::Info$identifier` and +:zeek:field:`Notice::Info$note` fields are the same for two notices, the notice +framework actually considers them to be the same thing and +can use that information to suppress duplicates for a configurable period of +time. + +.. note:: + + If the :zeek:field:`identifier` is left out of a notice, no notice suppression takes + place due to the framework’s inability to identify duplicates. This could be + completely legitimate usage if no notices could ever be considered to be + duplicates. + +The :zeek:field:`Notice::Info$identifier` field typically comprises several pieces of data related to +the notice that when combined represent a unique instance of that notice. Here +is an example of the script +:doc:`/scripts/policy/protocols/ssl/validate-certs.zeek` raising a notice for +session negotiations where the certificate or certificate chain did not +validate successfully against the available certificate authority certificates. + +.. code-block:: zeek + + NOTICE([$note=SSL::Invalid_Server_Cert, + $msg=fmt("SSL certificate validation failed with (%s)", c$ssl$validation_status), + $sub=c$ssl$subject, + $conn=c, + $identifier=cat(c$id$resp_h,c$id$resp_p,c$ssl$validation_status,c$ssl$cert_hash)]); + +In the above example you can see that the :zeek:field:`identifier` field contains a +string that is built from the responder IP address and port, the validation +status message, and the MD5 sum of the server certificate. Those fields in +particular are chosen because different SSL certificates could be seen on any +port of a host, certificates could fail validation for different reasons, and +multiple server certificates could be used on that combination of IP address +and port with the server_name SSL extension (explaining the addition of the MD5 +sum of the certificate). The result is that if a certificate fails validation +and all four pieces of data match (IP address, port, validation status, and +certificate hash) that particular notice won’t be raised again for the default +suppression period. + +Setting the :zeek:field:`Notice::Info$identifier` field is left to those raising notices because it’s +assumed that the script author who is raising the notice understands the full +problem set and edge cases of the notice which may not be readily apparent to +users. If users don’t want the suppression to take place or simply want a +different interval, they can set a notice’s suppression interval to ``0secs`` +or delete the value from the :zeek:field:`identifier` field in a +:zeek:see:`Notice::policy` hook. + +Extending Notice Framework +========================== + +There are a couple of mechanisms for extending the notice framework and adding +new capabilities. + +Configuring Notice Emails +------------------------- + +If :zeek:see:`Notice::mail_dest` is set, notices with an associated +e-mail action will be sent to that address. For additional +customization, users can use the :zeek:see:`Notice::policy` hook to +modify the :zeek:field:`Notice::Info$email_dest` field. The following example would result in three +separate e-mails: + +.. code-block:: zeek + + hook Notice::policy(n: Notice::Info) + { + n$email_dest = set( + "snow.white@example.net", + "doc@example.net", + "happy@example.net,sleepy@example.net,bashful@example.net" + ); + } + +You can also use :zeek:see:`Notice::policy` hooks to add extra information to +emails. The :zeek:see:`Notice::Info` record contains a vector of strings named +:zeek:field:`Notice::Info$email_body_sections` which Zeek will include verbatim when sending email. +An example of including some information from an HTTP request is included below. + +.. code-block:: zeek + + hook Notice::policy(n: Notice::Info) + { + if ( n?$conn && n$conn?$http && n$conn$http?$host ) + n$email_body_sections[|n$email_body_sections|] = fmt("HTTP host header: %s", n$conn$http$host); + } + +Cluster Considerations +====================== + +When running Zeek in a cluster, most of the information above stays the same. +Notices are generated, the :zeek:see:`Notice::policy` hook is evaluated, and +any actions are run on the node which generated the notice (most often a worker +node). Of note to users/developers of Zeek is that any files or access needed +to run the notice actions must be available to the respective node(s). + +The role of the manager is to receive and distribute notice suppression +information, so that duplicate notices do not get generated. Bear in mind that +some amount of latency is intrinsic in this synchronization, so it’s +possible that rapidly-generating notices will be duplicates. In this case, any +actions will also execute multiple times, once by each notice-generating +node. + +The Weird Log +============= + +A wide range of “weird” activity detected by Zeek can trigger corresponding +events that inform the script layer of this activity. These events exist at +various granularities, including :zeek:see:`conn_weird`, +:zeek:see:`flow_weird`, :zeek:see:`net_weird`, :zeek:see:`file_weird`, and +others. Built atop the notice framework, the :doc:`Weird +` module implements event handlers +that funnel the various “weirds” into the usual notice framework handlers. To +get an idea of the available weird-types, take a look at the +:zeek:see:`Weird::actions` table, which defines default actions for the various +types of activity. Weirds generally do not indicate security-relevant activity +— they’re just, well, weird things that you generally wouldn’t expect to +happen, such as odd TCP state machine violations, unexpected HTTP header +constellations, or DNS message properties that fall outside of the relevant RFC +specifications. That is, don’t consider them actionable detections in an IDS +sense, though they might well provide meaningful additional clues for a +security incident. + +The notice type for weirds is :zeek:see:`Weird::Activity`. You have a wide range of actions at +your disposal for how to handle weirds: you can ignore them, log them, or have +them trigger notice, all at various reduction/filtering granularities (see the +:zeek:see:`Weird::Action` enum values for details). For dynamic filtering, the +:zeek:see:`Weird::ignore_hosts` and :zeek:see:`Weird::weird_ignore` sets allow +exclusion of activity from reporting. + +The framework provides a few additional tuning knobs. See +:doc:`/scripts/base/frameworks/notice/weird.zeek` for details. diff --git a/doc/frameworks/notice_ssh_guesser.zeek b/doc/frameworks/notice_ssh_guesser.zeek new file mode 100644 index 0000000000..34ffe2e95e --- /dev/null +++ b/doc/frameworks/notice_ssh_guesser.zeek @@ -0,0 +1,10 @@ + +@load protocols/ssh/detect-bruteforcing + +redef SSH::password_guesses_limit=10; + +hook Notice::policy(n: Notice::Info) + { + if ( n$note == SSH::Password_Guessing && /192\.168\.56\.103/ in n$sub ) + add n$actions[Notice::ACTION_EMAIL]; + } diff --git a/doc/frameworks/packet-analysis-1-ethernet.zeek b/doc/frameworks/packet-analysis-1-ethernet.zeek new file mode 100644 index 0000000000..bea6014bcd --- /dev/null +++ b/doc/frameworks/packet-analysis-1-ethernet.zeek @@ -0,0 +1,26 @@ +module PacketAnalyzer::ETHERNET; + +export { + ## Default analyzer + const default_analyzer: PacketAnalyzer::Tag = PacketAnalyzer::ANALYZER_IP &redef; + + ## IEEE 802.2 SNAP analyzer + global snap_analyzer: PacketAnalyzer::Tag &redef; + ## Novell raw IEEE 802.3 analyzer + global novell_raw_analyzer: PacketAnalyzer::Tag &redef; + ## IEEE 802.2 LLC analyzer + global llc_analyzer: PacketAnalyzer::Tag &redef; +} + +event zeek_init() &priority=20 + { + PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ETHERNET, 0x8847, PacketAnalyzer::ANALYZER_MPLS); + PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ETHERNET, 0x0800, PacketAnalyzer::ANALYZER_IP); + PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ETHERNET, 0x86DD, PacketAnalyzer::ANALYZER_IP); + PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ETHERNET, 0x0806, PacketAnalyzer::ANALYZER_ARP); + PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ETHERNET, 0x8035, PacketAnalyzer::ANALYZER_ARP); + PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ETHERNET, 0x8100, PacketAnalyzer::ANALYZER_VLAN); + PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ETHERNET, 0x88A8, PacketAnalyzer::ANALYZER_VLAN); + PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ETHERNET, 0x9100, PacketAnalyzer::ANALYZER_VLAN); + PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ETHERNET, 0x8864, PacketAnalyzer::ANALYZER_PPPOE); + } diff --git a/doc/frameworks/packet-analysis-2-llc.cc b/doc/frameworks/packet-analysis-2-llc.cc new file mode 100644 index 0000000000..aa980b39de --- /dev/null +++ b/doc/frameworks/packet-analysis-2-llc.cc @@ -0,0 +1,18 @@ +bool LLCDemo::AnalyzePacket(size_t len, const uint8_t* data, Packet* packet) { + // Rudimentary parsing of 802.2 LLC + if ( 17 >= len ) { + packet->Weird("truncated_llc_header"); + return false; + } + + if ( ! llc_demo_message ) + return true; + + auto dsap = data[14]; + auto ssap = data[15]; + auto control = data[16]; + + event_mgr.Enqueue(llc_demo_message, val_mgr->Count(dsap), val_mgr->Count(ssap), val_mgr->Count(control)); + + return true; +} diff --git a/doc/frameworks/packet-analysis-pdu.svg b/doc/frameworks/packet-analysis-pdu.svg new file mode 100644 index 0000000000..65c5ed7c74 --- /dev/null +++ b/doc/frameworks/packet-analysis-pdu.svg @@ -0,0 +1 @@ +Layer1PDULayer2PDULayer1PayloadLayer2PayloadIDID \ No newline at end of file diff --git a/doc/frameworks/packet-analysis.rst b/doc/frameworks/packet-analysis.rst new file mode 100644 index 0000000000..b77ef607a4 --- /dev/null +++ b/doc/frameworks/packet-analysis.rst @@ -0,0 +1,170 @@ + +.. _packet-analysis: + +=============== +Packet Analysis +=============== + +.. TODO: integrate BoZ revisions + +.. rst-class:: opening + + The Packet Analysis plugin architecture handles parsing of packet headers at + layers below Zeek's existing Session analysis. In particular, this allows to + add new link and network layer protocols to Zeek. + This section provides an overview of the underlying architecture as well as + an example-based walk-through. For further details, consider to take a look + at the built-in packet analyzers as well as the packet analyzer tests. + +The Flow of Packets +=================== + +The basic packet flow through Zeek is as follows. First, an ``IOSource`` deals with +getting the packets into Zeek. While an ``IOSource`` can be used to interface all +sorts of capturing mechanisms, the default source makes use of libpcap to either +read PCAP files or sniff an interface. Once acquired, a packet is handed into +the packet analysis and processed layer by layer. + +.. figure:: packet-analysis-pdu.svg + :width: 600 + :align: center + :alt: Nesting of Protocol Data Units (PDUs) + :target: ../_images/netcontrol-openflow.png + + Nesting of Protocol Data Units (PDUs). + +At the lower layers, Protocol Data Units (PDUs) typically consist of a header +and a payload, where the payload is the next layer's PDU and the header carries +a numeric identifier that determines the encapsulated protocol (see figure +above, where "ID" denotes the location of such a numeric protocol identifier +within the header). + +Each packet analyzer parses the packet's header according to the implemented +protocol, determines a suitable analyzer for the encapsulated protocol and hands its +payload to that next analyzer. Once the IP layer is reached, packet analysis is +finished and Zeek continues by constructing a session for the observed +connection. After session analysis, which includes processing of TCP and UDP, +the packet continues its journey into the land of application layer analyzers. +There, Dynamic Protocol Detection is used to determine the application layer +protocol and continue the analysis. + +Packet Analyzer Configuration +============================= + +The following script shows an example configuration of the Ethernet packet +analyzer: + +.. literalinclude:: packet-analysis-1-ethernet.zeek + :caption: + :language: zeek + :linenos: + :tab-width: 4 + +Within :zeek:see:`zeek_init`, various EtherType-to-PacketAnalyzer mappings are +registered by using :zeek:see:`PacketAnalyzer::register_packet_analyzer`. For +example, for EtherType ``0x8864``, the packet's payload is passed to the PPPoE +analyzer. + +The ``default_analyzer`` analyzer specifies which packet analyzer to use if +none of the mappings matched. In case of Ethernet, we try to fall back to IP. + +Furthermore, Ethernet needs to handle different types of frames, with three of +them identified using the first payload bytes (see `Wikipedia +`_). +As the EtherType needs to be interpreted with respect to the frame type in these +cases, the Ethernet analyzer provides three additional configuration parameters, +``snap_analyzer``, ``novell_raw_analyzer``, and ``llc_analyzer``. +to configure analyzers that handle the different frame types. + +.. note:: + + There are a few conventions involved here: + + * The name of the module is expected to be ``PacketAnalyzer::``. + * The default analyzer is expected to be named ``default_analyzer``. + +Packet analysis starts at a root analyzer that dispatches based on the link +types obtained from the ``IOSource``. Accordingly +:doc:`/scripts/base/packet-protocols/root/main.zeek` contains the following +call to integrate the Ethernet analyzer: + +.. code-block:: zeek + + PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ROOT, DLT_EN10MB, PacketAnalyzer::ANALYZER_ETHERNET); + +Packet Analyzer API +=================== + +Just like for other parts of Zeek, a plugin may provide a packet analyzer by +adding a packet analysis component that instantiates an analyzer. The packet +analyzer itself is implemented by inheriting from +``zeek::packet_analysis::Analyzer`` and overriding the ``AnalyzePacket()`` +method. The following is an excerpt from a test case that shows the exemplary +analysis of LLC: + +.. literalinclude:: packet-analysis-2-llc.cc + :caption: + :language: C++ + :linenos: + :tab-width: 4 + +First, we verify that the size of the packet matches what we expect. If that is +not the case, we create a weird using the ``Packet`` object that is passed along +the chain of analyzers. To signal that the analysis failed, the method returns +``false``. For valid packets, we just read some protocol-specific values. As of +now, there is no mechanism to pass extracted meta data on to other analyzers. +While it is possible to trigger events that receive these values as parameters, +keep in mind that handling events for every packet can be *extremely +expensive*. However, for our test case we defined an event as follows in a +separate ``.bif`` file: + +.. code-block:: zeek + + event llc_demo_message%(dsap: count, ssap: count, control: count%); + +Before we can expect the event to be generated, we need to integrate the +analyzer. The configuration might be included in the scripts that are shipped +with the packet analyzer. For example, one could add a new EtherType by +adding a call to :zeek:see:`PacketAnalyzer::register_packet_analyzer` from +within a :zeek:see:`zeek_init` event handler. +For the LLC example we redefine one of the additional constants: + +.. code-block:: zeek + + redef PacketAnalyzer::ETHERNET::llc_analyzer = PacketAnalyzer::ANALYZER_LLC_DEMO; + +In this example, packet analysis as well as all further analysis ends with the +LLC analyzer. The ``ForwardPacket()`` method can be used to pass data to another +packet analyzer. The method takes a pointer to the beginning of the data to +process (usually the start of the payload in the current context), the length of +the data to process, a pointer to the ``Packet`` object and an identifier. The +identifier would be used to lookup the next analyzer based on which other +analyzers were previously associated with LLC as a parent analyzer in a call to +:zeek:see:`PacketAnalyzer::register_packet_analyzer`. If there is no +previously-registered analyzer that matches the identifier, it will fall back +to the ``default_analyzer`` if available. + +In case a packet analyzer requires initialization, e.g., reading additional +configuration values from script-land, this can be implemented by overriding +the ``Initialize()`` method. When overriding this method, always make sure to +call the base-class version to ensure proper initialization. + +With the addition of the transport-layer analyzer to the packet analysis framework, +it's now possible to register for ports as the identifier. This is natural, given +that a port number is just another numeric identifier for moving from one +protocol to another. Packet analyzers should call +``PacketAnalyzer::register_for_port`` or ``PacketAnalyzer::register_for_ports`` +to ensure that the ports are also stored in the global ``Analyzer::ports`` table for +use with BPF filters. + +The packet analysis framework also provides a ``register_protocol_detection`` +method that is used to register a packet analyzer to use protocol detection +instead of using a numeric identifier. Analyzers can use this method and then +override ``Analyzer::DetectProtocol`` to search the packet data for byte strings +or other markers to detect whether a protocol exists in the data. This is similar +to how DPD works for non-packet analyzers, but is not limited to pattern matching. + +.. note:: + + When writing your own packet analyzer, take a look into the existing code to + identify idiomatic ways to handle tasks like looking up configuration values. diff --git a/doc/frameworks/signatures.rst b/doc/frameworks/signatures.rst new file mode 100644 index 0000000000..c1dc8dbd18 --- /dev/null +++ b/doc/frameworks/signatures.rst @@ -0,0 +1,486 @@ + +=================== +Signature Framework +=================== + +.. rst-class:: opening + + Zeek relies primarily on its extensive scripting language for + defining and analyzing detection policies, but it also + provides an independent *signature language* for doing + low-level, Snort-style pattern matching. While signatures are + *not* Zeek's preferred detection tool, they sometimes come in handy + and are closer to what many people are familiar with from using + other NIDS. This page gives a brief overview on Zeek's signatures + and covers some of their technical subtleties. + +Basics +====== + +Let's look at an example signature first:: + + signature my-first-sig { + ip-proto == tcp + dst-port == 80 + payload /.*root/ + event "Found root!" + } + +This signature asks Zeek to match the regular expression ``.*root`` on +all TCP connections going to port 80. When the signature triggers, Zeek +will raise an event :zeek:id:`signature_match` of the form: + +.. code-block:: zeek + + event signature_match(state: signature_state, msg: string, data: string) + +Here, ``state`` contains more information on the connection that +triggered the match, ``msg`` is the string specified by the +signature's event statement (``Found root!``), and data is the last +piece of payload which triggered the pattern match. + +.. versionadded:: 7.1 + +An alternative form of :zeek:id:`signature_match` has an additional ``end_of_match`` parameter. + +.. code-block:: zeek + + event signature_match(state: signature_state, msg: string, data: string, end_of_match: count) + +The ``end_of_match`` parameter represents the offset into ``data`` where +the match ended, or said differently, the length of the matching data within ``data``. +If a signature matches across packet boundaries, ``data`` contains just the +payload of the packet where a signature match triggered. + +To turn such :zeek:id:`signature_match` events into actual alarms, you can +load Zeek's :doc:`/scripts/base/frameworks/signatures/main.zeek` script. +This script contains a default event handler that raises +:zeek:enum:`Signatures::Sensitive_Signature` :doc:`Notices ` +(as well as others; see the beginning of the script). + +As documented in :ref:`signatures-actions`, it's possible to use a custom +event instead of :zeek:id:`signature_match`. + +As signatures are independent of Zeek's scripts, they are put into +their own file(s). There are three ways to specify which files contain +signatures: By using the ``-s`` flag when you invoke Zeek, or by +extending the Zeek variable :zeek:id:`signature_files` using the ``+=`` +operator, or by using the ``@load-sigs`` directive inside a Zeek script. +If a signature file is given without a full path, it is searched for +along the normal ``ZEEKPATH``. Additionally, the ``@load-sigs`` +directive can be used to load signature files in a path relative to the +Zeek script in which it's placed, e.g. ``@load-sigs ./mysigs.sig`` will +expect that signature file in the same directory as the Zeek script. The +default extension of the file name is ``.sig``, and Zeek appends that +automatically when necessary. + +Signature Language for Network Traffic +====================================== + +Let's look at the format of a signature more closely. Each individual +signature has the format ``signature { }``, where ```` +is a unique label for the signature. There are two types of +attributes: *conditions* and *actions*. The conditions define when the +signature matches, while the actions declare what to do in the case of +a match. Conditions can be further divided into four types: *header*, +*content*, *dependency*, and *context*. We discuss these all in more +detail in the following. + +Conditions +---------- + +Header Conditions +~~~~~~~~~~~~~~~~~ + +Header conditions limit the applicability of the signature to a subset +of traffic that contains matching packet headers. This type of matching +is performed only for the first packet of a connection. + +There are pre-defined header conditions for some of the most used +header fields. All of them generally have the format `` +``, where ```` names the header field; ``cmp`` is +one of ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``; and +```` is a list of comma-separated values or value-ranges to +compare against (e.g. ``5,7-10`` for numbers 5 to 10, excluding 6). +The following keywords are defined: + +``src-ip``/``dst-ip `` + Source and destination address, respectively. Addresses can be given + as IPv4 or IPv6 addresses or CIDR masks. For IPv6 addresses/masks + the colon-hexadecimal representation of the address must be enclosed + in square brackets (e.g. ``[fe80::1]`` or ``[fe80::0]/16``). + +``src-port``/``dst-port `` + Source and destination port, respectively. + +``ip-proto tcp|udp|icmp|icmp6|ip|ip6`` + IPv4 header's Protocol field or the Next Header field of the final + IPv6 header (i.e. either Next Header field in the fixed IPv6 header + if no extension headers are present or that field from the last + extension header in the chain). Note that the IP-in-IP forms of + tunneling are automatically decapsulated by default and signatures + apply to only the inner-most packet, so specifying ``ip`` or ``ip6`` + is a no-op. + +For lists of multiple values, they are sequentially compared against +the corresponding header field. If at least one of the comparisons +evaluates to true, the whole header condition matches (exception: with +``!=``, the header condition only matches if all values differ). + +In addition to these pre-defined header keywords, a general header +condition can be defined either as:: + + header [:] [& ] + +This compares the value found at the given position of the packet header +with a list of values. ``offset`` defines the position of the value +within the header of the protocol defined by ``proto`` (which can be +``ip``, ``ip6``, ``tcp``, ``udp``, ``icmp`` or ``icmp6``). ``size`` is +either 1, 2, or 4 and specifies the value to have a size of this many +bytes. If the optional ``& `` is given, the packet's value is +first masked with the integer before it is compared to the value-list. +``cmp`` is one of ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``. +``value-list`` is a list of comma-separated integers or integer-ranges +similar to those described above. The integers within the list may be +followed by an additional ``/ mask`` where ``mask`` is a value from 0 to 32. +This corresponds to the CIDR notation for netmasks and is translated into a +corresponding bitmask applied to the packet's value prior to the +comparison (similar to the optional ``& integer``). IPv6 address values +are not allowed in the value-list, though you can still inspect any 1, +2, or 4 byte section of an IPv6 header using this keyword. + +Putting it all together, this is an example condition that is +equivalent to ``dst-ip == 1.2.3.4/16, 5.6.7.8/24``:: + + header ip[16:4] == 1.2.3.4/16, 5.6.7.8/24 + +Note that the analogous example for IPv6 isn't currently possible since +4 bytes is the max width of a value that can be compared. + +Content Conditions +~~~~~~~~~~~~~~~~~~ + +Content conditions are defined by regular expressions. We +differentiate two kinds of content conditions: first, the expression +may be declared with the ``payload`` statement, in which case it is +matched against the raw payload of a connection (for reassembled TCP +streams) or of each packet (for ICMP, UDP, and non-reassembled TCP). +Second, it may be prefixed with an analyzer-specific label, in which +case the expression is matched against the data as extracted by the +corresponding analyzer. + +A ``payload`` condition has the form:: + + payload // + +Currently, the following analyzer-specific content conditions are +defined (note that the corresponding analyzer has to be activated by +loading its policy script): + +``http-request //`` + The regular expression is matched against decoded URIs of HTTP + requests. Obsolete alias: ``http``. + +``http-request-header //`` + The regular expression is matched against client-side HTTP headers. + +``http-request-body //`` + The regular expression is matched against client-side bodys of + HTTP requests. + +``http-reply-header //`` + The regular expression is matched against server-side HTTP headers. + +``http-reply-body //`` + The regular expression is matched against server-side bodys of + HTTP replies. + +``ftp //`` + The regular expression is matched against the command line input + of FTP sessions. + +``finger //`` + The regular expression is matched against finger requests. + +For example, ``http-request /.*(etc/(passwd|shadow)/`` matches any URI +containing either ``etc/passwd`` or ``etc/shadow``. To filter on request +types, e.g. ``GET``, use ``payload /GET /``. + +Note that HTTP pipelining (that is, multiple HTTP transactions in a +single TCP connection) has some side effects on signature matches. If +multiple conditions are specified within a single signature, this +signature matches if all conditions are met by any HTTP transaction +(not necessarily always the same!) in a pipelined connection. + +Dependency Conditions +~~~~~~~~~~~~~~~~~~~~~ + +To define dependencies between signatures, there are two conditions: + + +``requires-signature [!] `` + Defines the current signature to match only if the signature given + by ``id`` matches for the same connection. Using ``!`` negates the + condition: The current signature only matches if ``id`` does not + match for the same connection (using this defers the match + decision until the connection terminates). + +``requires-reverse-signature [!] `` + Similar to ``requires-signature``, but ``id`` has to match for the + opposite direction of the same connection, compared to the current + signature. This allows to model the notion of requests and + replies. + +Context Conditions +~~~~~~~~~~~~~~~~~~ + +Context conditions pass the match decision on to other components of +Zeek. They are only evaluated if all other conditions have already +matched. The following context conditions are defined: + +``eval `` + The given policy function is called and has to return a boolean + confirming the match. If false is returned, no signature match is + going to be triggered. The function has to be of type ``function + cond(state: signature_state, data: string): bool``. Here, + ``data`` may contain the most recent content chunk available at + the time the signature was matched. If no such chunk is available, + ``data`` will be the empty string. See :zeek:type:`signature_state` + for its definition. + +``payload-size `` + Compares the integer to the size of the payload of a packet. For + reassembled TCP streams, the integer is compared to the size of + the first in-order payload chunk. Note that the latter is not very + well defined. + +``same-ip`` + Evaluates to true if the source address of the IP packets equals + its destination address. + +``tcp-state `` + Imposes restrictions on the current TCP state of the connection. + ``state-list`` is a comma-separated list of the keywords + ``established`` (the three-way handshake has already been + performed), ``originator`` (the current data is send by the + originator of the connection), and ``responder`` (the current data + is send by the responder of the connection). + +``udp-state `` + Imposes restrictions on which UDP flow direction to match. ``state-list`` + is a comma-separated list of either ``originator`` (the current data is + send by the originator of the connection) or ``responder`` (the current + data is send by the responder of the connection). The ``established`` + state is rejected as an error in the signature since it does not have a + useful meaning like it does for TCP. + +.. _signatures-actions: + +Actions +------- + +Actions define what to do if a signature matches. Currently, there are +two actions defined, ``event`` and ``enable``. + +``event `` + Raises a :zeek:id:`signature_match` event. The event handler has either + of the following types: + + .. code-block:: zeek + + event signature_match(state: signature_state, msg: string, data: string) + + event signature_match(state: signature_state, msg: string, data: string, end_of_match: count) + + The given string is passed in as ``msg``, and data is the current + part of the payload that has eventually lead to the signature + match (this may be empty for signatures without content + conditions). The ``end_of_match`` parameter represents the length of + the matching payload in ``data``. + +``event event_name [string]`` + + .. versionadded:: 6.2 + + To raise a custom event, the event's name can be inserted before the string:: + + event my_signature_match "Found root!" + + Instead of :zeek:id:`signature_match`, this raises ``my_signature_match``. + The parameters for the ``my_signature_match`` event are expected to be the + same as for :zeek:id:`signature_match`. + + It is further possible to omit the string altogether:: + + event found_root + + In this case, the type of the ``found_root`` event handler does not have + a ``msg`` parameter: + + .. code-block:: zeek + + event found_root(state: signature_state, data: string) + + Like the :zeek:id:`signature_match` event, custom events can have an additional + ``end_of_match`` parameter. + + .. code-block:: zeek + + event found_root(state: signature_state, data: string, end_of_match: count) + + .. note:: + + Matches for signatures that use custom events do not appear + in :file:`signatures.log`. + + +``enable `` + Enables the protocol analyzer ```` for the matching + connection (``"http"``, ``"ftp"``, etc.). This is used by Zeek's + dynamic protocol detection to activate analyzers on the fly. + +Signature Language for File Content +=================================== + +The signature framework can also be used to identify MIME types of files +irrespective of the network protocol/connection over which the file is +transferred. A special type of signature can be written for this +purpose and will be used automatically by the :doc:`Files Framework +` or by Zeek scripts that use the :zeek:see:`file_magic` +built-in function. + +Conditions +---------- + +File signatures use a single type of content condition in the form of a +regular expression: + +``file-magic //`` + +This is analogous to the ``payload`` content condition for the network +traffic signature language described above. The difference is that +``payload`` signatures are applied to payloads of network connections, +but ``file-magic`` can be applied to any arbitrary data, it does not +have to be tied to a network protocol/connection. + +Actions +------- + +Upon matching a chunk of data, file signatures use the following action +to get information about that data's MIME type: + +``file-mime [, ]`` + +The arguments include the MIME type string associated with the file +magic regular expression and an optional "strength" as a signed integer. +Since multiple file magic signatures may match against a given chunk of +data, the strength value may be used to help choose a "winner". Higher +values are considered stronger. + +Things to keep in mind when writing signatures +============================================== + +* Each signature is reported at most once for every connection, + further matches of the same signature are ignored. + +* The content conditions perform pattern matching on elements + extracted from an application protocol dialogue. For example, ``http + /.*passwd/`` scans URLs requested within HTTP sessions. The thing to + keep in mind here is that these conditions only perform any matching + when the corresponding application analyzer is actually *active* for + a connection. Note that by default, analyzers are not enabled if the + corresponding Zeek script has not been loaded. A good way to + double-check whether an analyzer "sees" a connection is checking its + log file for corresponding entries. If you cannot find the + connection in the analyzer's log, very likely the signature engine + has also not seen any application data. + +* As the name indicates, the ``payload`` keyword matches on packet + *payload* only. You cannot use it to match on packet headers; use + the header conditions for that. + +* For TCP connections, header conditions are only evaluated for the + *first packet from each endpoint*. If a header condition does not + match the initial packets, the signature will not trigger. Zeek + optimizes for the most common application here, which is header + conditions selecting the connections to be examined more closely + with payload statements. + +* For UDP and ICMP flows, the payload matching is done on a per-packet + basis; i.e., any content crossing packet boundaries will not be + found. For TCP connections, the matching semantics depend on whether + Zeek is *reassembling* the connection (i.e., putting all of a + connection's packets in sequence). By default, Zeek is reassembling + the first 1K of every TCP connection, which means that within this + window, matches will be found without regards to packet order or + boundaries (i.e., *stream-wise matching*). + +* For performance reasons, by default Zeek *stops matching* on a + connection after seeing 1K of payload; see the section on options + below for how to change this behaviour. The default was chosen with + Zeek's main user of signatures in mind: dynamic protocol detection + works well even when examining just connection heads. + +* Regular expressions are implicitly anchored, i.e., they work as if + prefixed with the ``^`` operator. For reassembled TCP connections, + they are anchored at the first byte of the payload *stream*. For all + other connections, they are anchored at the first payload byte of + each packet. To match at arbitrary positions, you can prefix the + regular expression with ``.*``, as done in the examples above. + +* To match on non-ASCII characters, Zeek's regular expressions support + the ``\x`` operator. CRs/LFs are not treated specially by the + signature engine and can be matched with ``\r`` and ``\n``, + respectively. Generally, Zeek follows `flex's regular expression + syntax + `_. + See the DPD signatures in ``base/frameworks/dpd/dpd.sig`` for some examples + of fairly complex payload patterns. + +* The data argument of the :zeek:id:`signature_match` handler might not carry + the full text matched by the regular expression. Zeek performs the + matching incrementally as packets come in; when the signature + eventually fires, it can only pass on the most recent chunk of data. + + +Options +======= + +The following options control details of Zeek's matching process: + +* :zeek:see:`dpd_reassemble_first_packets` + + If true, Zeek reassembles the beginning of every TCP connection (of + up to :zeek:see:`dpd_buffer_size` bytes, see below also), to facilitate + reliable matching across packet boundaries. If false, only + connections are reassembled for which an application-layer + analyzer gets activated (e.g., by Zeek's dynamic protocol + detection). + +* :zeek:see:`dpd_match_only_beginning` + + If true, Zeek performs packet matching only within the initial payload + window of :zeek:see:`dpd_buffer_size`. If false, it keeps matching + on subsequent payload as well. + +* :zeek:see:`dpd_buffer_size` + + Defines the buffer size for the two preceding options. In + addition, this value determines the amount of bytes Zeek buffers + for each connection in order to activate application analyzers + even after parts of the payload have already passed through. This + is needed by the dynamic protocol detection capability to defer + the decision of which analyzers to use. + +So, how about using Snort signatures with Zeek? +=============================================== + +There was once a script, ``snort2bro``, that converted Snort signatures +automatically into Zeek's (then called "Bro") signature syntax. +However, in our experience this didn't turn out to be a very useful +thing to do because by simply using Snort signatures, one can't benefit +from the additional capabilities that Zeek provides; the approaches of +the two systems are just too different. We therefore stopped maintaining +the ``snort2bro`` script, and there are now many newer Snort options +which it doesn't support. The script is now no longer part of the Zeek +distribution. diff --git a/doc/frameworks/sqlite-conn-filter.zeek b/doc/frameworks/sqlite-conn-filter.zeek new file mode 100644 index 0000000000..2e4a455f3e --- /dev/null +++ b/doc/frameworks/sqlite-conn-filter.zeek @@ -0,0 +1,12 @@ +event zeek_init() + { + local filter: Log::Filter = + [ + $name="sqlite", + $path="/var/db/conn", + $config=table(["tablename"] = "conn"), + $writer=Log::WRITER_SQLITE + ]; + + Log::add_filter(Conn::LOG, filter); + } diff --git a/doc/frameworks/sqlite-read-events.zeek b/doc/frameworks/sqlite-read-events.zeek new file mode 100644 index 0000000000..f14513e0cb --- /dev/null +++ b/doc/frameworks/sqlite-read-events.zeek @@ -0,0 +1,40 @@ +@load frameworks/files/hash-all-files + +type Val: record { + hash: string; + description: string; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, r: Val) + { + print fmt("malware-hit with hash %s, description %s", r$hash, r$description); + } + +global malware_source = "/var/db/malware"; + +event file_hash(f: fa_file, kind: string, hash: string) + { + + # check all sha1 hashes + if ( kind=="sha1" ) + { + Input::add_event( + [ + $source=malware_source, + $name=hash, + $fields=Val, + $ev=line, + $want_record=T, + $config=table( + ["query"] = fmt("select * from malware_hashes where hash='%s';", hash) + ), + $reader=Input::READER_SQLITE + ]); + } + } + +event Input::end_of_data(name: string, source:string) + { + if ( source == malware_source ) + Input::remove(name); + } diff --git a/doc/frameworks/sqlite-read-table.zeek b/doc/frameworks/sqlite-read-table.zeek new file mode 100644 index 0000000000..12499f0c4d --- /dev/null +++ b/doc/frameworks/sqlite-read-table.zeek @@ -0,0 +1,35 @@ +type Idx: record { + host: addr; +}; + +type Val: record { + users: set[string]; +}; + +global hostslist: table[addr] of Val = table(); + +event zeek_init() + { + Input::add_table([$source="/var/db/hosts", + $name="hosts", + $idx=Idx, + $val=Val, + $destination=hostslist, + $reader=Input::READER_SQLITE, + $config=table(["query"] = "select * from machines_to_users;") + ]); + + Input::remove("hosts"); + } + +event Input::end_of_data(name: string, source: string) + { + if ( name != "hosts" ) + return; + + # now all data is in the table + print "Hosts list has been successfully imported"; + + # List the users of one host. + print hostslist[192.168.17.1]$users; + } diff --git a/doc/frameworks/storage.rst b/doc/frameworks/storage.rst new file mode 100644 index 0000000000..98fc5c7688 --- /dev/null +++ b/doc/frameworks/storage.rst @@ -0,0 +1,216 @@ +.. _framework-storage: + +.. versionadded:: 7.2 + +================= +Storage Framework +================= + +The storage framework provides a plugin-based system for short- and long-term storage of +data, accessible from Zeek script-land. This is not packet data itself, but data artifacts +generated from the packet data. It has interchangeable asynchronous and synchronous +modes. The framework provides just a simple key-value store, using Zeek values as the keys +to store and lookup data. + +This chapter gives an overview of the storage framework, plus examples of using it. For +more examples, see the test cases in ``testing/btest/scripts/base/frameworks/storage`` and +an example storage plugin in ``testing/btest/plugin/storage-src``. + +Terminology +=========== + +Zeek's storage framework uses two main components: + + Backend + A backend plugin provides access to a storage system. Backends can be network-based + storage systems such as Redis, on-disk database systems such as SQLite, etc. Backend + plugins can define script-level records for configuring them when they're opened. Zeek + provides backends for Redis and SQLite by default, but others may be implemented as + external packages. + + Serializer + A serializer plugin provides a mechanism for converting data from Zeek scripts into + formats that backends can use. Serializers are intended to be agnostic to + backends. They convert between Zeek values and opaque byte buffers, and backends + should be able to handle the result of any individual serializer. Zeek provides a JSON + serializer by default, but others may be implemented as external packages. + +Asynchronous Mode vs Synchronous Mode +===================================== + +Storage backends support both asynchronous and synchronous modes. The difference between +using the two modes is that asynchronous calls must be used as part of :zeek:see:`when` +statements, whereas synchronous calls can be used either with ``when`` statements or +called directly. Synchronous functions will block until the backend returns +data. Otherwise, all of the arguments and return values are the same between them. They +are split between two script-level modules: :zeek:see:`Storage::Async` loaded from +``base/frameworks/storage/async`` and :zeek:see:`Storage::Sync` loaded from +``base/frameworks/storage/sync``. + +When reading pcap data via the ``-r`` Zeek argument, all backends operate in a synchronous +manner internally to ensure that Zeek's timers run correctly. Regardless of this behavior, +asynchronous functions are required to be used with the ``when`` statement, but they'll +essentially be translated to synchronous calls. + +Using the Storage Framework +=========================== + +All of the examples below use the SQLite backend. Usage of other backends follows the same +model. Switching the examples to a different backend involves only using a different tag +and options record with the :zeek:see:`Storage::Async::open_backend`/ +:zeek:see:`Storage::Sync::open_backend` functions. + +Operation Return Values +----------------------- + +All backend methods return a record of type :zeek:see:`Storage::OperationResult`. This +record contains a code that indicates the result of the operation. For failures, backends +may provide more details in the optional error message. The record will also contain data +for operations that return values, namely ``open_backend`` or ``get``. +:zeek:see:`Storage::ReturnCode` contains all of the codes that can be returned from the +various operations. Not all codes are valid for all operations. +:zeek:see:`Storage::ReturnCode` can be redefined by backends to add new backend-specific +statuscodes as needed. + +.. _storage-opening-closing: + +Opening and Closing a Backend +----------------------------- + +Opening a backend starts with defining a set of options for that backend. The +:zeek:see:`Storage::BackendOptions` is defined with some fields by default, but loading a +policy for a specific backend type may add new fields to it. In the example below, we +loaded the SQLite policy, which adds a new ``sqlite`` field with additional options. These +options are filled in to denote where to store the sqlite database file and what table to +use. This allows users to separate different instances of a backend from each other in a +single database file. + +The script then sets a serializer. The storage framework sets this to the JSON +(:zeek:see:`Storage::STORAGE_SERIALIZER_JSON`) serializer by default, but setting it +explicitly is included below as an example. + +Calling :zeek:see:`Storage::Sync::open_backend` instantiates a backend connection. As +described above, ``open_backend`` returns a :zeek:see:`Storage::OperationResult`. On +success, it stores the handle to the backend in the ``value`` field of the result +record. We check the ``code`` field as well to make sure the operation succeeded. Backend +handles can be stored in global values just like any other value. They can be opened +during startup, such as in a :zeek:see:`zeek_init` event handler, and reused throughout +the runtime of Zeek. When a backend is successfully opened, a +:zeek:see:`Storage::backend_opened` event will be emitted. + +The two type arguments to ``open_backend`` define the script-level types for keys and +values. Attempting to use other types with the backend results in +:zeek:see:`Storage::KEY_TYPE_MISMATCH` errors. + +Lastly, we call :zeek:see:`Storage::Sync::close_backend` to close the backend before +exiting. When a backend is successfully closed, a :zeek:see:`Storage::backend_lost` event +will be emitted. + +.. code-block:: zeek + + @load base/frameworks/storage/sync + @load policy/frameworks/storage/backend/sqlite + + local backend_opts: Storage::BackendOptions; + local backend: Storage::BackendHandle; + + # Loading the sqlite policy adds this field to the options record. + opts$sqlite = [$database_path="test.sqlite", $table_name="testing"]; + + # This is the default, but is shown here for how to set it. + opts$serializer = Storage::STORAGE_SERIALIZER_JSON; + + local res = Storage::Sync::open_backend(Storage::STORAGE_BACKEND_SQLITE, opts, string, string); + if ( res$code == Storage::SUCCESS ) + backend = res$value; + + res = Storage::Sync::close_backend(backend); + +Storing, Retrieving, and Erasing Data +------------------------------------- + +The true point of the storage framework is to store and retrieve data. This example shows +making synchronous calls to add a new key/value pair to a backend, retrieve it, and erase +the entry associated with the key. This assumes that the ``backend`` variable used below +points to an opened backend handle. The idea is that users do not need to worry about the +underlying backend implementation. In terms of Zeek's script-layer API, SQLite, Redis, or +other backends should behave identically. + +First, we make a call to :zeek:see:`Storage::Sync::put`, passing a key and a value to be +stored. These must be of the same types that were passed in the arguments to +``open_backend``, as described in the :ref:`earlier section `. +The arguments passed into ``put`` are contained in a record of type +:zeek:see:`Storage::PutArgs`. See the documentation for that type for descriptions of the +fields available. In this case, we specify a key and a value plus an expiration time. This +expiration time indicates when the data should be automatically removed from the +backend. We check the result value, and print the error string and return if the operation +failed. + +Next, we attempt to retrieve the same key from the backend. Assuming that the key hasn't +been erased, either manually or via expiration, the value is returned in the ``value`` +field of the result record. If the key has been removed already, the backend should return +a :zeek:see:`Storage::KEY_NOT_FOUND` code. + +Finally, we manually attempt to erase the key. This will remove the key/value pair from +the store, assuming that it hasn't already been removed manually or via expiration. Same +as with ``get``, :zeek:see:`Storage::KEY_NOT_FOUND` should be returned if the key doesn't +exist. + +.. code-block:: zeek + + local res = Storage::Sync::put(backend, [$key="abc", $value="def", $expire_time=45sec]); + if ( res$code != Storage::SUCCESS ) + { + print(res$error_str); + return; + } + + res = Storage::Sync::get(backend, "abc"); + if ( res$code != Storage::SUCCESS ) + { + print(res$error_str); + return; + } + + res = Storage:Sync::erase(backend, "abc"); + if ( res$code != Storage::SUCCESS ) + { + print(res$error_str); + return; + } + +Events +====== + +Two events exist for the storage framework: :zeek:see:`Storage::backend_lost` and +:zeek:see:`Storage::backend_opened`. Both events were mentioned in the :ref:`example of +opening and closing a backend `, but an additional point needs to +be made about the :zeek:see:`Storage::backend_lost` event. This event is also raised when +a connection is lost unexpectedly. This gives users information about connection failures, +as well an opportunity to handle those failures by reconnecting. + +Notes for Built-in Backends +=========================== + +Redis +----- + +- The Redis backend requires the ``hiredis`` library to installed on the system in order + to build. At least version 1.1.0 (Released Nov 2022) is required. + +- Redis server version 6.2.0 or later (or a third-party server implementing the equivalent + level of the Redis API) is required. This is due to some API features the backend uses + not being implemented until that version. + +SQLite +------ + +- The default batch of pragmas in :zeek:see:`Storage::Backend::SQLite::Options` set + ``journal_mode`` to ``WAL``. ``WAL`` mode does not work over network filesystems. If + this mode is used, the database file must be stored on the same computer as all of the + Zeek processes opening it. See the documentation in https://www.sqlite.org/wal.html for + more information. + +- Usage of in-memory databases (i.e. passing ``:memory:`` as the database path) will + result in data not being synced between nodes. Each process will open its own database + within that process's memory space. diff --git a/doc/frameworks/sumstats-countconns.zeek b/doc/frameworks/sumstats-countconns.zeek new file mode 100644 index 0000000000..5d899f9dd2 --- /dev/null +++ b/doc/frameworks/sumstats-countconns.zeek @@ -0,0 +1,36 @@ +@load base/frameworks/sumstats + +event connection_established(c: connection) + { + # Make an observation! + # This observation is global so the key is empty. + # Each established connection counts as one so the observation is always 1. + SumStats::observe("conn established", + SumStats::Key(), + SumStats::Observation($num=1)); + } + +event zeek_init() + { + # Create the reducer. + # The reducer attaches to the "conn established" observation stream + # and uses the summing calculation on the observations. + local r1 = SumStats::Reducer($stream="conn established", + $apply=set(SumStats::SUM)); + + # Create the final sumstat. + # We give it an arbitrary name and make it collect data every minute. + # The reducer is then attached and a $epoch_result callback is given + # to finally do something with the data collected. + SumStats::create([$name = "counting connections", + $epoch = 1min, + $reducers = set(r1), + $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) = + { + # This is the body of the callback that is called when a single + # result has been collected. We are just printing the total number + # of connections that were seen. The $sum field is provided as a + # double type value so we need to use %f as the format specifier. + print fmt("Number of connections established: %.0f", result["conn established"]$sum); + }]); + } diff --git a/doc/frameworks/sumstats-toy-scan.zeek b/doc/frameworks/sumstats-toy-scan.zeek new file mode 100644 index 0000000000..ba0e496603 --- /dev/null +++ b/doc/frameworks/sumstats-toy-scan.zeek @@ -0,0 +1,45 @@ +@load base/frameworks/sumstats + +# We use the connection_attempt event to limit our observations to those +# which were attempted and not successful. +event connection_attempt(c: connection) + { + # Make an observation! + # This observation is about the host attempting the connection. + # Each established connection counts as one so the observation is always 1. + SumStats::observe("conn attempted", + SumStats::Key($host=c$id$orig_h), + SumStats::Observation($num=1)); + } + +event zeek_init() + { + # Create the reducer. + # The reducer attaches to the "conn attempted" observation stream + # and uses the summing calculation on the observations. Keep + # in mind that there will be one result per key (connection originator). + local r1 = SumStats::Reducer($stream="conn attempted", + $apply=set(SumStats::SUM)); + + # Create the final sumstat. + # This is slightly different from the last example since we're providing + # a callback to calculate a value to check against the threshold with + # $threshold_val. The actual threshold itself is provided with $threshold. + # Another callback is provided for when a key crosses the threshold. + SumStats::create([$name = "finding scanners", + $epoch = 5min, + $reducers = set(r1), + # Provide a threshold. + $threshold = 5.0, + # Provide a callback to calculate a value from the result + # to check against the threshold field. + $threshold_val(key: SumStats::Key, result: SumStats::Result) = + { + return result["conn attempted"]$sum; + }, + # Provide a callback for when a key crosses the threshold. + $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = + { + print fmt("%s attempted %.0f or more connections", key$host, result["conn attempted"]$sum); + }]); + } diff --git a/doc/frameworks/sumstats.rst b/doc/frameworks/sumstats.rst new file mode 100644 index 0000000000..30b637de8b --- /dev/null +++ b/doc/frameworks/sumstats.rst @@ -0,0 +1,112 @@ + +.. _sumstats-framework: + +================== +Summary Statistics +================== + +.. TODO: integrate BoZ revisions + +.. rst-class:: opening + + Measuring aspects of network traffic is an extremely common task in Zeek. + Zeek provides data structures which make this very easy as well in + simplistic cases such as size limited trace file processing. In + real-world deployments though, there are difficulties that arise from + clusterization (many processes sniffing traffic) and unbounded data sets + (traffic never stops). The Summary Statistics (otherwise referred to as + SumStats) framework aims to define a mechanism for consuming unbounded + data sets and making them measurable in practice on large clustered and + non-clustered Zeek deployments. + +Overview +======== + +The Sumstat processing flow is broken into three pieces. Observations, where +some aspect of an event is observed and fed into the Sumstats framework. +Reducers, where observations are collected and measured, typically by taking +some sort of summary statistic measurement like average or variance (among +others). Sumstats, where reducers have an epoch (time interval) that their +measurements are performed over along with callbacks for monitoring thresholds +or viewing the collected and measured data. + +Terminology +=========== + + Observation + + A single point of data. Observations have a few components of their + own. They are part of an arbitrarily named observation stream, they + have a key that is something the observation is about, and the actual + observation itself. + + Reducer + + Calculations are applied to an observation stream here to reduce the + full unbounded set of observations down to a smaller representation. + Results are collected within each reducer per-key so care must be + taken to keep the total number of keys tracked down to a reasonable + level. + + Sumstat + + The final definition of a Sumstat where one or more reducers is + collected over an interval, also known as an epoch. Thresholding can + be applied here along with a callback in the event that a threshold is + crossed. Additionally, a callback can be provided to access each + result (per-key) at the end of each epoch. + +Examples +======== + +These examples may seem very simple to an experienced Zeek script developer and +they're intended to look that way. Keep in mind that these scripts will work +on small single process Zeek instances as well as large many-worker clusters. +The complications from dealing with flow based load balancing can be ignored +by developers writing scripts that use Sumstats due to its built-in cluster +transparency. + +Printing the number of connections +---------------------------------- + +Sumstats provides a simple way of approaching the problem of trying to count +the number of connections over a given time interval. Here is a script with +inline documentation that does this with the Sumstats framework: + +.. literalinclude:: sumstats-countconns.zeek + :caption: + :language: zeek + :linenos: + :tab-width: 4 + +When run on a sample PCAP file from the Zeek test suite, the following output +is created: + +.. code-block:: console + + $ zeek -r workshop_2011_browse.trace sumstats-countconns.zeek + Number of connections established: 6 + +Toy scan detection +------------------ + +Taking the previous example even further, we can implement a simple detection +to demonstrate the thresholding functionality. This example is a toy to +demonstrate how thresholding works in Sumstats and is not meant to be a +real-world functional example. + +.. literalinclude:: sumstats-toy-scan.zeek + :caption: + :language: zeek + :linenos: + :tab-width: 4 + +Let's see if there are any hosts that crossed the threshold in a PCAP file +containing a host running nmap: + +.. code-block:: console + + $ zeek -r nmap-vsn.trace sumstats-toy-scan.zeek + 192.168.1.71 attempted 5 or more connections + +It seems the host running nmap was detected! diff --git a/doc/frameworks/supervisor.rst b/doc/frameworks/supervisor.rst new file mode 100644 index 0000000000..9fa055519d --- /dev/null +++ b/doc/frameworks/supervisor.rst @@ -0,0 +1,162 @@ + +.. _framework-supervisor: + +==================== +Supervisor Framework +==================== + +.. rst-class:: opening + + The Supervisor framework enables an entirely new mode for Zeek, one that + supervises a set of Zeek processes that are meant to be persistent. A + Supervisor automatically revives any process that dies or exits prematurely + and also arranges for an ordered shutdown of the entire process tree upon + its own termination. This Supervisor mode for Zeek provides the basic + foundation for process configuration/management that could be used to + deploy a Zeek cluster similar to what ZeekControl does, but is also simpler + to integrate as a standard system service. + +Simple Example +============== + +A simple example of using the Supervisor to monitor one Zeek process +sniffing packets from an interface looks like the following: + +.. code-block:: console + + $ zeek -j simple-supervisor.zeek + +.. literalinclude:: supervisor/simple-supervisor.zeek + :caption: simple-supervisor.zeek + :language: zeek + :linenos: + :tab-width: 4 + +The command-line argument of ``-j`` toggles Zeek to run in "Supervisor mode" to +allow for creation and management of child processes. If you're going to test +this locally, be sure to change ``en0`` to a real interface name you can sniff. + +Notice that the ``simple-supervisor.zeek`` script is loaded and executed by +both the main Supervisor process and also the child Zeek process that it spawns +via :zeek:see:`Supervisor::create` with :zeek:see:`Supervisor::is_supervisor` +or :zeek:see:`Supervisor::is_supervised` being able to distinguish the +Supervisor process from the supervised child process, respectively. +You can also distinguish between multiple supervised child processes by +inspecting the contents of :zeek:see:`Supervisor::node` (e.g. comparing node +names). + +If you happened to be running this locally on an interface with checksum +offloading and want Zeek to ignore checksums, instead simply run with the +``-C`` command-line argument like: + +.. code-block:: console + + $ zeek -j -C simple-supervisor.zeek + +Most command-line arguments to Zeek are automatically inherited by any +supervised child processes that get created. The notable ones that are *not* +inherited are the options to read pcap files and live interfaces, ``-r`` and +``-i``, respectively. + +For node-specific configuration options, see :zeek:see:`Supervisor::NodeConfig` +which gets passed as argument to :zeek:see:`Supervisor::create`. + +Supervised Cluster Example +========================== + +To run a full Zeek cluster similar to what you may already know, try the +following script: + +.. code-block:: console + + $ zeek -j cluster-supervisor.zeek + +.. literalinclude:: supervisor/cluster-supervisor.zeek + :caption: cluster-supervisor.zeek + :language: zeek + :linenos: + :tab-width: 4 + +This script now spawns four nodes: a cluster manager, logger, worker, and +proxy. It also configures each node to use a separate working directory +corresponding to the node's name within the current working directory of the +Supervisor process. Any stdout/stderr output of the nodes is automatically +redirected through the Supervisor process and prefixes with relevant +information, like the node name that the output came from. + +The Supervisor process also listens on a port of its own for further +instructions from other external/remote processes via +:zeek:see:`Broker::listen`. For example, you could use this other script to +tell the Supervisor to restart all processes, perhaps to re-load Zeek scripts +you've changed in the meantime: + +.. code-block:: console + + $ zeek supervisor-control.zeek + +.. literalinclude:: supervisor/supervisor-control.zeek + :caption: supervisor-control.zeek + :language: zeek + :linenos: + :tab-width: 4 + +Any Supervisor instruction you can perform via an API call in a local script +can also be triggered via an associated external event. + +For further details, consult the ``Supervisor`` API at +:doc:`/scripts/base/frameworks/supervisor/api.zeek` and +``SupervisorControl`` API (for remote management) at +:doc:`/scripts/base/frameworks/supervisor/control.zeek`. + +Internal Architecture +===================== + +The following details aren't necessarily important for most users, but instead +aim to give developers a high-level overview of how the process supervision +framework is implemented. The process tree in "supervisor" mode looks like: + +.. figure:: supervisor/zeek-supervisor-architecture.png + +The top-level "Supervisor" process does not directly manage any of the +supervised nodes that are created. Instead, it spawns in intermediate process, +called "Stem", to manage the lifetime of supervised nodes. This is done for +two reasons: + +1. Avoids the need to ``exec()`` the supervised processes which requires + executing whatever version of the ``zeek`` binary happens to exist on + the filesystem at the time of call and it may have changed in the meantime. + This can help avoid potential incompatibility or race-condition pitfalls + associated with system maintenance/upgrades. The one situation that does + still require an ``exec()`` is if the Stem process dies prematurely, but + that is expected to be a rare scenario. +2. Zeek run-time operation generally taints global state, so creating an early + ``fork()`` for use as the Stem process provides a pure baseline image to use + for supervised processes. + +Ultimately, there are two tiers of process supervision happening: the +Supervisor will revive the Stem process if needed and the Stem process will +revive any of its children when needed. + +Also, either the Stem or any of its supervised children processes will +automatically detect if they are orphaned from their parent process and +self-terminate. The Stem checks for orphaning simply by waking up every second +from its ``poll()`` loop to look if its parent PID changed. A supervised node +checks for orphaning similarly, but instead does so from a recurring ``Timer``. +Other than the orphaning-check and how it establishes the desired +configuration from a combination of inheriting command-line arguments and +inspecting Supervisor-specific options, a supervised node does not operate +differently at run-time from a traditional Zeek process. + +Node Revival +============ + +The Supervisor framework assumes that supervised nodes run until something asks +the Supervisor to stop them. When a supervised node exits unexpectedly, the Stem +attempts to revive it during its periodic polling routine. This revival +procedure implements exponential delay, as follows: starting from a delay of one +second, the Stem revives the node up to 3 times. At that point, it doubles the +revival delay, and again tries up to 3 times. This continues indefinitely: the +Stem never gives up on a node, while the revival delay keeps growing. Once a +supervised node has remained up for at least 30 seconds, the revival state +clears and will start from scratch as just described, should the node exit +again. The Supervisor codebase currently hard-wires these thresholds and delays. diff --git a/doc/frameworks/supervisor/cluster-supervisor.zeek b/doc/frameworks/supervisor/cluster-supervisor.zeek new file mode 100644 index 0000000000..0a920586ed --- /dev/null +++ b/doc/frameworks/supervisor/cluster-supervisor.zeek @@ -0,0 +1,28 @@ +event zeek_init() + { + if ( ! Supervisor::is_supervisor() ) + return; + + Broker::listen("127.0.0.1", 9999/tcp); + + local cluster: table[string] of Supervisor::ClusterEndpoint; + cluster["manager"] = [$role=Supervisor::MANAGER, $host=127.0.0.1, $p=10000/tcp]; + cluster["logger"] = [$role=Supervisor::LOGGER, $host=127.0.0.1, $p=10001/tcp]; + cluster["proxy"] = [$role=Supervisor::PROXY, $host=127.0.0.1, $p=10002/tcp]; + cluster["worker"] = [$role=Supervisor::WORKER, $host=127.0.0.1, $p=10003/tcp, $interface="en0"]; + + for ( n, ep in cluster ) + { + local sn = Supervisor::NodeConfig($name=n); + sn$cluster = cluster; + sn$directory = n; + + if ( ep?$interface ) + sn$interface = ep$interface; + + local res = Supervisor::create(sn); + + if ( res != "" ) + print fmt("supervisor failed to create node '%s': %s", n, res); + } + } diff --git a/doc/frameworks/supervisor/simple-supervisor.zeek b/doc/frameworks/supervisor/simple-supervisor.zeek new file mode 100644 index 0000000000..f90c98fbad --- /dev/null +++ b/doc/frameworks/supervisor/simple-supervisor.zeek @@ -0,0 +1,23 @@ +event zeek_init() + { + if ( Supervisor::is_supervisor() ) + { + local sn = Supervisor::NodeConfig($name="foo", $interface="en0"); + local res = Supervisor::create(sn); + + if ( res == "" ) + print "supervisor created a new node"; + else + print "supervisor failed to create node", res; + } + else + print fmt("supervised node '%s' zeek_init()", Supervisor::node()$name); + } + +event zeek_done() + { + if ( Supervisor::is_supervised() ) + print fmt("supervised node '%s' zeek_done()", Supervisor::node()$name); + else + print "supervisor zeek_done()"; + } diff --git a/doc/frameworks/supervisor/supervisor-control.zeek b/doc/frameworks/supervisor/supervisor-control.zeek new file mode 100644 index 0000000000..596c974bf7 --- /dev/null +++ b/doc/frameworks/supervisor/supervisor-control.zeek @@ -0,0 +1,15 @@ +event zeek_init() + { + Broker::peer("127.0.0.1", 9999/tcp, 1sec); + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + Broker::publish(SupervisorControl::topic_prefix, SupervisorControl::restart_request, "", ""); + } + +event SupervisorControl::restart_response(reqid: string, result: bool) + { + print fmt("got result of supervisor restart request: %s", result); + terminate(); + } diff --git a/doc/frameworks/supervisor/zeek-supervisor-architecture.png b/doc/frameworks/supervisor/zeek-supervisor-architecture.png new file mode 100644 index 0000000000..250dba974e Binary files /dev/null and b/doc/frameworks/supervisor/zeek-supervisor-architecture.png differ diff --git a/doc/frameworks/telemetry.rst b/doc/frameworks/telemetry.rst new file mode 100644 index 0000000000..6af4660e98 --- /dev/null +++ b/doc/frameworks/telemetry.rst @@ -0,0 +1,441 @@ +.. _histogram_quantile(): https://prometheus.io/docs/prometheus/latest/querying/functions/#histogram_quantile +.. _Prometheus: https://prometheus.io +.. _Prometheus Getting Started Guide: https://prometheus.io/docs/prometheus/latest/getting_started/ +.. _Prometheus Metric Types: https://prometheus.io/docs/concepts/metric_types/ +.. _Prometheus HTTP Service Discovery: https://prometheus.io/docs/prometheus/latest/http_sd/ +.. _prometheus-cpp: https://github.com/jupp0r/prometheus-cpp + +.. _framework-telemetry: + +=================== +Telemetry Framework +=================== + +.. note:: + + This framework changed considerably with Zeek 7, and is not API-compatible + with earlier versions. While earlier versions relied on an implementation + in :ref:`Broker `, Zeek now maintains its + own implementation, building on `prometheus-cpp`_, with Broker adding its + telemetry to Zeek's internal registry of metrics. + +The telemetry framework continuously collects metrics during Zeek's operation, +and provides ways to export this telemetry to third-party consumers. Zeek ships +with a pre-defined set of metrics and allows you to add your own, via +script-layer and in-core APIs you use to instrument relevant parts of the +code. Metrics target Zeek's operational behavior, or track characteristics of +monitored traffic. Metrics are not an additional export vehicle for Zeek's +various regular logs. Zeek's telemetry data model closely resembles that of +`Prometheus`_, and supports its text-based exposition format for scraping by +third-party collectors. + +This section outlines usage examples, and gives brief API examples for +composing your own metrics. Head to the :zeek:see:`Telemetry` API documentation +for more details. + +Metric Types +============ + +Zeek supports the following metric types: + + Counter + A continuously increasing value, resetting on process restart. + Examples of counters are the number of log writes since process start, + packets processed, or ``process_seconds`` representing CPU usage. + + Gauge + A gauge metric is a numerical value that can increase and decrease + over time. Examples are table sizes or the :zeek:see:`val_footprint` + of Zeek script values over the lifetime of the process. More general + examples include a temperature or memory usage. + + Histogram + Pre-configured buckets of observations with corresponding counts. + Examples of histograms are connection durations, delays, or transfer + sizes. Generally, it is useful to know the expected range and distribution + of such values, as the bounds of a histogram's buckets are defined when + this metric gets created. + +Zeek uses :zeek:type:`double` throughout to track metric values. Since +terminology around telemetry can be complex, it helps to know a few additional +terms: + + Labels + A given metric sometimes doesn't exist in isolation, but comes with + additional labeling to disambiguate related observations. For example, Zeek + ships with gauge called ``zeek_active_sessions`` that labels counts for TCP, + UDP, and other transport protocols separately. Labels have a name (for + example, "protocol") to refer to value (such as "tcp"). A metric can have + multiple labels. Labels are thus a way to associate textual information with + the numerical values of metrics. + + Family + The set of such metrics, differing only by their labeling, is a known as a + Family. Zeek's script-layer metrics API lets you operate on individual + metrics and families. + +Zeek has no equivalent to Prometheus's Summary type. A good reference to +consult for more details is the official `Prometheus Metric Types`_ +documentation. + +Cluster Considerations +====================== + +When running Zeek as a cluster, every node maintains its own metrics registry, +independently of the other nodes. Zeek does not automatically synchronize, +centralize, or aggregate metrics across the cluster. Instead, it adds the name +of the node a particular metric originated from at collection time, leaving any +aggregation to post-processing where desired. + +.. note:: + + This is a departure from the design in earlier versions of Zeek, which could + (either by default, or after activation) centralize metrics in the cluster's + manager node. + +Accordingly, the :zeek:see:`Telemetry::collect_metrics` and +:zeek:see:`Telemetry::collect_histogram_metrics` functions only return +node-local metrics. + +Metrics Export +============== + +Zeek supports two mechanisms for exporting telemetry: traditional logs, and +Prometheus-compatible endpoints for scraping by a third-party service. We cover +them in turn. + +Zeek Logs +--------- + +Zeek can export current metrics continuously via :file:`telemetry.log` and +:file:`telemetry_histogram.log`. It does not do so by default. To enable, load the +policy script ``frameworks/telemetry/log`` on the command line, or via +``local.zeek``. + +The :zeek:see:`Telemetry::Info` and :zeek:see:`Telemetry::HistogramInfo` records +define the logs. Both records include a ``peer`` field that conveys the +cluster node the metric originated from. + +By default, Zeek reports current telemetry every 60 seconds, as defined by the +:zeek:see:`Telemetry::log_interval`, which you're free to adjust. + +Also, by default only metrics with the ``prefix`` (namespace) ``zeek`` and +``process`` are included in above logs. If you add new metrics with your own +prefix and expect these to be included, redefine the +:zeek:see:`Telemetry::log_prefixes` option:: + + @load frameworks/telemetry/log + + redef Telemetry::log_prefixes += { "my_prefix" }; + +Clearing the set will cause all metrics to be logged. As with any logs, you may +employ :ref:`policy hooks `, +:zeek:see:`Telemetry::log_policy` and +:zeek:see:`Telemetry::log_policy_histogram`, to define potentially more granular +filtering. + +Native Prometheus Export +------------------------ + +Every Zeek process, regardless of whether it's running long-term standalone or +as part of a cluster, can run an HTTP server that renders current telemetry in +Prometheus's `text-based exposition format +`_. + +The :zeek:see:`Telemetry::metrics_port` variable controls this behavior. Its +default of ``0/unknown`` disables exposing the port; setting it to another TCP +port will enable it. In clusterized operation, the cluster topology can specify +each node's metrics port via the corresponding :zeek:see:`Cluster::Node` field, +and the framework will adjust ``Telemetry::metrics_port`` accordingly. Both +zeekctl and the management framework let you define specific ports and can also +auto-populate their values, similarly to Broker's listening ports. + +To query a node's telemetry, point an HTTP client or Prometheus scraper at the +node's metrics port:: + + $ curl -s http://:/metrics + # HELP exposer_transferred_bytes_total Transferred bytes to metrics services + # TYPE exposer_transferred_bytes_total counter + exposer_transferred_bytes_total 0 + ... + # HELP zeek_event_handler_invocations_total Number of times the given event handler was called + # TYPE zeek_event_handler_invocations_total counter + zeek_event_handler_invocations_total{endpoint="manager",name="run_sync_hook"} 2 + ... + +To simplify telemetry collection from all nodes in a cluster, Zeek supports +`Prometheus HTTP Service Discovery`_ on the manager node. Using this approach, the +endpoint ``http://:/services.json`` returns a +JSON data structure that itemizes all metrics endpoints in the +cluster. Prometheus scrapers supporting service discovery then proceed to +collect telemetry from the listed endpoints in turn. + +The following is an example service discovery scrape config entry within +Prometheus server's ``prometheus.yml`` configuration file:: + + ... + scrape_configs: + - job_name: zeek-discovery + scrape_interval: 5s + http_sd_configs: + - url: http://localhost:9991/services.json + refresh_interval: 10s + +See the `Prometheus Getting Started Guide`_ for additional information. + +.. note:: + + .. versionchanged:: 7.0 + + The built-in aggregation for Zeek telemetry to the manager node has been + removed, in favor of the Prometheus-compatible service discovery + endpoint. The new approach requires cluster administrators to manage access + to the additional ports. However, it allows Prometheus to conduct the + aggregation, instead of burdening the Zeek manager with it, which has + historically proved expensive. + +If these setups aren't right for your environment, there's the possibility to +redefine the options in ``local.zeek`` to something more suitable. For example, +the following snippet selects the metrics port of each Zeek process relative +to the cluster port used in ``cluster-layout.zeek``:: + + @load base/frameworks/cluster + + global my_node = Cluster::nodes[Cluster::node]; + global my_metrics_port = count_to_port(port_to_count(my_node$p) - 1000, tcp); + + redef Telemetry::metrics_port = my_metrics_port; + + +Examples of Metrics Application +=============================== + +Counting Log Writes per Stream +------------------------------ + +In combination with the :zeek:see:`Log::log_stream_policy` hook, it is +straightforward to record :zeek:see:`Log::write` invocations over the dimension +of the :zeek:see:`Log::ID` value. This section shows three different approaches +to do this. Which approach is most applicable depends mostly on the expected +script layer performance overhead for updating the metric. For example, calling +:zeek:see:`Telemetry::counter_with` and :zeek:see:`Telemetry::counter_inc` +within a handler of a high-frequency event may be prohibitive, while for a +low-frequency event it's unlikely to matter. + +Assuming a :zeek:see:`Telemetry::metrics_port` of 9090, querying the Prometheus +endpoint using ``curl`` provides output resembling the following for each of +the three approaches. + +.. code-block:: + + $ curl -s localhost:9090/metrics | grep log_writes + # HELP zeek_log_writes_total Number of log writes per stream + # TYPE zeek_log_writes_total counter + zeek_log_writes_total{endpoint="zeek",log_id="packetfilter_log"} 1 + zeek_log_writes_total{endpoint="zeek",log_id="loadedscripts_log"} 477 + zeek_log_writes_total{endpoint="zeek",log_id="stats_log"} 1 + zeek_log_writes_total{endpoint="zeek",log_id="dns_log"} 200 + zeek_log_writes_total{endpoint="zeek",log_id="ssl_log"} 9 + zeek_log_writes_total{endpoint="zeek",log_id="conn_log"} 215 + zeek_log_writes_total{endpoint="zeek",log_id="captureloss_log"} 1 + +The above shows a family of 7 ``zeek_log_writes_total`` metrics, each with an +``endpoint`` label (here, ``zeek``, which would be a cluster node name if +scraped from a Zeek cluster) and a ``log_id`` one. + +Immediate +^^^^^^^^^ + +The following example creates a global counter family object and uses +the :zeek:see:`Telemetry::counter_family_inc` helper to increment the +counter metric associated with a string representation of the :zeek:see:`Log::ID` +value. + + +.. literalinclude:: telemetry/log-writes-immediate.zeek + :caption: log-writes-immediate.zeek + :language: zeek + :linenos: + :tab-width: 4 + +With a few lines of scripting code, Zeek now track log writes per stream +ready to be scraped by a Prometheus server. + + +Cached +^^^^^^ + +For cases where creating the label value (stringification, :zeek:see:`gsub` and :zeek:see:`to_lower`) +and instantiating the label vector as well as invoking the +:zeek:see:`Telemetry::counter_family_inc` methods cause too much +performance overhead, the counter instances can also be cached in a lookup table. +The counters can then be incremented with :zeek:see:`Telemetry::counter_inc` +directly. + +.. literalinclude:: telemetry/log-writes-cached.zeek + :caption: log-writes-cached.zeek + :language: zeek + :linenos: + :tab-width: 4 + + +For metrics without labels, the metric instances can also be cached as global +variables directly. The following example counts the number of http requests. + +.. literalinclude:: telemetry/global-http-counter.zeek + :caption: global-http-counter.zeek + :language: zeek + :linenos: + :tab-width: 4 + + +Sync +^^^^ + +In case the scripting overhead of the previous approach is still too high, +individual writes (or events) can be tracked in a table or global variable +and then synchronized / mirrored to concrete counter and gauge instances +during execution of the :zeek:see:`Telemetry::sync` hook. + +.. literalinclude:: telemetry/log-writes-sync.zeek + :caption: log-writes-sync.zeek + :language: zeek + :linenos: + :tab-width: 4 + +For tracking log writes, this is unlikely to be required (and Zeek exposes +various logging natively through the framework already), but for updating +metrics within high frequency events that otherwise have low script processing +overhead, it's a valuable approach. + + +.. versionchanged:: 7.1 + +The :zeek:see:`Telemetry::sync` hook is invoked on-demand only. Either when +one of the :zeek:see:`Telemetry::collect_metrics` +or :zeek:see:`Telemetry::collect_histogram_metrics` functions is invoked, or +when querying Prometheus endpoint. It's an error to call either of the +collection BiFs within the :zeek:see:`Telemetry::sync` hook and results +in a reporter warning. + + +.. note:: + + In versions before Zeek 7.1, :zeek:see:`Telemetry::sync` was invoked on a + fixed schedule, potentially resulting in stale metrics at collection time, + as well as generating small runtime overhead when metrics are not collected. + +Table Sizes +----------- + +It can be useful to expose the size of tables as metrics, as they often +indicate the approximate amount of state maintained in memory. +As table sizes may increase and decrease, a :zeek:see:`Telemetry::Gauge` +is appropriate for this purpose. + +The following example records the size of the :zeek:see:`Tunnel::active` table +and its footprint with two gauges. The gauges are updated during the +:zeek:see:`Telemetry::sync` hook. Note, there are no labels in use, both +gauge instances are simple globals. + +.. literalinclude:: telemetry/table-size-tracking.zeek + :caption: log-writes-sync.zeek + :language: zeek + :linenos: + :tab-width: 4 + +Example representation of these metrics when querying the Prometheus endpoint: + +.. code-block:: + + $ curl -s localhost:9090/metrics | grep tunnel + # HELP zeek_monitored_tunnels_active_footprint Footprint of the Tunnel::active table + # TYPE zeek_monitored_tunnels_active_footprint gauge + zeek_monitored_tunnels_active_footprint{endpoint="zeek"} 324 + # HELP zeek_monitored_tunnels_active Number of currently active tunnels as tracked in Tunnel::active + # TYPE zeek_monitored_tunnels_active gauge + zeek_monitored_tunnels_active{endpoint="zeek"} 12 + + +Instead of tracking footprints per variable, :zeek:see:`global_container_footprints`, +could be leveraged to track all global containers at once, using the variable +name as label. + +Connection Durations as Histogram +--------------------------------- + +To track the distribution of certain measurements, a :zeek:see:`Telemetry::Histogram` +can be used. The histogram's buckets have to be preconfigured. + +The following example observes the duration of each connection that Zeek has +monitored. + +.. literalinclude:: telemetry/connection-durations.zeek + :caption: connection-durations.zeek + :language: zeek + :linenos: + :tab-width: 4 + +Due to the way Prometheus represents histograms and the fact that durations +are broken down by protocol and service in the given example, the resulting +representation becomes rather verbose. + +.. code-block:: + + $ curl -s localhost:9090/metrics | grep monitored_connection_duration + # HELP zeek_monitored_connection_duration_seconds Duration of monitored connections + # TYPE zeek_monitored_connection_duration_seconds histogram + zeek_monitored_connection_duration_seconds_bucket{endpoint="zeek",proto="udp",service="dns",le="0.1"} 970 + zeek_monitored_connection_duration_seconds_bucket{endpoint="zeek",proto="udp",service="dns",le="1"} 998 + zeek_monitored_connection_duration_seconds_bucket{endpoint="zeek",proto="udp",service="dns",le="10"} 1067 + zeek_monitored_connection_duration_seconds_bucket{endpoint="zeek",proto="udp",service="dns",le="30"} 1108 + zeek_monitored_connection_duration_seconds_bucket{endpoint="zeek",proto="udp",service="dns",le="60"} 1109 + zeek_monitored_connection_duration_seconds_bucket{endpoint="zeek",proto="udp",service="dns",le="+Inf"} 1109 + zeek_monitored_connection_duration_seconds_sum{endpoint="zeek",proto="udp",service="dns"} 1263.085691 + zeek_monitored_connection_duration_seconds_count{endpoint="zeek",proto="udp",service="dns"} 1109 + zeek_monitored_connection_duration_seconds_bucket{endpoint="zeek",proto="tcp",service="http",le="0.1"} 16 + zeek_monitored_connection_duration_seconds_bucket{endpoint="zeek",proto="tcp",service="http",le="1"} 54 + zeek_monitored_connection_duration_seconds_bucket{endpoint="zeek",proto="tcp",service="http",le="10"} 56 + zeek_monitored_connection_duration_seconds_bucket{endpoint="zeek",proto="tcp",service="http",le="30"} 57 + zeek_monitored_connection_duration_seconds_bucket{endpoint="zeek",proto="tcp",service="http",le="60"} 57 + zeek_monitored_connection_duration_seconds_bucket{endpoint="zeek",proto="tcp",service="http",le="+Inf"} 57 + + +To work with histogram data, Prometheus provides specialized query functions. +For example `histogram_quantile()`_. + +Note, when using data from :file:`conn.log` and post-processing, a proper +histogram of connection durations can be calculated and possibly preferred. +The above example is meant for demonstration purposes. Histograms may be +primarily be useful for Zeek operational metrics such as processing times +or queueing delays, response times to external systems, etc. + + +Exporting the Zeek Version +-------------------------- + +A common pattern in the Prometheus ecosystem is to expose the version +information of the running process as gauge metric with a value of 1. + +The following example does just that with a Zeek script: + +.. literalinclude:: telemetry/version.zeek + :caption: version.zeek + :language: zeek + :linenos: + :tab-width: 4 + +In Prometheus's exposition format, this turns into the following: + +.. code-block:: + + $ curl -s localhost:9090/metrics | grep version + # HELP zeek_version_info The Zeek version + # TYPE zeek_version_info gauge + zeek_version_info{beta="true",commit="0",debug="true",major="7",minor="0",patch="0",version_number="70000",version_string="7.0.0-rc4-debug"} 1 + zeek_version_info{beta="false",commit="289",debug="true",endpoint="zeek",major="5",minor="1",patch="0",version_number="50100",version_string="5.1.0-dev.289-debug"} 1.000000 + + +Zeek already ships with this gauge, via +:doc:`/scripts/base/frameworks/telemetry/main.zeek`. There is no need to add +above snippet to your site. diff --git a/doc/frameworks/telemetry/connection-durations.zeek b/doc/frameworks/telemetry/connection-durations.zeek new file mode 100644 index 0000000000..d05532e203 --- /dev/null +++ b/doc/frameworks/telemetry/connection-durations.zeek @@ -0,0 +1,23 @@ +global conn_durations_hf = Telemetry::register_histogram_family([ + $prefix="zeek", + $name="monitored_connection_duration", + $unit="seconds", + $help_text="Duration of monitored connections", + $bounds=vector(0.1, 1.0, 10.0, 30.0, 60.0), + $label_names=vector("proto", "service") +]); + +event connection_state_remove(c: connection) + { + local proto = cat(c$conn$proto); + local service: set[string] = {"unknown"}; + + if ( |c$service| != 0 ) + service = c$service; + + for (s in service ) + { + local h = Telemetry::histogram_with(conn_durations_hf, vector(proto, to_lower(s))); + Telemetry::histogram_observe(h, interval_to_double(c$duration)); + } + } diff --git a/doc/frameworks/telemetry/global-http-counter.zeek b/doc/frameworks/telemetry/global-http-counter.zeek new file mode 100644 index 0000000000..2e030200f5 --- /dev/null +++ b/doc/frameworks/telemetry/global-http-counter.zeek @@ -0,0 +1,14 @@ +global http_counter_cf = Telemetry::register_counter_family([ + $prefix="zeek", + $name="monitored_http_requests", + $unit="1", + $help_text="Number of http requests observed" +]); + +global http_counter = Telemetry::counter_with(http_counter_cf); + +event http_request(c: connection, method: string, original_URI: string, + unescaped_URI: string, version: string) + { + Telemetry::counter_inc(http_counter); + } diff --git a/doc/frameworks/telemetry/log-writes-cached.zeek b/doc/frameworks/telemetry/log-writes-cached.zeek new file mode 100644 index 0000000000..fce51612a4 --- /dev/null +++ b/doc/frameworks/telemetry/log-writes-cached.zeek @@ -0,0 +1,22 @@ +global log_writes_cf = Telemetry::register_counter_family([ + $prefix="zeek", + $name="log_writes", + $unit="1", + $help_text="Number of log writes per stream", + $label_names=vector("log_id") +]); + +# Cache for the Telemetry::Counter instances. +global log_write_counters: table[Log::ID] of Telemetry::Counter; + +hook Log::log_stream_policy(rec: any, id: Log::ID) + { + if ( id !in log_write_counters ) + { + local log_id = to_lower(gsub(cat(id), /:+/, "_")); + log_write_counters[id] = Telemetry::counter_with(log_writes_cf, + vector(log_id)); + } + + Telemetry::counter_inc(log_write_counters[id]); + } diff --git a/doc/frameworks/telemetry/log-writes-immediate.zeek b/doc/frameworks/telemetry/log-writes-immediate.zeek new file mode 100644 index 0000000000..d7066f6b99 --- /dev/null +++ b/doc/frameworks/telemetry/log-writes-immediate.zeek @@ -0,0 +1,13 @@ +global log_writes_cf = Telemetry::register_counter_family([ + $prefix="zeek", + $name="log_writes", + $unit="1", + $help_text="Number of log writes per stream", + $label_names=vector("log_id") +]); + +hook Log::log_stream_policy(rec: any, id: Log::ID) + { + local log_id = to_lower(gsub(cat(id), /:+/, "_")); + Telemetry::counter_family_inc(log_writes_cf, vector(log_id)); + } diff --git a/doc/frameworks/telemetry/log-writes-sync.zeek b/doc/frameworks/telemetry/log-writes-sync.zeek new file mode 100644 index 0000000000..e6d7937f1a --- /dev/null +++ b/doc/frameworks/telemetry/log-writes-sync.zeek @@ -0,0 +1,23 @@ +global log_writes_cf = Telemetry::register_counter_family([ + $prefix="zeek", + $name="log_writes", + $unit="1", + $help_text="Number of log writes per stream", + $label_names=vector("log_id") +]); + +global log_writes: table[Log::ID] of count &default=0; + +hook Log::log_stream_policy(rec: any, id: Log::ID) + { + ++log_writes[id]; + } + +hook Telemetry::sync() + { + for ( id, v in log_writes ) + { + local log_id = to_lower(gsub(cat(id), /:+/, "_")); + Telemetry::counter_family_inc(log_writes_cf, vector(log_id)); + } + } diff --git a/doc/frameworks/telemetry/table-size-tracking.zeek b/doc/frameworks/telemetry/table-size-tracking.zeek new file mode 100644 index 0000000000..106c13519b --- /dev/null +++ b/doc/frameworks/telemetry/table-size-tracking.zeek @@ -0,0 +1,25 @@ +module Tunnel; + +global tunnels_active_size_gf = Telemetry::register_gauge_family([ + $prefix="zeek", + $name="monitored_tunnels_active", + $unit="1", + $help_text="Number of currently active tunnels as tracked in Tunnel::active" +]); + +global tunnels_active_size_gauge = Telemetry::gauge_with(tunnels_active_size_gf); + +global tunnels_active_footprint_gf = Telemetry::register_gauge_family([ + $prefix="zeek", + $name="monitored_tunnels_active_footprint", + $unit="1", + $help_text="Footprint of the Tunnel::active table" +]); + +global tunnels_active_footprint_gauge = Telemetry::gauge_with(tunnels_active_footprint_gf); + +hook Telemetry::sync() { + + Telemetry::gauge_set(tunnels_active_size_gauge, |Tunnel::active|); + Telemetry::gauge_set(tunnels_active_footprint_gauge, val_footprint(Tunnel::active)); +} diff --git a/doc/frameworks/telemetry/version.zeek b/doc/frameworks/telemetry/version.zeek new file mode 100644 index 0000000000..731f1bb5b8 --- /dev/null +++ b/doc/frameworks/telemetry/version.zeek @@ -0,0 +1,19 @@ +global version_gf = Telemetry::register_gauge_family([ + $prefix="zeek", + $name="version_info", + $unit="1", + $help_text="The Zeek version", + $label_names=vector("version_number", "major", "minor", "patch", "commit", "beta", "debug","version_string") +]); + +event zeek_init() + { + local v = Version::info; + local labels = vector(cat(v$version_number), + cat(v$major), cat(v$minor), cat (v$patch), + cat(v$commit), + v$beta ? "true" : "false", + v$debug ? "true" : "false", + v$version_string); + Telemetry::gauge_family_set(version_gf, labels, 1.0); + } diff --git a/doc/frameworks/tls-decryption.rst b/doc/frameworks/tls-decryption.rst new file mode 100644 index 0000000000..29e37ffcfe --- /dev/null +++ b/doc/frameworks/tls-decryption.rst @@ -0,0 +1,165 @@ + +.. _framework-tls-decryption: + +============== +TLS Decryption +============== + +.. rst-class:: opening + + + Zeek has limited support for decrypting TLS connections, if the necessary key material is + available. If decryption is possible, Zeek can forward the decrypted data to other analyzers - like + the HTTP analyzer. + + Note that this support is currently limited to a single version of TLS and a single cipher suite. + Zeek can currently only decrypt TLS 1.2 connections that use the TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + cipher. Any other TLS version or cipher will not be decrypted. We do not currently plan to extend + this support to other versions of TLS, or to other ciphersuites. + +Capturing and decrypting a trace file +===================================== + +The most common use-case for TLS decryption is to capture a trace file together with the necessary key material +to allow Zeek to decrypt it. In principle, it is possible to allow Zeek to decrypt TLS connections in live traffic. +However, this kind of setup is much more complex and will require the user to set up a way to transport the +key material to Zeek in real-time. We will talk a bit about the possibility of this below. + +Capturing a trace file with keys +-------------------------------- + +To be able to decrypt the TLS connections contained in a trace file, we need access to the symmetric keys that +were used to encrypt the connection. Specifically, for our supported version of TLS, we need the pre-master secret +for the TLS connection. + +Firefox and Chrome allow users to capture the key material of all TLS connections that they perform by setting +the ``SSLKEYLOGFILE`` environment variable. For example, on Mac OS, you can instruct Firefox to record the TLS +key material by starting it in the following way: + +.. code-block:: console + + export SSLKEYLOGFILE=$HOME/keylogfile.txt + open -a firefox + +After running Firefox like this, and accessing some web pages, you should end up with the file ``keylogfile.txt`` +in your home directory that contains lines like this:: + + # SSL/TLS secrets log file, generated by NSS + CLIENT_RANDOM 47d1becb619e0851ee363c2cf37187228227ca4e680f9a7c0bd15069aa7a5970 ad03ceda4890fa581e989f5e3862023e2a4e3e8ad81325238d908066e1d35cc875979e34c08e6fdfd9d8c6f356e385c1 + CLIENT_RANDOM 2095006fcb3f93d255cbb6562587f0dd010212fdee9d233aff64e6ed36cd5c45 0d36faaa2eadbda2a8095f951de1cbac46b81b008fbf391d91951b3485476bab73288a1e17cd0ce80e0fc0401dbe9e3f + CLIENT_RANDOM 8f58b32bf97e7d3856e2fccbbe80798ec2e3f515251082ad63bbc7c231d8bee0 9a7cf946a04718a19f4d20c3f80c1cf8c823c3e2b1c337ef64322d751b410543315f6ecf7dbf45ec9be194a3cc7c1a0f + +These log lines contain the pre-master secrets for the connections that your browser established. The secrets +are indexed with the client random of the connections. This allows applications (like Zeek) to identify which +secret to use to decrypt a connection. + +If you capture this key log file together with a trace-file, you will be able to decrypt the sessions using Zeek +(assuming they use a supported TLS version and ciphersuite). + +Decrypting a trace file +----------------------- + +The next step is to convert the keylogfile into a format that can be ingested by the Zeek. This bash-script +will perform the conversion: + +.. code-block:: bash + + #!/usr/bin/env bash + + if [ $# -ne 1 ]; then + echo "Script expects one argument (key log filename)" >/dev/stderr + exit -1 + fi + + FILE=$1 + + if [ ! -f ${FILE} ]; then + echo "${FILE} does not exist or is not readable" >/dev/stderr + exit -1 + fi + + echo "#fields client_random secret" + grep CLIENT_RANDOM ${FILE} | sed 's/^CLIENT_RANDOM ........\(.*\) \(.*\)$/\1 \2/' | sed 's/[A-Za-z0-9][A-Za-z0-9]/\\x&/g' + +Note that the script just converts the keylog file in a standard Zeek tsv-file. Furthermore, it removes +the first 16 characters of the CLIENT_RANDOM; this is needed due to a design-choice of Zeek that makes accessing +the first 8 bytes (equivalent to 16 hex-characters) of the client random inconvenient - thus these bytes are not +used for matching. + +If you run the bash script on the ``keylogfile.txt`` you created earlier, you will get a Zeek tsv-file. + +.. code-block:: console + + ./convert-keylog.sh ~/keylogfile.txt > ~/keylogfile.log + + cat ~/keylogfile.log + #fields client_random secret + \x0e\x78\x2d\x35\x63\x95\x5d\x8a\x30\xa9\xcf\xb6\x4f\x47\xf3\x96\x34\x8a\x1e\x79\x1a\xa2\x32\x55\xe2\x2f\xc5\x7a \x34\x4f\x12\x65\xbf\x43\x40\xb3\x61\x6b\xa0\x16\x5d\x2b\x4d\xb9\xb1\xe8\x4a\x3d\xa2\x42\x0e\x38\xab\x01\x50\x62\x84\xcc\x34\xcd\xe0\x34\x10\xfe\x1a\x02\x30\x49\x74\x6c\x46\x43\xa7\x0c\x67\x0d + \x24\x8c\x7e\x24\xee\xfb\x13\xcd\xee\xde\xb1\xf4\xb6\xd6\xd5\xee\x67\x8d\xd3\xff\xc7\xe7\x39\x23\x18\x3f\x99\xb4 \xe7\xed\x24\x26\x0d\x25\xd9\xfd\xf5\x0f\xc0\xf4\x56\x51\x0e\x4e\xec\x7f\x58\x9c\xaf\x39\x25\x14\x16\xa6\x71\xdd\xea\xfe\xe9\xc0\x93\xbe\x89\x4c\xab\xcc\xff\xb2\xf0\x9a\xea\x98\xf5\xb2\x53\x1e + \x57\xd7\xc7\x7a\x2d\x5e\x35\x29\x2c\xd7\xe7\x94\xee\xf8\x6f\x31\x45\xf6\xbe\x25\x08\xed\x1d\x92\xd2\x0b\x9b\x04 \xc1\x93\x17\x93\xd9\x7d\xd2\x98\xb3\xe0\xdb\x2c\x5d\xbe\x71\x31\xa7\x9a\xf5\x91\xf9\x87\x90\xee\xb7\x79\x9f\x6b\xb4\x1f\x47\xa7\x69\x62\x4b\xa3\x99\x0c\xa9\x43\xf9\xea\x3b\x4d\x5f\x2f\xfe\xfb + +Now we can run Zeek on the trace-file that we recorded. We need a small additional script for this, which +stops processing while the TLS keylog file is loaded. It also loads the required policy script. + +.. literalinclude:: tls_decryption-1-suspend-processing.zeek + :caption: + :language: zeek + :linenos: + :tab-width: 4 + +.. code-block:: console + + $ export ZEEK_TLS_KEYLOG_FILE=~/keylogfile.log + $ zeek -C -r tls/tls-1.2-stream-keylog.pcap tls_decryption-1-suspend-processing.zeek + + $ cat conn.log + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path conn + #open 2022-03-01-16-57-26 + #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents + #types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] + 1646150638.631834 CTy5Us4OUaTOcyrPvc 192.168.20.12 60679 193.99.144.85 443 tcp http,ssl 7.246461 10853 151695 SF - - 0 ShADadFf 98 15961 139 158931 - + #close 2022-03-01-16-57-26 + + $ cat http.log + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path http + #open 2022-03-01-16-57-25 + #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer version user_agent origin request_body_len response_body_len status_code status_msg info_code info_msg tags username password proxied orig_fuids orig_filenames orig_mime_types resp_fuids resp_filenames resp_mime_types + #types time string addr port addr port count string string string string string string string count count count string count string set[enum] string string set[string] vector[string] vector[string] vector[string] vector[string] vector[string] vector[string] + 1646150638.735969 CTy5Us4OUaTOcyrPvc 192.168.20.12 60679 193.99.144.85 443 1 GET www.heise.de /assets/akwa/v24/js/akwa.js?.ltc.c61e84978682308f631c https://www.heise.de/ 1.1 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:97.0) Gecko/20100101 Firefox/97.0 - 0 375340 200 OK - - (empty) - - - - - - FSJiWr34wfIujxxtm3 - text/plain + 1646150638.944774 CTy5Us4OUaTOcyrPvc 192.168.20.12 60679 193.99.144.85 443 2 GET www.heise.de /assets/heise/images/mit_technology_review_singleline.b768.ltc.svg https://www.heise.de/ 1.1 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:97.0) Gecko/20100101 Firefox/97.0 - 0 3430 200 OK - - (empty) - - - - - - FgivhC1pvnYeQS4u18 - text/plain + 1646150638.976118 CTy5Us4OUaTOcyrPvc 192.168.20.12 60679 193.99.144.85 443 3 GET www.heise.de /assets/heise/hobell/css/hobell.css?.ltc.3746e7e49abafa23b5fb https://www.heise.de/ 1.1 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:97.0) Gecko/20100101 Firefox/97.0 - 0 85280 200 OK - - (empty) - - - - - - FyvBkl2nwRXf0hkDO1 - text/plain + ... + +Now :file:`conn.log` shows that the HTTP as well as the SSL analyzers were attached. :file:`http.log` shows the +information from the decrypted HTTP session. + +If you try this yourself note that today a lot of encrypted Internet traffic uses HTTP/2. Zeek currently does not +ship with an HTTP/2 parser by default. If you capture your own traffic make sure that your browser uses +HTTP/1. Alternatively, you can add an HTTP/2 analyzer to Zeek, e.g. using a package. + +Decrypting live traffic +======================= + +In principle, it is possible to decrypt live traffic using this approach. When you want to do this, you have to supply the secrets +to Zeek as the connections are happening. Note that there are timing constraints here - the secrets should arrive at the Zeek instance +that will decrypt the traffic before encrypted application data is exchanged. + +The :doc:`/scripts/policy/protocols/ssl/decryption.zeek` policy script sets up a two events for this purpose. You can send key material +to the Zeek worker in question via Broker, using the ``/zeek/tls/decryption`` topic. The two events used for this are +:zeek:see:`SSL::add_keys` and :zeek:see:`SSL::add_secret`. + +TLS Decryption API +================== + +If the policy script does not suit your use-case, you can use the TLS decryption API directly to decrypt a connection. You can use either the +:zeek:see:`set_secret` or the :zeek:see:`set_keys` functions to provide the decryption keys for an ongoing SSL connection. + +Note that you will have to make sure to set :zeek:see:`SSL::disable_analyzer_after_detection` to false if you use this functionality directly. diff --git a/doc/frameworks/tls_decryption-1-suspend-processing.zeek b/doc/frameworks/tls_decryption-1-suspend-processing.zeek new file mode 100644 index 0000000000..6b6b26ee1a --- /dev/null +++ b/doc/frameworks/tls_decryption-1-suspend-processing.zeek @@ -0,0 +1,13 @@ +@load protocols/ssl/decryption +@load base/protocols/http + +event zeek_init() + { + suspend_processing(); + } + +event Input::end_of_data(name: string, source: string) + { + if ( name == "tls-keylog-file" ) + continue_processing(); + } diff --git a/doc/get-started.rst b/doc/get-started.rst new file mode 100644 index 0000000000..fb37934e3c --- /dev/null +++ b/doc/get-started.rst @@ -0,0 +1,11 @@ +=========== +Get Started +=========== + +.. toctree:: + :maxdepth: 2 + + install + quickstart + cluster-setup + building-from-source diff --git a/doc/images/architecture.png b/doc/images/architecture.png new file mode 100644 index 0000000000..fc775160ca Binary files /dev/null and b/doc/images/architecture.png differ diff --git a/doc/images/cluster-diagram.png b/doc/images/cluster-diagram.png new file mode 100644 index 0000000000..745a9220b4 Binary files /dev/null and b/doc/images/cluster-diagram.png differ diff --git a/doc/images/cluster/Makefile b/doc/images/cluster/Makefile new file mode 100644 index 0000000000..21fb7fb85e --- /dev/null +++ b/doc/images/cluster/Makefile @@ -0,0 +1,6 @@ +MMDC?=./node_modules/.bin/mmdc + +%.png : %.mermaid + $(MMDC) -i $< -e png -o $@ + +all: zeromq-cluster.png zeromq-pubsub.png zeromq-logging.png diff --git a/doc/images/cluster/README.md b/doc/images/cluster/README.md new file mode 100644 index 0000000000..ab93a3580c --- /dev/null +++ b/doc/images/cluster/README.md @@ -0,0 +1,24 @@ +## Install mermaid-cli + + npm install @mermaid-js/mermaid-cli + +## Apparmor Errors + +If running ``mmdc`` fails under Linux (e.g. with Ubuntu 24.04) with apparmor +errors about ``userns_create`` in the ``demsg`` output, put the following into +``/etc/apparmor.d/chrome-headless`` + + # This profile allows everything and only exists to give the + # application a name instead of having the label "unconfined" + abi , + include + + profile chrome /home/awelzel/.cache/puppeteer/**/chrome-headless-shell flags=(unconfined) { + userns, + + # Site-specific additions and overrides. See local/README for details. + include if exists + } + + +See also: https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md#option-2_a-safer-way diff --git a/doc/images/cluster/zeromq-cluster.mermaid b/doc/images/cluster/zeromq-cluster.mermaid new file mode 100644 index 0000000000..d5dd1bef86 --- /dev/null +++ b/doc/images/cluster/zeromq-cluster.mermaid @@ -0,0 +1,95 @@ +graph TD + + w1-xpub-->broker-xsub + broker-xpub-->w1-xsub + + w2-xpub-->broker-xsub + broker-xpub-->w2-xsub + + w3-xpub-->broker-xsub + broker-xpub-->w3-xsub + + p1-xpub-->broker-xsub + broker-xpub-->p1-xsub + + p2-xpub-->broker-xsub + broker-xpub-->p2-xsub + + l1-xpub-->broker-xsub + broker-xpub-->l1-xsub + + l2-xpub-->broker-xsub + broker-xpub-->l2-xsub + + m-xpub-->broker-xsub + broker-xpub-->m-xsub + + %% Logging + w1-push-->l1-pull + w1-push-->l2-pull + w2-push-->l1-pull + w2-push-->l2-pull + w3-push-->l1-pull + w3-push-->l2-pull + p1-push-->l1-pull + p1-push-->l2-pull + p2-push-->l1-pull + p2-push-->l2-pull + m-push-->l1-pull + m-push-->l2-pull + +subgraph broker ["broker"] + broker-xpub((XPUB)) + broker-xsub((XSUB)) + broker-xpub-->broker-xsub + broker-xsub-->broker-xpub +end + + +subgraph l1 [logger-1] + l1-xpub((XPUB)) + l1-xsub((XSUB)) + l1-pull(PULL) +end + +subgraph l2 [logger-2] + l2-xpub((XPUB)) + l2-xsub((XSUB)) + l2-pull(PULL) +end + +subgraph manager + m-xpub((XPUB)) + m-xsub((XSUB)) + m-push(PUSH) +end + +subgraph p1 [proxy-1] + p1-xpub((XPUB)) + p1-xsub((XSUB)) + p1-push(PUSH) +end + +subgraph p2 [proxy-2] + p2-xpub((XPUB)) + p2-xsub((XSUB)) + p2-push(PUSH) +end + +subgraph w1 [worker-1] + w1-xpub((XPUB)) + w1-xsub((XSUB)) + w1-push(PUSH) +end + +subgraph w2 [worker-2] + w2-xpub((XPUB)) + w2-xsub((XSUB)) + w2-push(PUSH) +end + +subgraph w3 [worker-3] + w3-xpub((XPUB)) + w3-xsub((XSUB)) + w3-push(PUSH) +end diff --git a/doc/images/cluster/zeromq-cluster.png b/doc/images/cluster/zeromq-cluster.png new file mode 100644 index 0000000000..bc2af6502a Binary files /dev/null and b/doc/images/cluster/zeromq-cluster.png differ diff --git a/doc/images/cluster/zeromq-logging.mermaid b/doc/images/cluster/zeromq-logging.mermaid new file mode 100644 index 0000000000..fcdd378dcf --- /dev/null +++ b/doc/images/cluster/zeromq-logging.mermaid @@ -0,0 +1,49 @@ +flowchart TD + + %% Logging + w1-push-->l1-pull + w1-push-->l2-pull + w2-push-->l1-pull + w2-push-->l2-pull + w3-push-->l1-pull + w3-push-->l2-pull + p1-push-->l1-pull + p1-push-->l2-pull + p2-push-->l1-pull + p2-push-->l2-pull + m-push-->l1-pull + m-push-->l2-pull + +subgraph l1 [logger-1] + l1-pull(PULL) +end + +subgraph l2 [logger-2] + l2-pull(PULL) +end + +subgraph m [manager] + m-push(PUSH) +end + +subgraph p1 [proxy-1] + p1-push(PUSH) +end + +subgraph p2 [proxy-2] + p2-push(PUSH) +end + +subgraph w1 [worker-1] + w1-push(PUSH) +end + +subgraph w2 [worker-2] + w2-push(PUSH) +end + +subgraph w3 [worker-3] + + w3-push(PUSH) +end + diff --git a/doc/images/cluster/zeromq-logging.png b/doc/images/cluster/zeromq-logging.png new file mode 100644 index 0000000000..6c9e9afb3e Binary files /dev/null and b/doc/images/cluster/zeromq-logging.png differ diff --git a/doc/images/cluster/zeromq-pubsub.mermaid b/doc/images/cluster/zeromq-pubsub.mermaid new file mode 100644 index 0000000000..7710f8774f --- /dev/null +++ b/doc/images/cluster/zeromq-pubsub.mermaid @@ -0,0 +1,72 @@ +graph TD + + w1-xpub-->broker-xsub + broker-xpub-->w1-xsub + + w2-xpub-->broker-xsub + broker-xpub-->w2-xsub + + w3-xpub-->broker-xsub + broker-xpub-->w3-xsub + + p1-xpub-->broker-xsub + broker-xpub-->p1-xsub + + p2-xpub-->broker-xsub + broker-xpub-->p2-xsub + + l1-xpub-->broker-xsub + broker-xpub-->l1-xsub + + l2-xpub-->broker-xsub + broker-xpub-->l2-xsub + + m-xpub-->broker-xsub + broker-xpub-->m-xsub + +subgraph broker ["broker"] + broker-xpub((XPUB)) + broker-xsub((XSUB)) + broker-xpub-->broker-xsub + broker-xsub-->broker-xpub +end + +subgraph l1 [logger-1] + l1-xpub((XPUB)) + l1-xsub((XSUB)) +end + +subgraph l2 [logger-2] + l2-xpub((XPUB)) + l2-xsub((XSUB)) +end +subgraph m [manager] + m-xpub((XPUB)) + m-xsub((XSUB)) +end + +subgraph p1 [proxy-1] + p1-xpub((XPUB)) + p1-xsub((XSUB)) +end + +subgraph p2 [proxy-2] + p2-xpub((XPUB)) + p2-xsub((XSUB)) +end + +subgraph w1 [worker-1] + w1-xpub((XPUB)) + w1-xsub((XSUB)) +end + +subgraph w2 [worker-2] + w2-xpub((XPUB)) + w2-xsub((XSUB)) +end + +subgraph w3 [worker-3] + w3-xpub((XPUB)) + w3-xsub((XSUB)) +end + diff --git a/doc/images/cluster/zeromq-pubsub.png b/doc/images/cluster/zeromq-pubsub.png new file mode 100644 index 0000000000..108946d75a Binary files /dev/null and b/doc/images/cluster/zeromq-pubsub.png differ diff --git a/doc/images/collection-figure1.png b/doc/images/collection-figure1.png new file mode 100644 index 0000000000..793bed5ee3 Binary files /dev/null and b/doc/images/collection-figure1.png differ diff --git a/doc/images/collection-figure2.png b/doc/images/collection-figure2.png new file mode 100644 index 0000000000..5ac1a59a16 Binary files /dev/null and b/doc/images/collection-figure2.png differ diff --git a/doc/images/collection-figure3.png b/doc/images/collection-figure3.png new file mode 100644 index 0000000000..2f45d84dfe Binary files /dev/null and b/doc/images/collection-figure3.png differ diff --git a/doc/images/deployment.png b/doc/images/deployment.png new file mode 100644 index 0000000000..abd9b92722 Binary files /dev/null and b/doc/images/deployment.png differ diff --git a/doc/images/intel-architecture.png b/doc/images/intel-architecture.png new file mode 100644 index 0000000000..e10a5adc79 Binary files /dev/null and b/doc/images/intel-architecture.png differ diff --git a/doc/images/management-all-in-one-two-zeeks.png b/doc/images/management-all-in-one-two-zeeks.png new file mode 100644 index 0000000000..18c1d68adf Binary files /dev/null and b/doc/images/management-all-in-one-two-zeeks.png differ diff --git a/doc/images/management-all-in-one-two-zeeks.svgz b/doc/images/management-all-in-one-two-zeeks.svgz new file mode 100644 index 0000000000..fae14164e9 Binary files /dev/null and b/doc/images/management-all-in-one-two-zeeks.svgz differ diff --git a/doc/images/management-all-in-one.png b/doc/images/management-all-in-one.png new file mode 100644 index 0000000000..c5f1fe2a4f Binary files /dev/null and b/doc/images/management-all-in-one.png differ diff --git a/doc/images/management-all-in-one.svgz b/doc/images/management-all-in-one.svgz new file mode 100644 index 0000000000..ae26d3a729 Binary files /dev/null and b/doc/images/management-all-in-one.svgz differ diff --git a/doc/images/management.png b/doc/images/management.png new file mode 100644 index 0000000000..2103c4d5d7 Binary files /dev/null and b/doc/images/management.png differ diff --git a/doc/images/management.svgz b/doc/images/management.svgz new file mode 100644 index 0000000000..067842635c Binary files /dev/null and b/doc/images/management.svgz differ diff --git a/doc/images/troubleshooting/flamegraph.png b/doc/images/troubleshooting/flamegraph.png new file mode 100644 index 0000000000..16bc2d37bf Binary files /dev/null and b/doc/images/troubleshooting/flamegraph.png differ diff --git a/doc/images/troubleshooting/http-fake-state-growth.gif b/doc/images/troubleshooting/http-fake-state-growth.gif new file mode 100644 index 0000000000..b1c9c6d8f8 Binary files /dev/null and b/doc/images/troubleshooting/http-fake-state-growth.gif differ diff --git a/doc/images/websocket-api/one-api-many-zeek-ws-bridge.svg b/doc/images/websocket-api/one-api-many-zeek-ws-bridge.svg new file mode 100644 index 0000000000..5bb224b906 --- /dev/null +++ b/doc/images/websocket-api/one-api-many-zeek-ws-bridge.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/images/websocket-api/one-api-many-zeek.svg b/doc/images/websocket-api/one-api-many-zeek.svg new file mode 100644 index 0000000000..7171dd712c --- /dev/null +++ b/doc/images/websocket-api/one-api-many-zeek.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/images/zeek-favicon.ico b/doc/images/zeek-favicon.ico new file mode 100644 index 0000000000..c979c54037 Binary files /dev/null and b/doc/images/zeek-favicon.ico differ diff --git a/doc/images/zeek-logo-sidebar.png b/doc/images/zeek-logo-sidebar.png new file mode 100644 index 0000000000..8a5c0cc528 Binary files /dev/null and b/doc/images/zeek-logo-sidebar.png differ diff --git a/doc/images/zeek-logo-text.png b/doc/images/zeek-logo-text.png new file mode 100644 index 0000000000..1ea172adc3 Binary files /dev/null and b/doc/images/zeek-logo-text.png differ diff --git a/doc/images/zeek-logo.png b/doc/images/zeek-logo.png new file mode 100644 index 0000000000..b4cb7c2158 Binary files /dev/null and b/doc/images/zeek-logo.png differ diff --git a/doc/index.rst b/doc/index.rst new file mode 100644 index 0000000000..77d152833b --- /dev/null +++ b/doc/index.rst @@ -0,0 +1,70 @@ + +.. image:: /images/zeek-logo-text.png + :align: center + +================== +Zeek Documentation +================== + +.. important:: + + Make sure to read the :ref:`appropriate documentation version + `. + +The purpose of this manual is to assist the Zeek community with implementing +Zeek in their environments. It includes material on Zeek's unique +capabilities, how to install it, how to interpret the default logs that Zeek +generates, and how to modify Zeek to fit your needs. This documentation is the +result of a volunteer community effort. If you would like to contribute, or +want more information, please visit the `Zeek web page +`_ for details on how +to connect with the community. + +.. toctree:: + :maxdepth: 2 + :caption: Table of Contents + + get-started + about + monitoring + log-formats + logs/index + scripting/index + frameworks/index + customizations + troubleshooting + script-reference/index + devel/index + components/index + acknowledgements + +* :ref:`Index ` + +.. _documentation-versioning: + +Documentation Versioning +======================== + +.. attention:: + + The Zeek codebase has three primary branches of interest to users so this + document is also maintained as three different versions, one associated with + each branch of Zeek. The default version of `docs.zeek.org + `_ tracks Zeek's latest Git development: + + * Git *master* branch: https://docs.zeek.org/en/master + + If you instead use a Zeek Long-Term Support (LTS) or Feature release these + are the appropriate starting points: + + * Long-Term Support Release: https://docs.zeek.org/en/lts + * Current Feature Release: https://docs.zeek.org/en/current + + To help clarify which release you are using, the version numbering + scheme for the two release branches is described in the `Release + Cadence `_ policy. + + Documentation for older Zeek releases remains available for approximately one + full major-version release cycle, i.e., about a year. You can browse recent + versions via the fly-out menu in the bottom left, and find all available + versions on the `RTD website `_. diff --git a/doc/install.rst b/doc/install.rst new file mode 100644 index 0000000000..f8cf15c652 --- /dev/null +++ b/doc/install.rst @@ -0,0 +1,140 @@ + +.. _Homebrew: https://brew.sh +.. _zkg package manager: https://docs.zeek.org/projects/package-manager/en/stable/ + +.. _installing-zeek: + +=============== +Installing Zeek +=============== + +To run Zeek, grab our official Docker images, download our Linux binary +packages, install via Homebrew_ on your Mac, use the ports collections on FreeBSD +and OpenBSD. See the :doc:`building-from-source` section to build Zeek yourself. +For details about our release cadence and the significance of Zeek's version +numbers, please refer to our `Release Cadence +`_ wiki page. + +.. _docker-images: + +Docker Images +============= + +We provide official Docker images on Docker Hub at https://hub.docker.com/u/zeek + + * For the latest feature release: ``docker pull zeek/zeek:latest`` + * For the latest LTS release: ``docker pull zeek/zeek:lts`` + * For the latest release in a given series: ``docker pull zeek/zeek:7.2`` + * For a specific release: ``docker pull zeek/zeek:7.0.8`` + * For the nightly build: ``docker pull zeek/zeek-dev:latest`` + +Additionally, we push these images to Amazon's Public Elastic Container +Registry (ECR) in the `Zeek Project `_ +public gallery. To use Amazon's container registry instead of Docker Hub, +prefix images with ``public.ecr.aws/zeek`` instead of ``zeek``. + + * For instance, to pull the latest feature release: ``docker pull public.ecr.aws/zeek/zeek:latest`` + +The images are Debian-based and feature a complete Zeek installation with ``zeek``, +``zkg``, and the Spicy toolchain, but are otherwise minimal to avoid bloat in +derived images. For example, if you'd like to install Zeek plugins in those +images, you'll need to install their needed toolchain, typically at least +``g++`` for compilation, ``cmake`` and ``make`` as build tools, and +``libpcap-dev`` to build against Zeek headers. Similarly, you'll need ``g++`` +for Spicy's JIT compilation, as well as ``cmake`` and ``make`` to build Spicy +analyzer packages. + + .. code-block:: console + + apt-get update + apt-get install -y --no-install-recommends g++ cmake make libpcap-dev + +The source files used to create the container images are on +`GitHub `_. + +.. _binary-packages: + +Binary Packages +=============== + +Linux +----- + +We provide `binary packages `_ +for a wide range of Linux distributions via the `openSUSE Build Service +`_. To install, first add the relevant OBS +package repository to your system, then use your system's package manager +as usual. + +We provide the following groups of packages: + + * ``zeek-X.0``: specific LTS release lines, currently `7.0.x `_ (`sources `__), `6.0.x `_ (`sources `__), and `5.0.x `_ (`sources `__). + * ``zeek``: the `latest Zeek release `_ (`sources `__) + * ``zeek-nightly``: our `nightly builds `_ (`sources `__) + * ``zeek-rc``: our `release candidates `_ (`sources `__) + +For example, for the latest Zeek 7.0 LTS release on Ubuntu 22.04 the steps look as follows: + + .. code-block:: console + + echo 'deb https://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /' | sudo tee /etc/apt/sources.list.d/security:zeek.list + curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null + sudo apt update + sudo apt install zeek-7.0 + +.. note:: Our motivation for this approach is twofold. First, it guarantees LTS + users that they won't unexpectedly end up on a newer LTS line when it comes + out. For example, when you install the ``zeek-6.0`` packages, you will not + end up on Zeek 7.0 until you decide to switch. Second, it reflects the fact + that we consider our x.1 and x.2 feature release lines transient, because + they go out of support immediately once we move to the next line of feature + releases. Therefore, users of the ``zeek`` packages automatically obtain the + latest releases as we publish them. + + In the past our binary packages also automatically transitioned our LTS users + to newer versions, via the older ``zeek-lts`` packages. These remain visible + on OBS but are no longer supported. + +The primary install prefix for binary packages is :file:`/opt/zeek` (depending +on which version you’re using), and includes a complete Zeek environment with +``zeek`` itself, the `zkg package manager`_, the Spicy toolchain, etc. + +See our `Binary Packages wiki page `_ +for the latest updates on binary releases. + +macOS +----- + +The Zeek `Homebrew formula `_ +provides binary packages ("bottles"). To install: + + .. code-block:: console + + brew install zeek + +These packages are not maintained by the Zeek project. + +FreeBSD +------- + +Zeek is available from the `FreeBSD ports collection `_. +To install: + + .. code-block:: console + + sudo pkg install -y zeek + +These packages are not maintained by the Zeek project. + +OpenBSD +------- + +Zeek is available from the `OpenBSD ports collection `_. +To install: + + .. code-block:: console + + sudo pkg_add zeek + +These packages are not maintained by the Zeek project. + diff --git a/doc/log-formats.rst b/doc/log-formats.rst new file mode 100644 index 0000000000..d239a1acf1 --- /dev/null +++ b/doc/log-formats.rst @@ -0,0 +1,657 @@ +.. _logschema: https://github.com/zeek/logschema + +=============================== +Zeek Log Formats and Inspection +=============================== + +Zeek creates a variety of logs when run in its default configuration. This +data can be intimidating for a first-time user. In this section, we will +process a sample packet trace with Zeek, and take a brief look at the sorts +of logs Zeek creates. We will look at logs created in Zeek's traditional TSV +format, how to switch to logging in JSON format, and assorted tooling to +help you work with the logs. Finally, we'll cover Zeek's support for log +schemas that describe what Zeek's logs look like in detail. + +Working with a Sample Trace +=========================== + +For the examples that follow, we will use Zeek on a Linux system to process +network traffic captured and stored to disk. We saved this trace file earlier +in packet capture (PCAP) format as :file:`tm1t.pcap`. The command line protocol +analyzer Tcpdump, which ships with most Unix-like distributions, summarizes the +contents of this file. + +.. code-block:: console + + zeek@zeek:~/zeek-test$ tcpdump -n -r tm1t.pcap + +:: + + reading from file tm1t.pcap, link-type EN10MB (Ethernet) + 14:39:59.305988 IP 192.168.4.76.36844 > 192.168.4.1.53: 19671+ A? testmyids.com. (31) + 14:39:59.306059 IP 192.168.4.76.36844 > 192.168.4.1.53: 8555+ AAAA? testmyids.com. (31) + 14:39:59.354577 IP 192.168.4.1.53 > 192.168.4.76.36844: 8555 0/1/0 (94) + 14:39:59.372840 IP 192.168.4.1.53 > 192.168.4.76.36844: 19671 1/0/0 A 31.3.245.133 (47) + 14:39:59.430166 IP 192.168.4.76.46378 > 31.3.245.133.80: Flags [S], seq 3723031366, win 65535, options [mss 1460,sackOK,TS val 3137978796 ecr 0,nop,wscale 11], length 0 + 14:39:59.512232 IP 31.3.245.133.80 > 192.168.4.76.46378: Flags [S.], seq 2993782376, ack 3723031367, win 28960, options [mss 1460,sackOK,TS val 346747623 ecr 3137978796,nop,wscale 7], length 0 + 14:39:59.512284 IP 192.168.4.76.46378 > 31.3.245.133.80: Flags [.], ack 1, win 32, options [nop,nop,TS val 3137978878 ecr 346747623], length 0 + 14:39:59.512593 IP 192.168.4.76.46378 > 31.3.245.133.80: Flags [P.], seq 1:78, ack 1, win 32, options [nop,nop,TS val 3137978878 ecr 346747623], length 77: HTTP: GET / HTTP/1.1 + 14:39:59.600488 IP 31.3.245.133.80 > 192.168.4.76.46378: Flags [.], ack 78, win 227, options [nop,nop,TS val 346747711 ecr 3137978878], length 0 + 14:39:59.604000 IP 31.3.245.133.80 > 192.168.4.76.46378: Flags [P.], seq 1:296, ack 78, win 227, options [nop,nop,TS val 346747713 ecr 3137978878], length 295: HTTP: HTTP/1.1 200 OK + 14:39:59.604020 IP 192.168.4.76.46378 > 31.3.245.133.80: Flags [.], ack 296, win 33, options [nop,nop,TS val 3137978970 ecr 346747713], length 0 + 14:39:59.604493 IP 192.168.4.76.46378 > 31.3.245.133.80: Flags [F.], seq 78, ack 296, win 33, options [nop,nop,TS val 3137978970 ecr 346747713], length 0 + 14:39:59.684281 IP 31.3.245.133.80 > 192.168.4.76.46378: Flags [F.], seq 296, ack 79, win 227, options [nop,nop,TS val 346747796 ecr 3137978970], length 0 + 14:39:59.684346 IP 192.168.4.76.46378 > 31.3.245.133.80: Flags [.], ack 297, win 33, options [nop,nop,TS val 3137979050 ecr 346747796], length 0 + +This is a simple exchange involving domain name system (DNS) traffic followed +by HyperText Transfer Protocol (HTTP) traffic. + +Rather than run Zeek against a live interface, we will ask Zeek to digest this +trace. This process allows us to vary Zeek’s run-time operation, keeping the +traffic constant. + +First we make two directories to store the log files that Zeek will produce. +Then we will move into the “default” directory. + +.. code-block:: console + + zeek@zeek:~/zeek-test$ mkdir default + zeek@zeek:~/zeek-test$ mkdir json + zeek@zeek:~/zeek-test$ cd default/ + +Zeek TSV Format Logs +==================== + +From this location on disk, we tell Zeek to digest the :file:`tm1t.pcap` file. + +.. code-block:: console + + zeek@zeek:~/zeek-test/default$ zeek -C -r ../tm1t.pcap + +The ``-r`` flag tells Zeek where to find the trace of interest. + +The ``-C`` flag tells Zeek to ignore any TCP checksum errors. This happens on +many systems due to a feature called “checksum offloading,” but it does not +affect our analysis. + +Zeek completes its task without reporting anything to the command line. This is +standard Unix-like behavior. Using the :program:`ls` command we see what files +Zeek created when processing the trace. + +.. code-block:: console + + zeek@zeek:~/zeek-test/default$ ls -al + +:: + + total 28 + drwxrwxr-x 2 zeek zeek 4096 Jun 5 14:48 . + drwxrwxr-x 4 zeek zeek 4096 Jun 5 14:43 .. + -rw-rw-r-- 1 zeek zeek 737 Jun 5 14:48 conn.log + -rw-rw-r-- 1 zeek zeek 778 Jun 5 14:48 dns.log + -rw-rw-r-- 1 zeek zeek 712 Jun 5 14:48 files.log + -rw-rw-r-- 1 zeek zeek 883 Jun 5 14:48 http.log + -rw-rw-r-- 1 zeek zeek 254 Jun 5 14:48 packet_filter.log + +Zeek created five files. We will look at the contents of Zeek log data in +detail in later sections. For now, we will take a quick look at each file, +beginning with the :file:`conn.log`. + +We use the :program:`cat` command to show the contents of each log. + +.. code-block:: console + + zeek@zeek:~/zeek-test/default$ cat conn.log + +:: + + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path conn + #open 2020-06-05-14-48-32 + #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents ip_proto + #types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] count + 1591367999.305988 CazOhH2qDUiJTWMCY 192.168.4.76 36844 192.168.4.1 53 udp dns 0.066852 62 141 SF - -0 Dd 2 118 2 197 - 17 + 1591367999.430166 CLqEx41jYPOdfHF586 192.168.4.76 46378 31.3.245.133 80 tcp http 0.254115 77 295 SF - -0 ShADadFf 6 397 4 511 - 6 + #close 2020-06-05-14-48-32 + +Next we look at Zeek’s :file:`dns.log`. + +.. code-block:: console + + zeek@zeek:~/zeek-test/default$ cat dns.log + +:: + + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path dns + #open 2020-06-05-14-48-32 + #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id rtt query qclass qclass_name qtypeqtype_name rcode rcode_name AA TC RD RA Z answers TTLs rejected + #types time string addr port addr port enum count interval string count string count string count string bool bool bool bool count vector[string] vector[interval] bool + 1591367999.306059 CazOhH2qDUiJTWMCY 192.168.4.76 36844 192.168.4.1 53 udp 8555 - testmyids.com 1 C_INTERNET 28 AAAA 0 NOERROR F F T F 0 - - F + 1591367999.305988 CazOhH2qDUiJTWMCY 192.168.4.76 36844 192.168.4.1 53 udp 19671 0.066852 testmyids.com 1 C_INTERNET 1 A 0 NOERROR F F T T 0 31.3.245.133 3600.000000 F + #close 2020-06-05-14-48-32 + +Next we look at Zeek’s :file:`files.log`. + +.. code-block:: console + + zeek@zeek:~/zeek-test/default$ cat files.log + +:: + + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path files + #open 2020-06-05-14-48-32 + #fields ts fuid uid id.orig_h id.origh_p id.resp_h id.resp_p source depth analyzers mime_type filename duration local_orig is_orig seen_bytes total_bytes missing_bytes overflow_bytes timedout parent_fuid md5 sha1 sha256 extracted extracted_cutoff extracted_size + #types time string string addr port addr port string count set[string] string string interval bool bool countcount count count bool string string string string string bool count + 1591367999.604000 FEEsZS1w0Z0VJIb5x4 CLqEx41jYPOdfHF586 192.168.4.76 46378 31.3.245.133 80 HTTP 0 (empty) text/plain - 0.000000 - F 39 39 0 0 F - - - - - - - + #close 2020-06-05-14-48-32 + +Next we look at Zeek’s :file:`http.log`. + +.. code-block:: console + + zeek@zeek:~/zeek-test/default$ cat http.log + +:: + + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path http + #open 2020-06-05-14-48-32 + #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer version user_agent origin request_body_len response_body_len status_code status_msg info_code info_msg tags username password proxied orig_fuids orig_filenames orig_mime_types resp_fuids resp_filenames resp_mime_types + #types time string addr port addr port count string string string string string string string count count count string countstring set[enum] string string set[string] vector[string] vector[string] vector[string] vector[string] vector[string] vector[string] + 1591367999.512593 CLqEx41jYPOdfHF586 192.168.4.76 46378 31.3.245.133 80 1 GET testmyids.com / - 1.1 curl/7.47.0 - 0 39 200 OK - - (empty) - - - - - - FEEsZS1w0Z0VJIb5x4 - text/plain + #close 2020-06-05-14-48-32 + +Finally, we look at Zeek’s :file:`packet_filter.log`. This log shows any +filters that Zeek applied when processing the trace. + +.. code-block:: console + + zeek@zeek:~/zeek-test/default$ cat packet_filter.log + +:: + + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path packet_filter + #open 2020-06-05-14-48-32 + #fields ts node filter init success + #types time string string bool bool + 1591368512.420771 zeek ip or not ip T T + #close 2020-06-05-14-48-32 + +As we can see with each log file, there is a set of headers beginning with the +hash character (``#``) followed by metadata about the trace. This format is the +standard version of Zeek data, represented as tab separated values (TSV). + +Interpreting this data as shown requires remembering which “column” applies to +which “value.” For example, in the :file:`dns.log`, the third field is +``id.orig_h``, so when we see data in that field, such as ``192.168.4.76``, we +know that ``192.168.4.76`` is ``id.orig_h``. + +One of the common use cases for interacting with Zeek log files requires +analyzing specific fields. Investigators may not need to see all of the fields +produced by Zeek when solving a certain problem. The following sections offer a +few ways to address this concern when processing Zeek logs in text format. + +Zeek TSV Format and :program:`awk` +================================== + +A very traditional way of interacting with Zeek logs involves using native +Unix-like text processing tools like :program:`awk`. Awk requires specifying +the fields of interest as positions in the log file. Take a second look at the +:file:`dns.log` entry above, and consider the parameters necessary to view only +the source IP address, the query, and the response. These values appear in the +3rd, 10th, and 22nd fields in the Zeek TSV log entries. Therefore, we could +invoke :program:`awk` using the following syntax: + +.. code-block:: console + + zeek@zeek:~/zeek-test/default$ awk '/^[^#]/ {print $3, $10, $22}' dns.log + +:: + + 192.168.4.76 testmyids.com - + 192.168.4.76 testmyids.com 31.3.245.133 + +Now we have a much more compact view, with just the fields we want. +Unfortunately, this requires specifying fields by location. If we were to +modify the log output, or if the Zeek project were to change the log output, +any scripts we built using :program:`awk` and field locations would require +modification. For this reason, the Zeek project recommends alternatives like +the following. + +Zeek TSV Format and :program:`zeek-cut` +======================================= + +The Zeek project provides a tool called :program:`zeek-cut` to make it easier +for analysts to interact with Zeek logs in TSV format. It parses the header in +each file and allows the user to refer to the specific columnar data available. +This is in contrast to tools like :program:`awk` that require the user to refer +to fields referenced by their position. + +Consider the :file:`dns.log` generated earlier. If we process it with +:program:`zeek-cut`, without any modifications, this is the result: + +.. code-block:: console + + zeek@zeek:~/zeek-test/default$ cat dns.log | zeek-cut + +:: + + 1591367999.306059 CazOhH2qDUiJTWMCY 192.168.4.76 36844 192.168.4.1 53 udp 8555 - testmyids.com 1 C_INTERNET 28 AAAA 0 NOERROR F F T F 0 - - F + 1591367999.305988 CazOhH2qDUiJTWMCY 192.168.4.76 36844 192.168.4.1 53 udp 19671 0.066852 testmyids.com 1 C_INTERNET 1 A 0 NOERROR F F T T 0 31.3.245.133 3600.000000 F + +That is the :file:`dns.log`, minus the header fields showed earlier. Note we +have to invoke the cat utility in a pipeline to process files with +:program:`zeek-cut`. + +If we pass :program:`zeek-cut` the fields we wish to see, the output looks like +this: + +.. code-block:: console + + zeek@zeek:~/zeek-test/default$ cat dns.log | zeek-cut id.orig_h query answers + +:: + + 192.168.4.76 testmyids.com - + 192.168.4.76 testmyids.com 31.3.245.133 + +The sequence of field names given to :program:`zeek-cut` determines the output +order. This means you can also use :program:`zeek-cut` to reorder fields. For +example: + +.. code-block:: console + + zeek@zeek:~/zeek-test/default$ cat dns.log | zeek-cut query answers id.orig_h + +:: + + testmyids.com - 192.168.4.76 + testmyids.com 31.3.245.133 192.168.4.76 + +This feature can be helpful when piping output into programs like :program:`sort`. + +:program:`zeek-cut` uses output redirection through the :program:`cat` command +and ``|`` operator. Whereas tools like :program:`awk` allow you to indicate the +log file as a command line option, :program:`zeek-cut` only takes input through +redirection such as ``|`` and ``<``. + +For example, instead of using :program:`cat` and the pipe redirector, we could +obtain the previous output with this syntax: + +.. code-block:: console + + zeek@zeek:~/zeek-test/default$ zeek-cut id.orig_h query answers < dns.log + +:: + + 192.168.4.76 testmyids.com - + 192.168.4.76 testmyids.com 31.3.245.133 + +Note that in its default setup using ZeekControl (but not with a simple +command-line invocation like ``zeek -i eth0``), watching a live interface and +writing logs to disk, Zeek will rotate log files on an hourly basis. Zeek will +move the current log file into a directory named using the format +``YYYY-MM-DD``. Zeek will use :program:`gzip` to compress the file with a naming +convention that includes the log file type and time range of the file. + +When processing a compressed log file, use the :program:`zcat` tool instead of +:program:`cat` to read the file. Consider working with the gzip-encoding file +created in the following example. For demonstration purposes, we create a copy +of the :file:`dns.log` file as :file:`dns1.log`, :program:`gzip` it, and then +read it with :program:`zcat` instead of :program:`cat`. + +.. code-block:: console + + so16@so16:~/zeek-test/default$ cp dns.log dns1.log + so16@so16:~/zeek-test/default$ gzip dns1.log + so16@so16:~/zeek-test/default$ zcat dns1.log.gz + +:: + + #separator \x09 + #set_separator , + #empty_field (empty) + #unset_field - + #path dns + #open 2020-06-05-14-48-32 + #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id rtt query qclass qclass_name qtypeqtype_name rcode rcode_name AA TC RD RA Z answers TTLs rejected + #types time string addr port addr port enum count interval string count string count string count string bool bool bool bool count vector[string] vector[interval] bool + 1591367999.306059 CazOhH2qDUiJTWMCY 192.168.4.76 36844 192.168.4.1 53 udp 8555 - testmyids.com 1 C_INTERNET 28 AAAA 0 NOERROR F F T F 0 - - F + 1591367999.305988 CazOhH2qDUiJTWMCY 192.168.4.76 36844 192.168.4.1 53 udp 19671 0.066852 testmyids.com 1 C_INTERNET 1 A 0 NOERROR F F T T 0 31.3.245.133 3600.000000 F + #close 2020-06-05-14-48-32 + +:program:`zeek-cut` accepts the flag ``-d`` to convert the epoch time values in +the log files to human-readable format. For example, observe the default +timestamp value: + +.. code-block:: console + + zeek@zeek:~/zeek-test/default$ zcat dns1.log.gz | zeek-cut ts id.orig_h query answers + +:: + + 1591367999.306059 192.168.4.76 testmyids.com - + 1591367999.305988 192.168.4.76 testmyids.com 31.3.245.133 + +Now see the effect of using the ``-d`` flag: + +.. code-block:: console + + zeek@zeek:~/zeek-test/default$ cat dns.log | zeek-cut -d ts id.orig_h query answers + +:: + + 2020-06-05T14:39:59+0000 192.168.4.76 testmyids.com - + 2020-06-05T14:39:59+0000 192.168.4.76 testmyids.com 31.3.245.133 + +Converting the timestamp from a log file to UTC can be accomplished with the +``-u`` option. + +The default time format when using the ``-d`` or ``-u`` is the ``strftime`` +format string ``%Y-%m-%dT%H:%M:%S%z`` which results in a string with year, +month, day of month, followed by hour, minutes, seconds and the timezone +offset. + +The default format can be altered by using the ``-D`` and ``-U`` flags, using the +standard ``strftime`` syntax. For example, to format the timestamp in the +US-typical “Middle Endian” you could use a format string of: +``%m-%d-%YT%H:%M:%S%z`` + +.. code-block:: console + + zeek@zeek:~/zeek-test/default$ cat dns.log | zeek-cut -D %d-%m-%YT%H:%M:%S%z ts id.orig_h query answers + +:: + + 06-05-2020T14:39:59+0000 192.168.4.76 testmyids.com - + 06-05-2020T14:39:59+0000 192.168.4.76 testmyids.com 31.3.245.133 + +Using :program:`awk` and :program:`zeek-cut` have been the traditional method +of interacting with Zeek logs. In the next section we will look at the +possibilities once we enable an alternative output format. + +Zeek JSON Format Logs +===================== + +During the last decade, the JavaScript Object Notation (JSON) format has become +a standard way to label and store many types of data. Zeek offers support for +this format. In the following example we will re-run the :file:`tm1t.pcap` trace +through Zeek, but request that it output logs in JSON format. + +First we change into the json directory to avoid overwriting our existing log +files. + +.. code-block:: console + + zeek@zeek:~/zeek-test/default$ cd ../json/ + +Next we tell Zeek to output logs in JSON format using the command as shown. + +.. code-block:: console + + zeek@zeek:~/zeek-test/json$ zeek -C -r ../tm1t.pcap LogAscii::use_json=T + +When we look at the directory contents, we see the same five output files. + +.. code-block:: console + + zeek@zeek:~/zeek-test/json$ ls -al + +:: + + total 28 + drwxrwxr-x 2 zeek zeek 4096 Jun 5 14:47 . + drwxrwxr-x 4 zeek zeek 4096 Jun 5 14:43 .. + -rw-rw-r-- 1 zeek zeek 708 Jun 5 14:47 conn.log + -rw-rw-r-- 1 zeek zeek 785 Jun 5 14:47 dns.log + -rw-rw-r-- 1 zeek zeek 325 Jun 5 14:47 files.log + -rw-rw-r-- 1 zeek zeek 405 Jun 5 14:47 http.log + -rw-rw-r-- 1 zeek zeek 90 Jun 5 14:47 packet_filter.log + +However, if we look at the file contents, the format is much different. + +First we look at :file:`packet_filter.log`. + +.. code-block:: console + + zeek@zeek:~/zeek-test/json$ cat packet_filter.log + +:: + + {"ts":1591368442.854585,"node":"zeek","filter":"ip or not ip","init":true,"success":true} + +Next we look at :file:`conn.log` and :file:`dns.log`: + +.. code-block:: console + + zeek@zeek:~/zeek-test/json$ cat conn.log + +:: + + {"ts":1591367999.305988,"uid":"CMdzit1AMNsmfAIiQc","id.orig_h":"192.168.4.76","id.orig_p":36844,"id.resp_h":"192.168.4.1","id.resp_p":53,"proto":"udp","service":"dns","duration":0.06685185432434082,"orig_bytes":62,"resp_bytes":141,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":2,"orig_ip_bytes":118,"resp_pkts":2,"resp_ip_bytes":197,"ip_proto":17} + {"ts":1591367999.430166,"uid":"C5bLoe2Mvxqhawzqqd","id.orig_h":"192.168.4.76","id.orig_p":46378,"id.resp_h":"31.3.245.133","id.resp_p":80,"proto":"tcp","service":"http","duration":0.25411510467529297,"orig_bytes":77,"resp_bytes":295,"conn_state":"SF","missed_bytes":0,"history":"ShADadFf","orig_pkts":6,"orig_ip_bytes":397,"resp_pkts":4,"resp_ip_bytes":511,"ip_proto":6} + +.. code-block:: console + + zeek@zeek:~/zeek-test/json$ cat dns.log + +:: + + {"ts":1591367999.306059,"uid":"CMdzit1AMNsmfAIiQc","id.orig_h":"192.168.4.76","id.orig_p":36844,"id.resp_h":"192.168.4.1","id.resp_p":53,"proto":"udp","trans_id":8555,"query":"testmyids.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} + {"ts":1591367999.305988,"uid":"CMdzit1AMNsmfAIiQc","id.orig_h":"192.168.4.76","id.orig_p":36844,"id.resp_h":"192.168.4.1","id.resp_p":53,"proto":"udp","trans_id":19671,"rtt":0.06685185432434082,"query":"testmyids.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["31.3.245.133"],"TTLs":[3600.0],"rejected":false} + +Next we look at :file:`files.log`. + +.. code-block:: console + + zeek@zeek:~/zeek-test/json$ cat files.log + +:: + + {"ts":1591367999.604,"fuid":"FEEsZS1w0Z0VJIb5x4","uid":"C5bLoe2Mvxqhawzqqd","id.orig_h":"192.168.4.76","id.orig_p":46378,"id.resp_h":"31.3.245.133","id.resp_p":80,"source":"HTTP","depth":0,"analyzers":[],"mime_type":"text/plain","duration":0.0,"is_orig":false,"seen_bytes":39,"total_bytes":39,"missing_bytes":0,"overflow_bytes":0,"timedout":false} + +Next we look at the :file:`http.log`. + +.. code-block:: console + + zeek@zeek:~/zeek-test/json$ cat http.log + +:: + + {"ts":1591367999.512593,"uid":"C5bLoe2Mvxqhawzqqd","id.orig_h":"192.168.4.76","id.orig_p":46378,"id.resp_h":"31.3.245.133","id.resp_p":80,"trans_depth":1,"method":"GET","host":"testmyids.com","uri":"/","version":"1.1","user_agent":"curl/7.47.0","request_body_len":0,"response_body_len":39,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FEEsZS1w0Z0VJIb5x4"],"resp_mime_types":["text/plain"]} + +Comparing the two log styles, we see strengths and weaknesses for each. For +example, the TSV format shows the Zeek types associated with each entry, such +as ``string``, ``addr``, ``port``, and so on. The JSON format does not include +that data. However, the JSON format associates each field “key” with a +“value,” such as ``"id.orig_p":46378``. While this necessarily increases the +amount of disk space used to store the raw logs, it makes it easier for +analysts and software to interpret the data, as the key is directly associated +with the value that follows. For this reason, most developers and analysts have +adopted the JSON output format for Zeek logs. That is the format we will use +for the log analysis sections of the documentation. + +Zeek JSON Format and :program:`jq` +================================== + +Analysts sometimes choose to inspect JSON-formatted Zeek files using +applications that recognize JSON format, such as :program:`jq`, which is a +JSON parser by Stephen Dolan, available at GitHub +(https://stedolan.github.io/jq/). It may already be installed on your Unix-like +system. + +In the following example we process the :file:`dns.log` file with the ``.`` +filter, which tells :program:`jq` to simply output what it finds in the file. +By default :program:`jq` outputs JSON formatted data in its “pretty-print” +style, which puts one key:value pair on each line as shown. + +.. code-block:: console + + so16@so16:~/zeek-test/json$ jq . dns.log + +:: + + { + "ts": 1591367999.306059, + "uid": "CMdzit1AMNsmfAIiQc", + "id.orig_h": "192.168.4.76", + "id.orig_p": 36844, + "id.resp_h": "192.168.4.1", + "id.resp_p": 53, + "proto": "udp", + "trans_id": 8555, + "query": "testmyids.com", + "qclass": 1, + "qclass_name": "C_INTERNET", + "qtype": 28, + "qtype_name": "AAAA", + "rcode": 0, + "rcode_name": "NOERROR", + "AA": false, + "TC": false, + "RD": true, + "RA": false, + "Z": 0, + "rejected": false + } + { + "ts": 1591367999.305988, + "uid": "CMdzit1AMNsmfAIiQc", + "id.orig_h": "192.168.4.76", + "id.orig_p": 36844, + "id.resp_h": "192.168.4.1", + "id.resp_p": 53, + "proto": "udp", + "trans_id": 19671, + "rtt": 0.06685185432434082, + "query": "testmyids.com", + "qclass": 1, + "qclass_name": "C_INTERNET", + "qtype": 1, + "qtype_name": "A", + "rcode": 0, + "rcode_name": "NOERROR", + "AA": false, + "TC": false, + "RD": true, + "RA": true, + "Z": 0, + "answers": [ + "31.3.245.133" + ], + "TTLs": [ + 3600 + ], + "rejected": false + } + +We can tell :program:`jq` to output what it sees in “compact” format using the +``-c`` switch. + +.. code-block:: console + + so16@so16:~/zeek-test/json$ jq . -c dns.log + +:: + + {"ts":1591367999.306059,"uid":"CMdzit1AMNsmfAIiQc","id.orig_h":"192.168.4.76","id.orig_p":36844,"id.resp_h":"192.168.4.1","id.resp_p":53,"proto":"udp","trans_id":8555,"query":"testmyids.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} + {"ts":1591367999.305988,"uid":"CMdzit1AMNsmfAIiQc","id.orig_h":"192.168.4.76","id.orig_p":36844,"id.resp_h":"192.168.4.1","id.resp_p":53,"proto":"udp","trans_id":19671,"rtt":0.06685185432434082,"query":"testmyids.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["31.3.245.133"],"TTLs":[3600],"rejected":false} + +The power of :program:`jq` becomes evident when we decide we only want to see +specific values. For example, the following tells :program:`jq` to look at the +:file:`dns.log` and report the source IP of systems doing DNS queries, followed +by the query, and any answer to the query. + +.. code-block:: console + + so16@so16:~/zeek-test/json$ jq -c '[."id.orig_h", ."query", ."answers"]' dns.log + +:: + + ["192.168.4.76","testmyids.com",null] + ["192.168.4.76","testmyids.com",["31.3.245.133"]] + +For a more comprehensive description of the capabilities of :program:`jq`, +see the `jq manual `_. + +With this basic understanding of how to interact with Zeek logs, we can now +turn to specific logs and interpret their values. + +Log Schemas +=========== + +It's important to note that the exact set and shape of Zeek's logs is highly +site-dependent. While every Zeek version ships with a set of logs enabled by +default, it also includes optional ones that you're welcome to enable. (Feel +free to peruse the :ref:`full set`.) In addition, many of Zeek's +`add-on packages `_ introduce logs of their own, or +enrich existing ones with additional metadata. And finally, Zeek's +:ref:`logging framework ` lets you apply your own log +customizations with a bit of scripting. + +Zeek's `logschema `_ package helps you +understand your Zeek logs. It produces log schemas that detail your +installation's set of logs and their fields. For each field, the schemas +provide rich metadata including name, type, and docstrings. They can also +explain the source of a field, such as the specific script or the name of the +Zeek package that added it. Log schemas are also a great way to understand how +and whether your logs change when you upgrade to a newer version of Zeek. + +To produce schemas, you need to tell Zeek which schema exporters to load. +An easy way to do this is to simply start Zeek with your installed packages +and an exporter of your choice. To get started, try the following: + + .. code-block:: console + + $ zkg install logschema + $ zeek logschema/export/jsonschema packages + +Your local directory will now contain a JSON Schema description for each of your +installation's logs. + + .. code-block:: console + + $ cat zeek-conn-log.schema.json | jq + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Schema for Zeek conn.log", + "description": "JSON Schema for Zeek conn.log", + "type": "object", + "properties": { + "ts": { + "description": "This is the time of the first packet.", + "type": "number", + "examples": [ + "1737691432.132607" + ], + "x-zeek": { + "type": "time", + "record_type": "Conn::Info", + "is_optional": false, + "script": "base/protocols/conn/main.zeek" + } + }, + ... + } + +The logschema package supports a range of schema formats including JSON Schema +and CSV, and works with Zeek 5.2 and newer. Take a look at the package's +`documentation `_ for details. diff --git a/doc/logs/analyzer.rst b/doc/logs/analyzer.rst new file mode 100644 index 0000000000..67933a40f1 --- /dev/null +++ b/doc/logs/analyzer.rst @@ -0,0 +1,403 @@ +============ +analyzer.log +============ + +Dynamic protocol detection (DPD) is a method by which Zeek identifies protocols +on ports beyond those used as standard services. Rather than selecting which +application protocol analyzer to use based on a connection’s server port, +Zeek’s dynamic analyzer framework associates an analyzer tree with every +connection. This analyzer tree permits Zeek to perform protocol analysis +independently of port numbers. + +By using a set of signatures which match typical protocol dialogues, Zeek is +able to look at payload to find the correct analyzers. When such a signature +matches, it turns on the corresponding analyzer to confirm it. Zeek can turn +off analyzers when it becomes obvious that they are parsing the wrong protocol. +This allows Zeek to use “loose” protocol signatures, and, if in doubt, try +multiple analyzers in parallel. + +Zeek’s :file:`analyzer.log` reports problems with the DPD mechanism. This section +provides examples of this reporting in action. + +For full details on each field in the :file:`analyzer.log` file, please refer to +:zeek:see:`Analyzer::Logging::Info`. + +.. note:: + + This log underwent a revamp in Zeek 8.0 and resembles what dpd.log provided + in older versions of Zeek. Please review Zeek 8.0's release notes for more + details on the changes in this log. + +One Specific Example +==================== + +The following is an example of traffic that generated a :file:`analyzer.log` entry. + +:program:`tcpdump` and :program:`tshark` +---------------------------------------- + +:program:`tcpdump` reports the traffic as follows:: + + 02:44:24.274569 IP 192.168.4.142.50540 > 184.168.176.1.443: Flags [S], seq 163388510, win 64240, options [mss 1460,nop,wscale 8,nop,nop,sackOK], length 0 + 02:44:24.339007 IP 184.168.176.1.443 > 192.168.4.142.50540: Flags [S.], seq 3902980842, ack 163388511, win 14600, options [mss 1460,nop,wscale 8], length 0 + 02:44:24.340486 IP 192.168.4.142.50540 > 184.168.176.1.443: Flags [.], ack 1, win 513, length 0 + 02:44:24.340668 IP 192.168.4.142.50540 > 184.168.176.1.443: Flags [P.], seq 1:518, ack 1, win 513, length 517 + 02:44:24.407539 IP 184.168.176.1.443 > 192.168.4.142.50540: Flags [.], ack 518, win 62, length 0 + 02:44:24.410681 IP 184.168.176.1.443 > 192.168.4.142.50540: Flags [P.], seq 1:468, ack 518, win 62, length 467 + 02:44:24.411048 IP 184.168.176.1.443 > 192.168.4.142.50540: Flags [F.], seq 468, ack 518, win 62, length 0 + 02:44:24.412575 IP 192.168.4.142.50540 > 184.168.176.1.443: Flags [.], ack 469, win 511, length 0 + 02:44:24.412857 IP 192.168.4.142.50540 > 184.168.176.1.443: Flags [P.], seq 518:525, ack 469, win 511, length 7 + 02:44:24.412860 IP 192.168.4.142.50540 > 184.168.176.1.443: Flags [F.], seq 525, ack 469, win 511, length 0 + 02:44:24.477936 IP 184.168.176.1.443 > 192.168.4.142.50540: Flags [.], ack 526, win 62, length 0 + +On the face of it, there does not appear to be anything unusual about this +traffic. It appears to be a brief session to TCP port 443. + +:program:`tshark` reports the traffic as follows: + +.. literal-emph:: + + 2 192.168.4.142 50540 184.168.176.1 443 TCP 66 50540 → 443 [SYN] Seq=0 Win=64240 Len=0 MSS=1460 WS=256 SACK_PERM=1 + 4 184.168.176.1 443 192.168.4.142 50540 TCP 62 443 → 50540 [SYN, ACK] Seq=0 Ack=1 Win=14600 Len=0 MSS=1460 WS=256 + 6 192.168.4.142 50540 184.168.176.1 443 TCP 60 50540 → 443 [ACK] Seq=1 Ack=1 Win=131328 Len=0 + 7 192.168.4.142 50540 184.168.176.1 443 TLSv1 571 Client Hello + 9 184.168.176.1 443 192.168.4.142 50540 TCP 60 443 → 50540 [ACK] Seq=1 Ack=518 Win=15872 Len=0 + **10 184.168.176.1 443 192.168.4.142 50540 HTTP 521 HTTP/1.1 400 Bad Request (text/html)** + 11 184.168.176.1 443 192.168.4.142 50540 TCP 60 443 → 50540 [FIN, ACK] Seq=468 Ack=518 Win=15872 Len=0 + 13 192.168.4.142 50540 184.168.176.1 443 TCP 60 50540 → 443 [ACK] Seq=518 Ack=469 Win=130816 Len=0 + 14 192.168.4.142 50540 184.168.176.1 443 TCP 61 50540 → 443 [PSH, ACK] Seq=518 Ack=469 Win=130816 Len=7 + 15 192.168.4.142 50540 184.168.176.1 443 TCP 60 50540 → 443 [FIN, ACK] Seq=525 Ack=469 Win=130816 Len=0 + 24 184.168.176.1 443 192.168.4.142 50540 TCP 60 443 → 50540 [ACK] Seq=469 Ack=526 Win=15872 Len=0 + +:program:`tshark` reveals something weird is happening here. Frame 10 shows +that :program:`tshark` decoded a plain-text HTTP message from port 443 TCP. +This should not be happening. A second look shows that the TLS session did not +appear to complete, as there is no response to the TLS client hello message. + +Here is frame 10 in detail. I passed :program:`tshark` the ``-x`` switch to +provide a hex and ASCII output at the end. + +.. literal-emph:: + + Frame 10: 521 bytes on wire (4168 bits), 521 bytes captured (4168 bits) + Encapsulation type: Ethernet (1) + Arrival Time: Dec 10, 2020 02:44:24.410681000 UTC + [Time shift for this packet: 0.000000000 seconds] + Epoch Time: 1607568264.410681000 seconds + [Time delta from previous captured frame: 0.003142000 seconds] + [Time delta from previous displayed frame: 0.003142000 seconds] + [Time since reference or first frame: 0.136113000 seconds] + Frame Number: 10 + Frame Length: 521 bytes (4168 bits) + Capture Length: 521 bytes (4168 bits) + [Frame is marked: False] + [Frame is ignored: False] + [Protocols in frame: eth:ethertype:ip:tcp:http:data-text-lines] + Ethernet II, Src: fc:ec:da:49:e0:10, Dst: 60:f2:62:3c:9c:68 + Destination: 60:f2:62:3c:9c:68 + Address: 60:f2:62:3c:9c:68 + .... ..0. .... .... .... .... = LG bit: Globally unique address (factory default) + .... ...0 .... .... .... .... = IG bit: Individual address (unicast) + Source: fc:ec:da:49:e0:10 + Address: fc:ec:da:49:e0:10 + .... ..0. .... .... .... .... = LG bit: Globally unique address (factory default) + .... ...0 .... .... .... .... = IG bit: Individual address (unicast) + Type: IPv4 (0x0800) + Internet Protocol Version 4, Src: 184.168.176.1, Dst: 192.168.4.142 + 0100 .... = Version: 4 + .... 0101 = Header Length: 20 bytes (5) + Differentiated Services Field: 0x00 (DSCP: CS0, ECN: Not-ECT) + 0000 00.. = Differentiated Services Codepoint: Default (0) + .... ..00 = Explicit Congestion Notification: Not ECN-Capable Transport (0) + Total Length: 507 + Identification: 0xcc4e (52302) + Flags: 0x4000, Don't fragment + 0... .... .... .... = Reserved bit: Not set + .1.. .... .... .... = Don't fragment: Set + ..0. .... .... .... = More fragments: Not set + ...0 0000 0000 0000 = Fragment offset: 0 + Time to live: 55 + Protocol: TCP (6) + Header checksum: 0x47ce [validation disabled] + [Header checksum status: Unverified] + Source: 184.168.176.1 + Destination: 192.168.4.142 + Transmission Control Protocol, Src Port: 443, Dst Port: 50540, Seq: 1, Ack: 518, Len: 467 + Source Port: 443 + Destination Port: 50540 + [Stream index: 1] + [TCP Segment Len: 467] + Sequence number: 1 (relative sequence number) + [Next sequence number: 468 (relative sequence number)] + Acknowledgment number: 518 (relative ack number) + 0101 .... = Header Length: 20 bytes (5) + Flags: 0x018 (PSH, ACK) + 000. .... .... = Reserved: Not set + ...0 .... .... = Nonce: Not set + .... 0... .... = Congestion Window Reduced (CWR): Not set + .... .0.. .... = ECN-Echo: Not set + .... ..0. .... = Urgent: Not set + .... ...1 .... = Acknowledgment: Set + .... .... 1... = Push: Set + .... .... .0.. = Reset: Not set + .... .... ..0. = Syn: Not set + .... .... ...0 = Fin: Not set + [TCP Flags: ·······AP···] + Window size value: 62 + [Calculated window size: 15872] + [Window size scaling factor: 256] + Checksum: 0xde95 [unverified] + [Checksum Status: Unverified] + Urgent pointer: 0 + [SEQ/ACK analysis] + [iRTT: 0.065917000 seconds] + [Bytes in flight: 467] + [Bytes sent since last PSH flag: 467] + [Timestamps] + [Time since first frame in this TCP stream: 0.136112000 seconds] + [Time since previous frame in this TCP stream: 0.003142000 seconds] + TCP payload (467 bytes) + **Hypertext Transfer Protocol** + **[Expert Info (Warning/Security): Unencrypted HTTP protocol detected over encrypted port, could indicate a dangerous misconfiguration.]** + **[Unencrypted HTTP protocol detected over encrypted port, could indicate a dangerous misconfiguration.]** + **[Severity level: Warning]** + **[Group: Security]** + **HTTP/1.1 400 Bad Request\r\n** + [Expert Info (Chat/Sequence): HTTP/1.1 400 Bad Request\r\n] + [HTTP/1.1 400 Bad Request\r\n] + [Severity level: Chat] + [Group: Sequence] + Response Version: HTTP/1.1 + Status Code: 400 + [Status Code Description: Bad Request] + Response Phrase: Bad Request + Date: Thu, 10 Dec 2020 02:44:24 GMT\r\n + Server: Apache\r\n + Content-Length: 301\r\n + [Content length: 301] + Connection: close\r\n + Content-Type: text/html; charset=iso-8859-1\r\n + \r\n + [HTTP response 1/1] + File Data: 301 bytes + Line-based text data: text/html (10 lines) + \n + \n + 400 Bad Request\n + \n +

    Bad Request

    \n +

    Your browser sent a request that this server could not understand.
    \n +

    \n +
    \n +
    Apache Server at virtualhost.184.168.176.1 Port 80
    \n + \n + + 0000 60 f2 62 3c 9c 68 fc ec da 49 e0 10 08 00 45 00 `.b<.h...I....E. + 0010 01 fb cc 4e 40 00 37 06 47 ce b8 a8 b0 01 c0 a8 ...N@.7.G....... + 0020 04 8e 01 bb c5 6c e8 a2 c2 eb 09 bd 1e 64 50 18 .....l.......dP. + 0030 00 3e de 95 00 00 **48 54 54** 50 2f 31 2e 31 20 34 .>....**HTT**P/1.1 4 + 0040 30 30 20 42 61 64 20 52 65 71 75 65 73 74 0d 0a 00 Bad Request.. + 0050 44 61 74 65 3a 20 54 68 75 2c 20 31 30 20 44 65 Date: Thu, 10 De + 0060 63 20 32 30 32 30 20 30 32 3a 34 34 3a 32 34 20 c 2020 02:44:24 + 0070 47 4d 54 0d 0a 53 65 72 76 65 72 3a 20 41 70 61 GMT..Server: Apa + 0080 63 68 65 0d 0a 43 6f 6e 74 65 6e 74 2d 4c 65 6e che..Content-Len + 0090 67 74 68 3a 20 33 30 31 0d 0a 43 6f 6e 6e 65 63 gth: 301..Connec + 00a0 74 69 6f 6e 3a 20 63 6c 6f 73 65 0d 0a 43 6f 6e tion: close..Con + 00b0 74 65 6e 74 2d 54 79 70 65 3a 20 74 65 78 74 2f tent-Type: text/ + 00c0 68 74 6d 6c 3b 20 63 68 61 72 73 65 74 3d 69 73 html; charset=is + 00d0 6f 2d 38 38 35 39 2d 31 0d 0a 0d 0a 3c 21 44 4f o-8859-1.....< + 0110 68 74 6d 6c 3e 3c 68 65 61 64 3e 0a 3c 74 69 74 html>.400 Bad Reque + 0130 73 74 3c 2f 74 69 74 6c 65 3e 0a 3c 2f 68 65 61 st..

    Bad + 0150 20 52 65 71 75 65 73 74 3c 2f 68 31 3e 0a 3c 70 Request

    .

    Your browser se + 0170 6e 74 20 61 20 72 65 71 75 65 73 74 20 74 68 61 nt a request tha + 0180 74 20 74 68 69 73 20 73 65 72 76 65 72 20 63 6f t this server co + 0190 75 6c 64 20 6e 6f 74 20 75 6e 64 65 72 73 74 61 uld not understa + 01a0 6e 64 2e 3c 62 72 20 2f 3e 0a 3c 2f 70 3e 0a 3c nd.
    .

    .< + 01b0 68 72 3e 0a 3c 61 64 64 72 65 73 73 3e 41 70 61 hr>.
    Apa + 01c0 63 68 65 20 53 65 72 76 65 72 20 61 74 20 76 69 che Server at vi + 01d0 72 74 75 61 6c 68 6f 73 74 2e 31 38 34 2e 31 36 rtualhost.184.16 + 01e0 38 2e 31 37 36 2e 31 20 50 6f 72 74 20 38 30 3c 8.176.1 Port 80< + 01f0 2f 61 64 64 72 65 73 73 3e 0a 3c 2f 62 6f 64 79 /address>.. + +You can see the HTTP headers and page content in the payload of this frame. I +bolded the hex and ASCII output for the ``HTT`` part of the HTTP header in the +payload. :program:`tshark` reports a warning as seen in the bolded output. + +:file:`conn.log` +---------------- + +Here is the :file:`conn.log` that Zeek generated for this activity: + +.. literal-emph:: + + { + "ts": 1607568264.274569, + **"uid": "C8blOJ21azairPrWf8",** + "id.orig_h": "192.168.4.142", + "id.orig_p": 50540, + "id.resp_h": "184.168.176.1", + "id.resp_p": 443, + "proto": "tcp", + "duration": 0.1382908821105957, + "orig_bytes": 524, + "resp_bytes": 467, + "conn_state": "SF", + "missed_bytes": 0, + "history": "ShADadfF", + "orig_pkts": 6, + "orig_ip_bytes": 776, + "resp_pkts": 5, + "resp_ip_bytes": 675, + "ip_proto": 6 + } + +The :file:`conn.log` entry is fairly normal. + +:file:`ssl.log` +--------------- + +Here is the :file:`ssl.log` that Zeek generated for this activity: + +.. literal-emph:: + + { + "ts": 1607568264.340668, + "uid": "C8blOJ21azairPrWf8", + "id.orig_h": "192.168.4.142", + "id.orig_p": 50540, + "id.resp_h": "184.168.176.1", + "id.resp_p": 443, + "server_name": "usafaikidonews.com", + "resumed": false, + **"established": false** + } + +The :file:`ssl.log` shows that a TLS encrypted session was not established. + +:file:`analyzer.log` +-------------------- + +Here is the :file:`analyzer.log` that Zeek generated for this activity: + +.. literal-emph:: + + { + "ts": 1607568264.410681, + "analyzer_kind": "protocol", + **"analyzer_name": "SSL",** + "uid": "C8blOJ21azairPrWf8", + "id.orig_h": "192.168.4.142", + "id.orig_p": 50540, + "id.resp_h": "184.168.176.1", + "id.resp_p": 443, + "proto": "tcp", + **"failure_reason": "Invalid version late in TLS connection. Packet reported version: 21588"** + } + +Here we see that DPD and the SSL analyzer report an error in the TLS +connection, as expected. The question is, to what does ``version: 21588`` +refer? + +Decoding 21588 +============== + +Let’s take a look at part of frame 9, which is the TLS client hello: + +.. literal-emph:: + + Secure Sockets Layer + TLSv1 Record Layer: Handshake Protocol: Client Hello + **Content Type: Handshake (22)** + **Version: TLS 1.0 (0x0301)** + Length: 512 + Handshake Protocol: Client Hello + Handshake Type: Client Hello (1) + Length: 508 + **Version: TLS 1.2 (0x0303)** + ...truncated... + + 0000 fc ec da 49 e0 10 60 f2 62 3c 9c 68 08 00 45 00 ...I..`.b<.h..E. + 0010 02 2d 97 6c 40 00 80 06 33 7e c0 a8 04 8e b8 a8 .-.l@...3~...... + 0020 b0 01 c5 6c 01 bb 09 bd 1c 5f e8 a2 c2 eb 50 18 ...l....._....P. + 0030 02 01 6e 33 00 00 **16 03 01** 02 00 01 00 01 fc **03** ..n3............ + 0040 **03** 97 16 82 4f e0 ff e3 3e 6f d8 33 28 9a 97 b8 ....O...>o.3(... + 0050 1a f0 73 6b 12 98 af 25 e2 a5 bc 6c 2e aa b1 69 ..sk...%...l...i + 0060 be 20 bf d4 27 c5 22 bf 0d 90 83 24 80 36 ad 11 . ..'."....$.6.. + 0070 17 8a 2d a2 a1 42 1d ef 6b 1f ef ce cf 9a e2 f5 ..-..B..k....... + 0080 be 79 00 20 2a 2a 13 01 13 02 13 03 c0 2b c0 2f .y. **.......+./ + 0090 c0 2c c0 30 cc a9 cc a8 c0 13 c0 14 00 9c 00 9d .,.0............ + 00a0 00 2f 00 35 01 00 01 93 ca ca 00 00 00 00 00 17 ./.5............ + 00b0 00 15 00 00 12 75 73 61 66 61 69 6b 69 64 6f 6e .....usafaikidon + 00c0 65 77 73 2e 63 6f 6d 00 17 00 00 ff 01 00 01 00 ews.com......... + +I’ve bolded a few points. The important ones are ``0x160301``. These are the +values indicating a TLS handshake and TLS 1.0. This is apparently not an +attempt at a TLS 1.0 connection, however, as the second bolded hex value of +``0x0303`` shows TLS 1.2 in play. + +Now, compare this output with what appeared in the odd “HTTP” frame shown +earlier: + +.. literal-emph:: + + 0000 60 f2 62 3c 9c 68 fc ec da 49 e0 10 08 00 45 00 `.b<.h...I....E. + 0010 01 fb cc 4e 40 00 37 06 47 ce b8 a8 b0 01 c0 a8 ...N@.7.G....... + 0020 04 8e 01 bb c5 6c e8 a2 c2 eb 09 bd 1e 64 50 18 .....l.......dP. + 0030 00 3e de 95 00 00 **48 54 54** 50 2f 31 2e 31 20 34 .>....**HTT**P/1.1 4 + 0040 30 30 20 42 61 64 20 52 65 71 75 65 73 74 0d 0a 00 Bad Request.. + +The ``0x48`` value is in the location where a TLS content type message would +sit. In the previous frame, the value was ``0x16``, for a handshake. Here it +is ``0x48``, which is ASCII letter H. Next we see ``0x5454``, which is ASCII +letters ``T T``. In decimal, the value for ``0x5454`` is 21588. In other words, +where Zeek was looking to find a TLS version, it found decimal 21588. In the +previous frame, the corresponding value was ``0x0301`` for TLSv1.0. That is why +Zeek generated an error in its :file:`analyzer.log` with the message "Invalid +version late in TLS connection. Packet reported version: 21588". + +Assorted Examples +================= + +The following represents a summary of some :file:`analyzer.log` entries, sorted by count, +observed in my reference network. + +.. code-block:: console + + $ find ./logs/ -name "analyzer*20**.gz" | while read -r file; do zcat -f "$file"; done | jq -c '[."proto", ."analyzer_kind", ."failure_reason"]' | sort | uniq -c | sort -nr + +:: + + 165341 ["tcp","HTTP","not a http reply line"] + 162 ["tcp","SSL","Invalid version late in TLS connection. Packet reported version: 0"] + 114 ["tcp","SSL","Invalid version late in TLS connection. Packet reported version: 21588"] + 36 ["tcp","SSL","Invalid version late in TLS connection. Packet reported version: 25344"] + 28 ["udp","NTP","Binpac exception: binpac exception: out_of_bound: Extension_Field:value: 3476019 > 52"] + 17 ["udp","SIP","Binpac exception: binpac exception: string mismatch at /bro/src/analyzer/protocol/sip/sip-protocol.pac:43: \nexpected pattern: \"SIP/\"\nactual data: \"\\x05\""] + 9 ["tcp","SSL","Invalid version late in TLS connection. Packet reported version: 8516"] + 8 ["udp","SIP","Binpac exception: binpac exception: string mismatch at /bro/src/analyzer/protocol/sip/sip-protocol.pac:43: \nexpected pattern: \"SIP/\"\nactual data: \"\\x01\""] + ...edited... + 1 ["udp","SIP","Binpac exception: binpac exception: out_of_bound: SIP_Version:anonymous_field_009: 4 > 2"] + 1 ["udp","DTLS","Invalid version in DTLS connection. Packet reported version: 59228"] + 1 ["udp","DTLS","Invalid version in DTLS connection. Packet reported version: 52736"] + 1 ["udp","DTLS","Invalid version in DTLS connection. Packet reported version: 52480"] + 1 ["tcp","SSL","Invalid version late in TLS connection. Packet reported version: 5123"] + 1 ["tcp","SSL","Invalid version late in TLS connection. Packet reported version: 40499"] + 1 ["tcp","IRC","too many long lines"] + +As you can see, Zeek saw problems with HTTP, SSL, NTP, Session Initiation +Protocol (SIP), Datagram Transport Layer Security (DTLS), and IRC. + +Conclusion +========== + +Zeek’s :file:`analyzer.log` may help analysts identify suspicious activity, +depending on how it violates Zeek’s protocol parsers. In that sense, it is sort +of a specialized version of Zeek’s :file:`weird.log`. Periodic analysis of the +entries may identify traffic worthy of additional investigation. diff --git a/doc/logs/capture-loss-and-reporter.rst b/doc/logs/capture-loss-and-reporter.rst new file mode 100644 index 0000000000..d946f02ba4 --- /dev/null +++ b/doc/logs/capture-loss-and-reporter.rst @@ -0,0 +1,98 @@ +================================= +capture_loss.log and reporter.log +================================= + +Zeek produces several logs that tell administrators how well Zeek is managing +its analysis and reporting on network traffic. + +This :file:`capture_loss.log` reports analysis of missing traffic. Zeek bases +its conclusions on analysis of TCP sequence numbers. When it detects a “gap,” +it assumes that the missing traffic corresponds to traffic loss. + +The :file:`reporter.log` reports internal warnings and errors. Zeek generates +these based on how it is handling traffic and computing requirements. + +Details on the format of each log appears in :zeek:see:`CaptureLoss::Info` +and :zeek:see:`Reporter::Info`. + +:file:`capture_loss.log` +======================== + +The following is an example of entries in a :file:`capture_loss.log`: + +.. literal-emph:: + + { + "ts": "2021-01-04T00:04:24.688236Z", + "ts_delta": 900.0000550746918, + "peer": "so16-enp0s8-1", + "gaps": 41, + "acks": 9944, + **"percent_lost": 0.412308930008045** + } + { + "ts": "2021-01-04T00:19:24.688265Z", + "ts_delta": 900.0000290870667, + "peer": "so16-enp0s8-1", + "gaps": 9, + "acks": 8530, + **"percent_lost": 0.10550996483001172** + } + { + "ts": "2021-01-04T00:34:24.688449Z", + "ts_delta": 900.0001838207245, + "peer": "so16-enp0s8-1", + "gaps": 0, + "acks": 52019, + **"percent_lost": 0** + } + { + "ts": "2021-01-04T00:49:24.688552Z", + "ts_delta": 900.0001029968262, + "peer": "so16-enp0s8-1", + "gaps": 0, + "acks": 108863, + **"percent_lost": 0** + } + +In these logs, capture loss never exceeded 1%. For example, when Zeek reports +``0.412308930008045``, that means 0.4123% capture loss, not 41.23% capture +loss. In other words, this sensor is doing well capturing the traffic on the +link it monitors (a small amount of loss is tolerable). + +:file:`reporter.log` +==================== + +The following is an example entries in the :file:`reporter.log`: + +.. literal-emph:: + + { + "ts": "2021-01-04T01:15:02.622164Z", + "level": "Reporter::INFO", + **"message": "received termination signal",** + "location": "" + } + { + "ts": "2021-01-04T01:19:15.713689Z", + "level": "Reporter::INFO", + **"message": "BPFConf filename set: /etc/nsm/so16-enp0s8/bpf-bro.conf (logger)",** + "location": "/opt/bro/share/zeek/securityonion/./bpfconf.zeek, line 81" + } + { + "ts": "2021-01-04T01:19:22.786812Z", + "level": "Reporter::INFO", + **"message": "BPFConf filename set: /etc/nsm/so16-enp0s8/bpf-bro.conf (proxy)",** + "location": "/opt/bro/share/zeek/securityonion/./bpfconf.zeek, line 81" + } + +The first message refers to Zeek receiving a termination signal. The second two +messages refer to Zeek setting a file for configuring Berkeley Packet Filters. + +Conclusion +========== + +The :file:`capture_loss.log` and :file:`reporter.log` files are helpful when +administrators need to understand how their Zeek deployment is performing. Keep +an eye on the :file:`capture_loss.log` to keep the performance within an +acceptable level. diff --git a/doc/logs/conn.rst b/doc/logs/conn.rst new file mode 100644 index 0000000000..6cb995807c --- /dev/null +++ b/doc/logs/conn.rst @@ -0,0 +1,444 @@ +======== +conn.log +======== + +The connection log, or :file:`conn.log`, is one of the most important logs Zeek +creates. It may seem like the idea of a “connection” is most closely associated +with stateful protocols like Transmission Control Protocol (TCP), unlike +stateless protocols like User Datagram Protocol (UDP). Zeek’s :file:`conn.log`, +however, tracks both sorts of protocols. This section of the manual will +explain key elements of the :file:`conn.log`. + +The Zeek script reference, derived from the Zeek code, completely explains the +meaning of each field in the :file:`conn.log` (and other logs). It would be +duplicative to manually recreate that information in another format here. +Therefore, this entry seeks to show how an analyst would make use of the +information in the conn.log. Those interested in getting details on every +element of the :file:`conn.log` should reference :zeek:see:`Conn::Info`. +For additional explanation, including Zeek's notions of originator and +responder, see :ref:`writing-scripts-connection-record`. + +Throughout the sections that follow, we will inspect Zeek logs in JSON format. + +Inspecting the :file:`conn.log` +=============================== + +To inspect the :file:`conn.log`, we will use the same techniques we learned in +the last section of the manual. First, we have a JSON-formatted log file, +either collected by Zeek watching a live interface, or by Zeek processing +stored traffic. We use the :program:`jq` utility to review the contents. + +.. code-block:: console + + zeek@zeek:~zeek-test/json$ jq . -c conn.log + +:: + + {"ts":1591367999.305988,"uid":"CMdzit1AMNsmfAIiQc","id.orig_h":"192.168.4.76","id.orig_p":36844,"id.resp_h":"192.168.4.1","id.resp_p":53,"proto":"udp","service":"dns","duration":0.06685185432434082,"orig_bytes":62,"resp_bytes":141,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":2,"orig_ip_bytes":118,"resp_pkts":2,"resp_ip_bytes":197,"ip_proto":17} + + {"ts":1591367999.430166,"uid":"C5bLoe2Mvxqhawzqqd","id.orig_h":"192.168.4.76","id.orig_p":46378,"id.resp_h":"31.3.245.133","id.resp_p":80,"proto":"tcp","service":"http","duration":0.25411510467529297,"orig_bytes":77,"resp_bytes":295,"conn_state":"SF","missed_bytes":0,"history":"ShADadFf","orig_pkts":6,"orig_ip_bytes":397,"resp_pkts":4,"resp_ip_bytes":511,"ip_proto":6} + +Alternatively, we could see each field printed on its own line: + +.. code-block:: console + + zeek@zeek:~zeek-test/json$ jq . conn.log + +:: + + { + "ts": 1591367999.305988, + "uid": "CMdzit1AMNsmfAIiQc", + "id.orig_h": "192.168.4.76", + "id.orig_p": 36844, + "id.resp_h": "192.168.4.1", + "id.resp_p": 53, + "proto": "udp", + "service": "dns", + "duration": 0.06685185432434082, + "orig_bytes": 62, + "resp_bytes": 141, + "conn_state": "SF", + "missed_bytes": 0, + "history": "Dd", + "orig_pkts": 2, + "orig_ip_bytes": 118, + "resp_pkts": 2, + "resp_ip_bytes": 197, + "ip_proto": 17 + } + { + "ts": 1591367999.430166, + "uid": "C5bLoe2Mvxqhawzqqd", + "id.orig_h": "192.168.4.76", + "id.orig_p": 46378, + "id.resp_h": "31.3.245.133", + "id.resp_p": 80, + "proto": "tcp", + "service": "http", + "duration": 0.25411510467529297, + "orig_bytes": 77, + "resp_bytes": 295, + "conn_state": "SF", + "missed_bytes": 0, + "history": "ShADadFf", + "orig_pkts": 6, + "orig_ip_bytes": 397, + "resp_pkts": 4, + "resp_ip_bytes": 511, + "ip_proto": 6 + } + +What an analyst derives from any log is a function of the questions that he or +she is trying to ask of it. The :file:`conn.log` primarily captures so-called +“layer 3” and “layer 4” elements of network activity. This is essentially who +is talking to whom, when, for how long, and with what protocol. + +Understanding the Second :file:`conn.log` Entry +=============================================== + +Let’s use this framework to parse the two log entries. We will start with the +second entry first. I will explain why shortly. For reference, that entry is +the following: + +:: + + { + "ts": 1591367999.430166, + "uid": "C5bLoe2Mvxqhawzqqd", + "id.orig_h": "192.168.4.76", + "id.orig_p": 46378, + "id.resp_h": "31.3.245.133", + "id.resp_p": 80, + "proto": "tcp", + "service": "http", + "duration": 0.25411510467529297, + "orig_bytes": 77, + "resp_bytes": 295, + "conn_state": "SF", + "missed_bytes": 0, + "history": "ShADadFf", + "orig_pkts": 6, + "orig_ip_bytes": 397, + "resp_pkts": 4, + "resp_ip_bytes": 511, + "ip_proto": 6 + } + +For the second log, ``192.168.4.76`` talked to ``31.3.245.133``. + +The log *timestamp*, indicated by the ``ts`` field, is ``1591367999.430166``, +which translates as shown below, courtesy of the Unix :program:`date` command: + +.. code-block:: console + + zeek@zeek:~zeek-test/json$ date -d @"1591367999.430166" + +:: + + Fri Jun 5 14:39:59 UTC 2020 + +The two systems conversation only lasted ``0.25411510467529297`` seconds. (The +operating system provides this value.) + +They spoke the HyperText Transfer Protocol (HTTP), identified by Zeek as HTTP +over TCP using TCP port 80 listening on ``31.3.245.133``. + +If we wanted to move beyond who talked with whom, when, for how long, and with +what protocol, the second :file:`conn.log` entry offers a few more items of interest. +For example, we know that ``192.168.4.76`` sent 77 bytes of data in its application +layer payload, and 397 bytes in its IP layer payload. + +We can verify that 77 byte figure by decoding the HTTP traffic sent from +``192.168.4.76`` during this session. We use :program:`tshark`, the command +line version of Wireshark, to do so. + +.. code-block:: console + + zeek@zeek:~zeek-test/json$ tshark -V -r ../../tmi1.pcap http and ip.src==192.168.4.76 + +.. literal-emph:: + + Frame 21: 143 bytes on wire (1144 bits), 143 bytes captured (1144 bits) + Encapsulation type: Ethernet (1) + Arrival Time: Jun 5, 2020 14:39:59.512593000 UTC + [Time shift for this packet: 0.000000000 seconds] + Epoch Time: 1591367999.512593000 seconds + [Time delta from previous captured frame: 0.000309000 seconds] + [Time delta from previous displayed frame: 0.000000000 seconds] + [Time since reference or first frame: 17.461008000 seconds] + Frame Number: 21 + Frame Length: 143 bytes (1144 bits) + Capture Length: 143 bytes (1144 bits) + [Frame is marked: False] + [Frame is ignored: False] + [Protocols in frame: eth:ethertype:ip:tcp:http] + Ethernet II, Src: 08:00:27:97:99:0d, Dst: fc:ec:da:49:e0:10 + Destination: fc:ec:da:49:e0:10 + Address: fc:ec:da:49:e0:10 + .... ..0. .... .... .... .... = LG bit: Globally unique address (factory default) + .... ...0 .... .... .... .... = IG bit: Individual address (unicast) + Source: 08:00:27:97:99:0d + Address: 08:00:27:97:99:0d + .... ..0. .... .... .... .... = LG bit: Globally unique address (factory default) + .... ...0 .... .... .... .... = IG bit: Individual address (unicast) + Type: IPv4 (0x0800) + Internet Protocol Version 4, **Src: 192.168.4.76, Dst: 31.3.245.133** + 0100 .... = Version: 4 + .... 0101 = Header Length: 20 bytes (5) + Differentiated Services Field: 0x00 (DSCP: CS0, ECN: Not-ECT) + 0000 00.. = Differentiated Services Codepoint: Default (0) + .... ..00 = Explicit Congestion Notification: Not ECN-Capable Transport (0) + Total Length: 129 + Identification: 0xfdf1 (65009) + Flags: 0x4000, Don't fragment + 0... .... .... .... = Reserved bit: Not set + .1.. .... .... .... = Don't fragment: Set + ..0. .... .... .... = More fragments: Not set + ...0 0000 0000 0000 = Fragment offset: 0 + Time to live: 64 + Protocol: TCP (6) + Header checksum: 0x6308 [validation disabled] + [Header checksum status: Unverified] + **Source: 192.168.4.76** + **Destination: 31.3.245.133** + Transmission Control Protocol, **Src Port: 46378, Dst Port: 80**, Seq: 1, Ack: 1, **Len: 77** + **Source Port: 46378** + **Destination Port: 80** + [Stream index: 0] + **[TCP Segment Len: 77]** + Sequence number: 1 (relative sequence number) + [Next sequence number: 78 (relative sequence number)] + Acknowledgment number: 1 (relative ack number) + 1000 .... = Header Length: 32 bytes (8) + Flags: 0x018 (PSH, ACK) + 000. .... .... = Reserved: Not set + ...0 .... .... = Nonce: Not set + .... 0... .... = Congestion Window Reduced (CWR): Not set + .... .0.. .... = ECN-Echo: Not set + .... ..0. .... = Urgent: Not set + .... ...1 .... = Acknowledgment: Set + .... .... 1... = Push: Set + .... .... .0.. = Reset: Not set + .... .... ..0. = Syn: Not set + .... .... ...0 = Fin: Not set + [TCP Flags: ·······AP···] + Window size value: 32 + [Calculated window size: 65536] + [Window size scaling factor: 2048] + Checksum: 0xd9f0 [unverified] + [Checksum Status: Unverified] + Urgent pointer: 0 + Options: (12 bytes), No-Operation (NOP), No-Operation (NOP), Timestamps + TCP Option - No-Operation (NOP) + Kind: No-Operation (1) + TCP Option - No-Operation (NOP) + Kind: No-Operation (1) + TCP Option - Timestamps: TSval 3137978878, TSecr 346747623 + Kind: Time Stamp Option (8) + Length: 10 + Timestamp value: 3137978878 + Timestamp echo reply: 346747623 + [SEQ/ACK analysis] + [iRTT: 0.082118000 seconds] + **[Bytes in flight: 77]** + [Bytes sent since last PSH flag: 77] + [Timestamps] + [Time since first frame in this TCP stream: 0.082427000 seconds] + [Time since previous frame in this TCP stream: 0.000309000 seconds] + **TCP payload (77 bytes)** + Hypertext Transfer Protocol + **GET / HTTP/1.1\r\n** + [Expert Info (Chat/Sequence): GET / HTTP/1.1\r\n] + [GET / HTTP/1.1\r\n] + [Severity level: Chat] + [Group: Sequence] + Request Method: GET + Request URI: / + Request Version: HTTP/1.1 + **Host: testmyids.com\r\n** + **User-Agent: curl/7.47.0\r\n** + **Accept: */*\r\n** + **\r\n** + [Full request URI: http://testmyids.com/] + [HTTP request 1/1] + +In the highlighted output, we see that :program:`tshark` notes 77 bytes of data +carried by TCP from ``192.168.4.76``. I highlighted what that data was, +beginning with a GET request. + +The ``orig_pkts`` and ``resp_pkts`` fields report the number of IP packets +transferred in the respective directions. The ``orig_ip_bytes`` and +``resp_ip_bytes`` indicate the total IP packet-level byte counts, respectively. + +Another way to look at this TCP segment is to dump the hex contents using a +different :program:`tshark` option, as shown below. + +.. code-block:: console + + zeek@zeek:~zeek-test/json$ tshark -x -r ../../tmi1.pcap http and ip.src==192.168.4.76 + +.. literal-emph:: + + 0000 fc ec da 49 e0 10 08 00 27 97 99 0d 08 00 45 00 ...I....'.....E. + 0010 00 81 fd f1 40 00 40 06 63 08 c0 a8 04 4c 1f 03 ....@.@.c....L.. + 0020 f5 85 b5 2a 00 50 dd e8 f3 47 b2 71 7e 69 80 18 ...*.P...G.q~i.. + 0030 00 20 d9 f0 00 00 01 01 08 0a bb 09 c1 fe 14 aa . .............. + 0040 f2 e7 **47 45 54 20 2f 20 48 54 54 50 2f 31 2e 31 ..GET / HTTP/1.1** + 0050 **0d 0a 48 6f 73 74 3a 20 74 65 73 74 6d 79 69 64 ..Host: testmyid** + 0060 **73 2e 63 6f 6d 0d 0a 55 73 65 72 2d 41 67 65 6e s.com..User-Agen** + 0070 **74 3a 20 63 75 72 6c 2f 37 2e 34 37 2e 30 0d 0a t: curl/7.47.0..** + 0080 **41 63 63 65 70 74 3a 20 2a 2f 2a 0d 0a 0d 0a Accept: */***.... + +The hexadecimal values appear on the left, and the ASCII decode appears on the +right. If you count the highlighted hex values, you will find 77 of them, hence +the 77 bytes of application layer data carried by TCP. + +The connection state field, ``conn_state``, showed that the connection +terminated normally, as depicted by the ``SF`` entry. This means that, for this +TCP session, both sides adopted a “graceful close” mechanism. If you remember +this trace from the last chapter, you’ll remember seeing that it opened with a +TCP three way handshake (SYN - SYN ACK - ACK) and terminated with a graceful +close (FIN ACK - FIN ACK - ACK). + +Finally, the ``history`` field contains the string ``ShADadFf``. Remember that +capitalized letters indicate an action by the connection originator. Lowercase +letters indicate an action by the responder. This means that ``ShADadFf`` +translates to the following: + +:: + + S - The originator sent a SYN segment. + h - The responder sent a SYN ACK segment. + A - The originator sent an ACK segment. + D - The originator sent at least one segment with payload data. In this case, that was HTTP over TCP. + a - The responder replied with an ACK segment. + d - The responder replied with at least one segment with payload data. + F - The originator sent a FIN ACK segment. + f - The responder replied with a FIN ACK segment. + +This log entry demonstrates how Zeek is able to pack so much information into a +compact representation. + +Understanding the First :file:`conn.log` Entry +============================================== + +Now let’s turn to the first :file:`conn.log` entry, reproduced below for easy +reference. + +:: + + { + "ts": 1591367999.305988, + "uid": "CMdzit1AMNsmfAIiQc", + "id.orig_h": "192.168.4.76", + "id.orig_p": 36844, + "id.resp_h": "192.168.4.1", + "id.resp_p": 53, + "proto": "udp", + "service": "dns", + "duration": 0.06685185432434082, + "orig_bytes": 62, + "resp_bytes": 141, + "conn_state": "SF", + "missed_bytes": 0, + "history": "Dd", + "orig_pkts": 2, + "orig_ip_bytes": 118, + "resp_pkts": 2, + "resp_ip_bytes": 197, + "ip_proto": 17 + } + +For the first entry, ``192.168.4.76`` talked to ``192.168.4.1``. + +The log timestamp is ``1591367999.305988``, which translates as shown below, +courtesy of the Unix :program:`date` command: + +.. code-block:: console + + zeek@zeek:~zeek-test/json$ date -d @"1591367999.305988" + +:: + + Fri Jun 5 14:39:59 UTC 2020 + +The two systems’ “conversation” only lasted ``0.06685185432434082`` seconds. +(Again, such precision!) + +They spoke the Domain Name System (DNS) protocol, identified by Zeek as DNS +over UDP using UDP port 53 listening on ``192.168.4.1``. + +The connection state for this conversation is listed as ``SF``, the same as the +TCP version. However, UDP has no concept of state, leaving that duty to a +higher level protocol. In the context of UDP, ``SF`` means that Zeek assesses +the conversations as “normal establishment and termination” of the +“connection.” + +Similarly, the ``history`` field is simply ``Dd``, indicating that each party +to the conversation sent data to the other. + +The ``ip_proto`` Field +====================== + +.. versionadded:: 7.1 + +The numeric ``ip_proto`` field reports the `IP protocol number +`_ of +the connection. It relates to the ``proto`` field, but while the former +represents a :zeek:type:`transport_proto` value that exclusively covers +*transport* protocols Zeek knows how to parse (and ties into Zeek's +:zeek:type:`port` type), the ``ip_proto`` field is always present, including for +non-transport IP packet flows such as IGMP or OSPF. For example, an OSPF flow +might look as follows: + +:: + + { + "ts": 1098361214.420459, + "uid": "C9EV8R4fN8bfSj08f", + "id.orig_h": "192.168.170.2", + "id.orig_p": 0, + "id.resp_h": "224.0.0.6", + "id.resp_p": 0, + "proto": "unknown_transport", + "duration": 6.437546968460083, + "orig_bytes": 0, + "resp_bytes": 0, + "conn_state": "OTH", + "local_orig": true, + "local_resp": false, + "missed_bytes": 0, + "orig_pkts": 4, + "orig_ip_bytes": 768, + "resp_pkts": 0, + "resp_ip_bytes": 0, + "ip_proto": 89 + } + +You can adapt this feature in several ways. Load the +:doc:`/scripts/policy/protocols/conn/ip-proto-name-logging.zeek` policy script +to add an ``ip_proto_name`` column with a string version of the ``ip_proto`` +value. Also, you may disable the whole feature by loading the +:doc:`/scripts/policy/protocols/conn/disable-unknown-ip-proto-support.zeek` +script, returning :file:`conn.log` to its pre-7.1 state. Zeek's :ref:`logging framework +` supports additional customizations. + +The ``uid`` and Other Fields +============================ + +Notice that both :file:`conn.log` entries contain ``uid`` fields. These are +unique identifiers assigned by Zeek that we will use to track related activity +in other transaction logs. + +There are other fields which may appear in the :file:`conn.log`, depending on +the protocol being summarized. For details on the meaning of those fields, see +:zeek:see:`Conn::Info`. + +Conclusion +========== + +Zeek’s :file:`conn.log` is a foundational log that offers a great deal of +information on its own. However, it becomes even more useful when it acts as +the starting point for investigating related Zeek logs. We turn to that +capability in the following sections. diff --git a/doc/logs/dhcp.rst b/doc/logs/dhcp.rst new file mode 100644 index 0000000000..f64d2d7492 --- /dev/null +++ b/doc/logs/dhcp.rst @@ -0,0 +1,439 @@ +======== +dhcp.log +======== + +Dynamic Host Configuration Protocol is a core protocol found in Internet +Protocol (IP) networks. Using the protocol, DHCP servers provide clients with +IP addresses and other key information needed to make use of the network. This +entry will describe some aspects of Zeek’s :file:`dhcp.log` that may be of use to +network and security personnel. + +As with all entries in this chapter, see :zeek:see:`DHCP::Info` for full +explanation of each field in the log. + +DORA via Tcpdump +================ + +The method by which a client requests and receives an IP address and other +parameters from a DHCP server is represented by the acronym DORA. DORA stands +for Discover - Offer - Request - Acknowledge. The following :program:`tcpdump` +output of a complete DORA exchange demonstrates this protocol in action. + +.. code-block:: console + + $ tcpdump -n -r snort.log.1601610971.bootp.pcap + +.. literal-emph:: + + reading from file snort.log.1601610971.bootp.pcap, link-type EN10MB (Ethernet) + + 04:14:39.119370 IP **0.0.0.0.68 > 255.255.255.255.67**: BOOTP/DHCP, Request from 3c:58:c2:2f:91:21, length 302 + 04:14:39.120138 IP **192.168.4.1.67 > 192.168.4.152.68**: BOOTP/DHCP, Reply, length 302 + 04:14:39.158211 IP **0.0.0.0.68 > 255.255.255.255.67**: BOOTP/DHCP, Request from 3c:58:c2:2f:91:21, length 337 + 04:14:39.456915 IP **192.168.4.1.67 > 192.168.4.152.68**: BOOTP/DHCP, Reply, length 302 + +The default output for :program:`tcpdump` doesn’t say much, other than showing +the IP addresses (or lack thereof, in the case of the ``0.0.0.0``` source IP +addresses). It is helpful to see this “simplified” output, however, before +delving into the details. It is slightly deceptive in the “request” and “reply” +messages, as strictly speaking these are more detailed and are DORA messages. + +DORA via Tcpdump Verbose Mode +============================= + +We can add the ``-vvv`` flag to :program:`tcpdump` to provide more verbose +output, as shown in the examples that follow. + +The first datagram shows that a host that does not have an IP address set +(i.e., it’s using ``0.0.0.0``) sends a broadcast to ``255.255.255.255`` on port +67 UDP. This client has had an IP address before as shown by its request for +``192.168.4.152``. Note the hostname and the presence of a Microsoft 5.0 vendor +class. + +This is a DHCP Discover message from a client to any DHCP server listening on +the local network: + +.. literal-emph:: + + 04:14:39.119370 IP (tos 0x0, ttl 128, id 44414, offset 0, flags [none], **proto UDP** (17), length 330) + **0.0.0.0.68 > 255.255.255.255.67**: [udp sum ok] BOOTP/DHCP, Request from 3c:58:c2:2f:91:21, length 302, **xid 0xfd9859a7**, Flags [none] (0x0000) + **Client-Ethernet-Address 3c:58:c2:2f:91:21** + Vendor-rfc1048 Extensions + Magic Cookie 0x63825363 + **DHCP-Message Option 53, length 1: Discover** + Client-ID Option 61, length 7: ether 3c:58:c2:2f:91:21 + **Requested-IP Option 50, length 4: 192.168.4.152** + **Hostname Option 12, length 15: "3071N0098017422"** + **Vendor-Class Option 60, length 8: "MSFT 5.0"** + Parameter-Request Option 55, length 14: + Subnet-Mask, Default-Gateway, Domain-Name-Server, Domain-Name + Router-Discovery, Static-Route, Vendor-Option, Netbios-Name-Server + Netbios-Node, Netbios-Scope, Option 119, Classless-Static-Route + Classless-Static-Route-Microsoft, Option 252 + END Option 255, length 0 + +The second datagram is a reply from the local DHCP server running on +``192.168.4.1``. The server replies directly to ``192.168.4.152``, which in +this case will end up at the system using MAC address ``3c:58:c2:2f:91:21``, +such that the destination IP address is probably not relevant here. Remember +that if the client at MAC address ``3c:58:c2:2f:91:21`` had no IP address to +begin with, it would only receive the DHCP offer by virtue of the DHCP offer +datagram being addressed to its MAC address. The server is not offering a +specified domain name other than “localdomain.” + +This is a DHCP Offer message, from the DHCP server to the client: + +.. literal-emph:: + + 04:14:39.120138 IP (tos 0x10, ttl 128, id 0, offset 0, flags [none], proto UDP (17), length 330) + **192.168.4.1.67 > 192.168.4.152.68**: [udp sum ok] **BOOTP/DHCP, Reply**, length 302, **xid 0xfd9859a7**, Flags [none] (0x0000) + **Your-IP 192.168.4.152** + **Client-Ethernet-Address 3c:58:c2:2f:91:21** + Vendor-rfc1048 Extensions + Magic Cookie 0x63825363 + **DHCP-Message Option 53, length 1: Offer** + **Server-ID Option 54, length 4: 192.168.4.1** + **Lease-Time Option 51, length 4: 86400** + **Subnet-Mask Option 1, length 4: 255.255.255.0** + **Default-Gateway Option 3, length 4: 192.168.4.1** + **Domain-Name-Server Option 6, length 4: 192.168.4.1** + Domain-Name Option 15, length 11: "localdomain" + T119 Option 119, length 13: 11.108.111.99.97.108.100.111.109.97.105.110.0 + END Option 255, length 0 + +The third datagram is a reply to the server’s reply. Here the client requests +the IP address ``192.168.4.152``. We also see it provide a fully qualified +domain name (FQDN) for itself, belonging to the FCPS educational domain. Again +note the client does not include an IP address for itself in the layer 3 +header. It uses ``0.0.0.0`` as in the initial Discover message. + +This is a DHCP Request message from the client to the DHCP server: + +.. literal-emph:: + + 04:14:39.158211 IP (tos 0x0, ttl 128, id 44415, offset 0, flags [none], proto UDP (17), length 365) + **0.0.0.0.68 > 255.255.255.255.67**: [udp sum ok] **BOOTP/DHCP, Request from 3c:58:c2:2f:91:21**, length 337, **xid 0xfd9859a7**, Flags [none] (0x0000) + **Client-Ethernet-Address 3c:58:c2:2f:91:21** + Vendor-rfc1048 Extensions + Magic Cookie 0x63825363 + **DHCP-Message Option 53, length 1: Request** + **Client-ID Option 61, length 7: ether 3c:58:c2:2f:91:21** + **Requested-IP Option 50, length 4: 192.168.4.152** + **Server-ID Option 54, length 4: 192.168.4.1** + Hostname Option 12, length 15: "3071N0098017422" + **FQDN Option 81, length 27: "3071N0098017422.fcps.edu"** + **Vendor-Class Option 60, length 8: "MSFT 5.0"** + Parameter-Request Option 55, length 14: + Subnet-Mask, Default-Gateway, Domain-Name-Server, Domain-Name + Router-Discovery, Static-Route, Vendor-Option, Netbios-Name-Server + Netbios-Node, Netbios-Scope, Option 119, Classless-Static-Route + Classless-Static-Route-Microsoft, Option 252 + END Option 255, length 0 + +Finally the server sends its last message, essentially confirming the +information sent in the DHCP Offer message. Note that :program:`tcpdump` is +unable to make sense of what it renders as ``T119 Option 119``. We will return +to that shortly. + +This is a DHCP Acknowledgement message, sent from the DHCP server to the client: + +.. literal-emph:: + + 04:14:39.456915 IP (tos 0x10, ttl 128, id 0, offset 0, flags [none], proto UDP (17), length 330) + **192.168.4.1.67 > 192.168.4.152.68**: [udp sum ok] **BOOTP/DHCP, Reply**, length 302, xid 0xfd9859a7, Flags [none] (0x0000) + **Your-IP 192.168.4.152** + **Client-Ethernet-Address 3c:58:c2:2f:91:21** + Vendor-rfc1048 Extensions + Magic Cookie 0x63825363 + **DHCP-Message Option 53, length 1: ACK** + **Server-ID Option 54, length 4: 192.168.4.1** + **Lease-Time Option 51, length 4: 86400** + **Subnet-Mask Option 1, length 4: 255.255.255.0** + **Default-Gateway Option 3, length 4: 192.168.4.1** + **Domain-Name-Server Option 6, length 4: 192.168.4.1** + Domain-Name Option 15, length 11: "localdomain" + T119 Option 119, length 13: 11.108.111.99.97.108.100.111.109.97.105.110.0 + END Option 255, length 0 + +Acknowledgement via :program:`tshark` +===================================== + +We could look at the entire trace using :program:`tshark` (the command line +version of Wireshark), but it would largely be redundant. Rather, I would like +to look at the Acknowledgment message to explain about the T119 Option that +:program:`tcpdump` could not decode. + +To find the datagram of interest, I tell :program:`tshark` to read the packet +capture of interest. I tell it to look for the “bootp” transaction identifier +associated with the DORA exchange of interest. (BOOTP refers to Bootstrap, a +precursor protocol that Tshark still uses for DHCP filters.) I also tell +:program:`tshark` to look for the specific BOOTP (DHCP) option value (5) +associated with the ACK message. + +.. code-block:: console + + $ tshark -V -n -r snort.log.1601610971.bootp.pcap bootp.id == 0xfd9859a7 and bootp.option.dhcp == 5 + +.. literal-emph:: + + Frame 4: 344 bytes on wire (2752 bits), 344 bytes captured (2752 bits) on interface 0 + Interface id: 0 (unknown) + Interface name: unknown + Encapsulation type: Ethernet (1) + Arrival Time: Oct 2, 2020 04:14:39.456915000 UTC + [Time shift for this packet: 0.000000000 seconds] + Epoch Time: 1601612079.456915000 seconds + [Time delta from previous captured frame: 0.298704000 seconds] + [Time delta from previous displayed frame: 0.000000000 seconds] + [Time since reference or first frame: 0.337545000 seconds] + Frame Number: 4 + Frame Length: 344 bytes (2752 bits) + Capture Length: 344 bytes (2752 bits) + [Frame is marked: False] + [Frame is ignored: False] + [Protocols in frame: eth:ethertype:ip:udp:bootp] + **Ethernet II, Src: fc:ec:da:49:e0:10, Dst: 3c:58:c2:2f:91:21** + Destination: 3c:58:c2:2f:91:21 + Address: 3c:58:c2:2f:91:21 + .... ..0. .... .... .... .... = LG bit: Globally unique address (factory default) + .... ...0 .... .... .... .... = IG bit: Individual address (unicast) + Source: fc:ec:da:49:e0:10 + Address: fc:ec:da:49:e0:10 + .... ..0. .... .... .... .... = LG bit: Globally unique address (factory default) + .... ...0 .... .... .... .... = IG bit: Individual address (unicast) + Type: IPv4 (0x0800) + **Internet Protocol Version 4, Src: 192.168.4.1, Dst: 192.168.4.152** + 0100 .... = Version: 4 + .... 0101 = Header Length: 20 bytes (5) + Differentiated Services Field: 0x10 (DSCP: Unknown, ECN: Not-ECT) + 0001 00.. = Differentiated Services Codepoint: Unknown (4) + .... ..00 = Explicit Congestion Notification: Not ECN-Capable Transport (0) + Total Length: 330 + Identification: 0x0000 (0) + Flags: 0x0000 + 0... .... .... .... = Reserved bit: Not set + .0.. .... .... .... = Don't fragment: Not set + ..0. .... .... .... = More fragments: Not set + ...0 0000 0000 0000 = Fragment offset: 0 + Time to live: 128 + Protocol: UDP (17) + Header checksum: 0xafa9 [validation disabled] + [Header checksum status: Unverified] + Source: 192.168.4.1 + Destination: 192.168.4.152 + **User Datagram Protocol, Src Port: 67, Dst Port: 68** + Source Port: 67 + Destination Port: 68 + Length: 310 + Checksum: 0x92db [unverified] + [Checksum Status: Unverified] + [Stream index: 1] + **Bootstrap Protocol (ACK)** + Message type: Boot Reply (2) + Hardware type: Ethernet (0x01) + Hardware address length: 6 + Hops: 0 + **Transaction ID: 0xfd9859a7** + Seconds elapsed: 0 + Bootp flags: 0x0000 (Unicast) + 0... .... .... .... = Broadcast flag: Unicast + .000 0000 0000 0000 = Reserved flags: 0x0000 + Client IP address: 0.0.0.0 + **Your (client) IP address: 192.168.4.152** + Next server IP address: 0.0.0.0 + Relay agent IP address: 0.0.0.0 + **Client MAC address: 3c:58:c2:2f:91:21** + Client hardware address padding: 00000000000000000000 + Server host name not given + Boot file name not given + Magic cookie: DHCP + **Option: (53) DHCP Message Type (ACK)** + Length: 1 + **DHCP: ACK (5)** + Option: (54) DHCP Server Identifier + Length: 4 + **DHCP Server Identifier: 192.168.4.1** + Option: (51) IP Address Lease Time + Length: 4 + IP Address Lease Time: (86400s) 1 day + Option: (1) Subnet Mask + Length: 4 + **Subnet Mask: 255.255.255.0** + Option: (3) Router + Length: 4 + **Router: 192.168.4.1** + Option: (6) Domain Name Server + Length: 4 + **Domain Name Server: 192.168.4.1** + Option: (15) Domain Name + Length: 11 + Domain Name: localdomain + **Option: (119) Domain Search** + **Length: 13** + **FQDN: localdomain** + Option: (255) End + Option End: 255 + +This output looks similar to what :program:`tcpdump` reported, except here we +can see the decode for Option 119. It looks like the DHCP server is providing +the FQDN of “localdomain.” + +Zeek’s Rendition of DORA +======================== + +With this background, let’s look at Zeek’s depiction of this DHCP exchange. + +:: + + { + "ts": "2020-10-02T04:14:39.135304Z", + "uids": [ + "COoA8M1gbTowuPlVT", + "CapFoX32zVg3R6TATc" + ], + "client_addr": "192.168.4.152", + "server_addr": "192.168.4.1", + "mac": "3c:58:c2:2f:91:21", + "host_name": "3071N0098017422", + "client_fqdn": "3071N0098017422.fcps.edu", + "domain": "localdomain", + "requested_addr": "192.168.4.152", + "assigned_addr": "192.168.4.152", + "lease_time": 86400, + "msg_types": [ + "DISCOVER", + "OFFER", + "REQUEST", + "ACK" + ], + "duration": 0.416348934173584 + } + +As you can see, Zeek has taken the important elements from all four DORA +messages and produced a single log entry. Every field is interesting, so I did +not highlight them all. + +Two UIDs +======== + +You might be wondering why there are two UID fields for this single DHCP +exchange. Let’s look at the two corresponding :file:`conn.log` entries. + +The first one shows a “conversation” between ``0.0.0.0`` and ``255.255.255.0``. +This represents the DHCP Discover message, caused by a client not knowing its +source IP address, sending its search to the local network for a DHCP server. + +.. literal-emph:: + + { + "ts": "2020-10-02T04:14:14.443346Z", + "uid": "COoA8M1gbTowuPlVT", + **"id.orig_h": "0.0.0.0",** + **"id.orig_p": 68,** + **"id.resp_h": "255.255.255.255",** + **"id.resp_p": 67,** + "proto": "udp", + "service": "dhcp", + "duration": 63.16645097732544, + "orig_bytes": 1211, + "resp_bytes": 0, + "conn_state": "S0", + "local_orig": false, + "local_resp": false, + "missed_bytes": 0, + "history": "D", + **"orig_pkts": 4,** + "orig_ip_bytes": 1323, + "resp_pkts": 0, + "resp_ip_bytes": 0, + "ip_proto": 17 + } + +Notice that Zeek has tracked 4 “orig packets” here, which does not strictly +correspond to the 2 datagrams from ``0.0.0.0`` to ``255.255.255.255``. Remember +the DORA via :program:`tcpdump` output? + +It’s possible Zeek included other packets involving ``0.0.0.0`` and +``255.255.255.255`` when it created this log entry since this is a broadcast +and Zeek generally may trouble with that because it doesn't fit the +"connection" abstraction. + +The second message shows a conversation between ``192.168.4.152``, the DHCP +client, and ``192.168.4.1``, the DHCP server. + +.. literal-emph:: + + { + "ts": "2020-10-02T04:14:39.120138Z", + "uid": "CapFoX32zVg3R6TATc", + **"id.orig_h": "192.168.4.152",** + **"id.orig_p": 68,** + **"id.resp_h": "192.168.4.1",** + **"id.resp_p": 67,** + "proto": "udp", + "service": "dhcp", + "duration": 0.3367769718170166, + "orig_bytes": 0, + "resp_bytes": 604, + "conn_state": "SHR", + "local_orig": true, + "local_resp": true, + "missed_bytes": 0, + "history": "^d", + "orig_pkts": 0, + "orig_ip_bytes": 0, + "resp_pkts": 2, + "resp_ip_bytes": 660, + "ip_proto": 17 + } + +Here the count of 2 ``resp_pkts`` is correct. + +Enumerating DHCP Servers +======================== + +Analysts can use Zeek’s :file:`dhcp.log` to enumerate systems providing DHCP +services. Consider the output of the following query. + +.. code-block:: console + + $ find . -name "dhcp**.gz" | while read -r file; do zcat -f "$file"; done | jq -c '[."server_addr"]' | sort | uniq -c | sort -nr | head -10 + +:: + + 1337 [null] + 119 ["192.168.4.1"] + +Here we see that ``192.168.4.1`` is providing DHCP services on this network. +The null entries refer to DHCP log entries that do not have a ``server_addr`` +field. One example is Zeek’s log for this DHCP Discover message: + +.. literal-emph:: + + { + "ts": "2020-10-06T23:59:48.577749Z", + "uids": [ + "CctZMx18mIK1qj9Vci" + ], + "mac": "80:ee:73:52:eb:59", + "host_name": "ds61", + "msg_types": [ + **"DISCOVER"** + ], + "duration": 0 + } + +This log entry does not have a ``server_addr`` field, so the query above returns a null result. + +Conclusion +========== + +DHCP is crucial to the proper operation of any IP network. DHCP logs help +analysts map IP addresses to MAC addresses, and may also reveal hostnames. When +investigating suspicious or malicious activity, analysts need to know what +system was assigned what IP address, as DHCP leases expire. However, depending +on the network, systems may retain specific IP addresses for a long time as +they may request an old address as was seen in this example. Of course, +administrators who have configured DHCP to provide fixed IP addresses based on +MAC address will ensure that these machines receive the same IP address, +despite relying on the “dynamic” nature of DHCP. diff --git a/doc/logs/dns.rst b/doc/logs/dns.rst new file mode 100644 index 0000000000..ee670b7061 --- /dev/null +++ b/doc/logs/dns.rst @@ -0,0 +1,267 @@ +======= +dns.log +======= + +The Domain Name System (DNS) log, or :file:`dns.log`, is one of the most +important data sources generated by Zeek. Although recent developments in +domain name resolution have challenged traditional methods for collecting DNS +data, :file:`dns.log` remains a powerful tool for security and network +administrators. + +Those interested in getting details on every element of the :file:`dns.log` +should refer to :zeek:see:`DNS::Info`. + +Throughout the sections that follow, we will inspect Zeek logs in JSON format. + +Inspecting the :file:`dns.log` +============================== + +To inspect the :file:`dns.log`, we will use the same techniques we learned +earlier in the manual. First, we have a JSON-formatted log file, either +collected by Zeek watching a live interface, or by Zeek processing stored +traffic. We use the :program:`jq` utility to review the contents. + +.. code-block:: console + + zeek@zeek:~/zeek-test/json$ jq . -c dns.log + +:: + + {"ts":1591367999.306059,"uid":"CMdzit1AMNsmfAIiQc","id.orig_h":"192.168.4.76","id.orig_p":36844,"id.resp_h":"192.168.4.1","id.resp_p":53,"proto":"udp","trans_id":8555,"query":"testmyids.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} + + {"ts":1591367999.305988,"uid":"CMdzit1AMNsmfAIiQc","id.orig_h":"192.168.4.76","id.orig_p":36844,"id.resp_h":"192.168.4.1","id.resp_p":53,"proto":"udp","trans_id":19671,"rtt":0.06685185432434082,"query":"testmyids.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["31.3.245.133"],"TTLs":[3600],"rejected":false} + +As before, we could see each field printed on its own line: + +.. code-block:: console + + zeek@zeek:~/zeek-test/json$ jq . dns.log + +:: + + { + "ts": 1591367999.306059, + "uid": "CMdzit1AMNsmfAIiQc", + "id.orig_h": "192.168.4.76", + "id.orig_p": 36844, + "id.resp_h": "192.168.4.1", + "id.resp_p": 53, + "proto": "udp", + "trans_id": 8555, + "query": "testmyids.com", + "qclass": 1, + "qclass_name": "C_INTERNET", + "qtype": 28, + "qtype_name": "AAAA", + "rcode": 0, + "rcode_name": "NOERROR", + "AA": false, + "TC": false, + "RD": true, + "RA": false, + "Z": 0, + "rejected": false + } + { + "ts": 1591367999.305988, + "uid": "CMdzit1AMNsmfAIiQc", + "id.orig_h": "192.168.4.76", + "id.orig_p": 36844, + "id.resp_h": "192.168.4.1", + "id.resp_p": 53, + "proto": "udp", + "trans_id": 19671, + "rtt": 0.06685185432434082, + "query": "testmyids.com", + "qclass": 1, + "qclass_name": "C_INTERNET", + "qtype": 1, + "qtype_name": "A", + "rcode": 0, + "rcode_name": "NOERROR", + "AA": false, + "TC": false, + "RD": true, + "RA": true, + "Z": 0, + "answers": [ + "31.3.245.133" + ], + "TTLs": [ + 3600 + ], + "rejected": false + } + +As emphasized in the :file:`conn.log` material, what an analyst derives from +any log is a function of the questions that he or she is trying to ask of it. +The :file:`dns.log` captures application-level name resolution activity, +assuming that traffic is not encrypted, as is the case with DNS over HTTPS +(DoH) or DNS over TLS (DoT). Applications mainly use DNS to resolve names to IP +addresses, IP addresses to names, and certain other functions. Intruders use +DNS for the same purposes, but may also subvert the protocol to carry +command-and-control traffic, obfuscated or encrypted payload data, or other +unwanted functions. DNS is a suitable protocol for these nefarious activities +because administrators tend to allow it throughout their purview, as it is +necessary for normal network operation. + +In brief, when looking at the :file:`dns.log`, analysts will primarily want to +know who is asking a question, what is the nature of the question, who answered +the question, and how was the question answered. + +Understanding the Second :file:`dns.log` Entry +============================================== + +Let’s use this framework to parse the two log entries. We will start with the +second entry. For reference, that entry is the following: + +:: + + { + "ts": 1591367999.305988, + "uid": "CMdzit1AMNsmfAIiQc", + "id.orig_h": "192.168.4.76", + "id.orig_p": 36844, + "id.resp_h": "192.168.4.1", + "id.resp_p": 53, + "proto": "udp", + "trans_id": 19671, + "rtt": 0.06685185432434082, + "query": "testmyids.com", + "qclass": 1, + "qclass_name": "C_INTERNET", + "qtype": 1, + "qtype_name": "A", + "rcode": 0, + "rcode_name": "NOERROR", + "AA": false, + "TC": false, + "RD": true, + "RA": true, + "Z": 0, + "answers": [ + "31.3.245.133" + ], + "TTLs": [ + 3600 + ], + "rejected": false + } + +According to this log entry, ``192.168.4.76`` asked ``192.168.4.1`` for the A +record of the host ``testmyids.com``, and received the answer ``31.3.245.133``. +There are more details in the log, but those are the key elements an analyst +should be able to extract. + +Understanding the First :file:`dns.log` Entry +============================================= + +Let’s take a look at the first :file:`dns.log` entry. For reference, that entry +is the following: + +:: + + { + "ts": 1591367999.306059, + "uid": "CMdzit1AMNsmfAIiQc", + "id.orig_h": "192.168.4.76", + "id.orig_p": 36844, + "id.resp_h": "192.168.4.1", + "id.resp_p": 53, + "proto": "udp", + "trans_id": 8555, + "query": "testmyids.com", + "qclass": 1, + "qclass_name": "C_INTERNET", + "qtype": 28, + "qtype_name": "AAAA", + "rcode": 0, + "rcode_name": "NOERROR", + "AA": false, + "TC": false, + "RD": true, + "RA": false, + "Z": 0, + "rejected": false + } + +According to this log entry, ``192.168.4.76`` asked ``192.168.4.1`` for the +AAAA record of the host ``testmyids.com``, and did not receive an answer. + +This is technically true, but it is not the whole story. If we augment stock +Zeek with an additional script available from the project, we get a bit more +information. + +Specifically, we can enable a new script, +:doc:`/scripts/policy/protocols/dns/auth-addl.zeek`. + +We can invoke the script using this syntax: + +.. code-block:: console + + zeek@zeek:~/zeek-test/json2$ zeek -C LogAscii::use_json=T protocols/dns/auth-addl.zeek -r ../tm1t.pcap + +The end result shows more information for the first :file:`dns.log` entry: + +.. code-block:: console + + zeek@zeek:~/zeek-test/json2$ cat dns.log | head -1 + +.. literal-emph:: + + {"ts":1591367999.306059,"uid":"CQsafSKqmlOyqrgC6","id.orig_h":"192.168.4.76","id.orig_p":36844,"id.resp_h":"192.168.4.1","id.resp_p":53,"proto":"udp","trans_id":8555,"query":"testmyids.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false,**"auth":["ns59.1and1.co.uk"]**} + +The bolded ``auth`` item in the log entry shows that ``ns59.1and1.co.uk`` is the +authoritative name server that is designated to answer questions about the AAAA +record for ``testmyids.com``. + +There are more details in the log, but those are the key elements an analyst +should be able to extract. + +The ``uid`` and Other Fields +============================ + +Note the ``uid`` field in both log entries is ``CMdzit1AMNsmfAIiQc``. This is +the same UID value that appeared in the :file:`conn.log` entry for a DNS +record. That means the DNS activity in the :file:`conn.log` and the DNS +activity in this :file:`dns.log` entry are the same. + +You could have used the UID in the :file:`conn.log` to search for the +corresponding records in the :file:`dns.log` using this UID. For example: + +.. code-block:: console + + zeek@zeek:~/zeek-test/json$ grep CMdzit1AMNsmfAIiQc dns.log + +:: + + {"ts":1591367999.306059,"uid":"CMdzit1AMNsmfAIiQc","id.orig_h":"192.168.4.76","id.orig_p":36844,"id.resp_h":"192.168.4.1","id.resp_p":53,"proto":"udp","trans_id":8555,"query":"testmyids.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} + + {"ts":1591367999.305988,"uid":"CMdzit1AMNsmfAIiQc","id.orig_h":"192.168.4.76","id.orig_p":36844,"id.resp_h":"192.168.4.1","id.resp_p":53,"proto":"udp","trans_id":19671,"rtt":0.06685185432434082,"query":"testmyids.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["31.3.245.133"],"TTLs":[3600.0],"rejected":false} + +Note the matching ``uid`` fields in the :file:`dns.log` entries. In this simple +example, these are the only two entries in the :file:`dns.log`. Extrapolate +this technique to logs with billions of records and you will appreciate the +value! + +Remember that a single :file:`conn.log` entry summarized all of the DNS traffic +associate with the “connection” bearing UID ``CMdzit1AMNsmfAIiQc``. Zeek +treated the 4 packets associated with this conversation as a connection because +they shared the same source and destination IP addresses and ports, and +occurred over the UDP protocol. The single :file:`conn.log` entry had the +timestamp ``1591367999.305988``, which is also the timestamp of the first +:file:`dns.log` entry. + +Zeek’s DNS protocol analyzer created two log entries because it recognized two +different DNS exchanges. The first involved a query and response for +IPv6-related information, i.e., a AAAA record for ``testmyids.com``. The second +involved a query and response for IPv4-related information, i.e., an A record +for ``testmyids.com``. It is interesting to note that the DNS resolver on the +``192.168.4.76`` system requested IPv6 information first, and then IPv4. + +Conclusion +========== + +Zeek’s :file:`dns.log` is a critical log that offers a great deal of +information on how systems are interacting with the Internet and each other. In +the next section we will look at other core Internet protocols. diff --git a/doc/logs/files.rst b/doc/logs/files.rst new file mode 100644 index 0000000000..be625d9ac0 --- /dev/null +++ b/doc/logs/files.rst @@ -0,0 +1,371 @@ +========= +files.log +========= + +One of Zeek’s powerful features is the ability to extract content from network +traffic and write it to disk as a file, via its +:ref:`File Analysis framework `. This is easiest to understand with a +protocol like File Transfer Protocol (FTP), a classic means to exchange files +over a channel separate from that used to exchange commands. Protocols like +HTTP are slightly more complicated, as it includes headers which must be +interpreted and not included in any file content transferred by the protocol. + +Zeek’s :file:`files.log` is a record of files that Zeek observed while +inspecting network traffic. The existence of an entry in :file:`files.log` does +not mean that Zeek necessarily extracted file content and wrote it to disk. +Analysts must configure Zeek to extract files by type in order to have them +written to disk. + +In the following example, an analyst has configured Zeek to extract files of +MIME type ``application/x-dosexec`` and write them to disk. To understand the +chain of events that result in having a file on disk, we will start with the +:file:`conn.log`, progress to the :file:`http.log`, and conclude with the +:file:`files.log`. + +The Zeek scripting manual, derived from the Zeek source code, completely +explains the meaning of each field in the :file:`files.log` (and other logs). +It would be duplicative to manually recreate that information in another format +here. Therefore, this entry seeks to show how an analyst would make use of the +information in the :file:`files.log`. Those interested in getting details on +every element of the :file:`files.log` should refer to :zeek:see:`Files::Info`. + +Throughout the sections that follow, we will inspect Zeek logs in JSON format. +As we have shown how to access logs like this previously using the command +line, we will only show the log entries themselves. + +Inspecting the :file:`conn.log` +=============================== + +The log with which we begin our analysis for this case is the :file:`conn.log`. +It contains the following entry of interest. + +.. literal-emph:: + + { + "ts": 1596820191.94147, + **"uid": "CzoFRWTQ6YIzfFXHk"**, + **"id.orig_h": "192.168.4.37",** + "id.orig_p": 58264, + **"id.resp_h": "23.195.64.241",** + **"id.resp_p": 80,** + "proto": "tcp", + **"service": "http",** + "duration": 0.050640106201171875, + "orig_bytes": 211, + "resp_bytes": 179604, + "conn_state": "SF", + "missed_bytes": 0, + "history": "ShADadtFf", + "orig_pkts": 93, + "orig_ip_bytes": 5091, + "resp_pkts": 129, + "resp_ip_bytes": 186320 + } + +We see that ``192.168.4.37`` contacted ``23.195.64.241`` via HTTP and connected +to port 80 TCP. The responder sent 179604 bytes of data during the +conversation. + +Because this conversation appears to have taken place using HTTP, a clear text +protocol, there is a good chance that we can directly inspect the HTTP headers +and the payloads that were exchanged. + +We will use the UID, ``CzoFRWTQ6YIzfFXHk``, to find corresponding entries in +other log sources to better understand what happened during this conversation. + +Inspecting the :file:`http.log` +=============================== + +We search our :file:`http.log` files for samples containing the UID of interest +and find the following entry: + +.. literal-emph:: + + { + "ts": 1596820191.94812, + **"uid": "CzoFRWTQ6YIzfFXHk",** + "id.orig_h": "192.168.4.37", + "id.orig_p": 58264, + "id.resp_h": "23.195.64.241", + "id.resp_p": 80, + "trans_depth": 1, + **"method": "GET",** + **"host": "download.microsoft.com",** + **"uri": "/download/d/e/5/de5351d6-4463-4cc3-a27c-3e2274263c43/wfetch.exe",** + "version": "1.1", + **"user_agent": "Wget/1.19.4 (linux-gnu)",** + "request_body_len": 0, + "response_body_len": 179272, + **"status_code": 200,** + **"status_msg": "OK",** + "tags": [], + "resp_fuids": [ + **"FBbQxG1GXLXgmWhbk9"** + ], + "resp_mime_types": [ + **"application/x-dosexec"** + ] + } + +The most interesting elements of this log entry include the following:: + + "method": "GET", + "host": "download.microsoft.com", + "uri": "/download/d/e/5/de5351d6-4463-4cc3-a27c-3e2274263c43/wfetch.exe", + +This shows us what file the client was trying to retrieve, ``wfetch.exe``, +from what site, ``download.microsoft.com``. + +The following element shows us the client that made the request:: + + "user_agent": "Wget/1.19.4 (linux-gnu)", + +According to this log entry, the user agent was not a Microsoft product, but +was a Linux version of the :program:`wget` utility. User agent fields can be +manipulated, so we cannot trust that this was exactly what happened. It is +probable however that :program:`wget` was used in this case. + +The following entry shows us that the Web server responding positively to the +request:: + + "status_code": 200, + "status_msg": "OK", + +Based on this entry and the amount of bytes transferred, it is likely that the +client received the file it requested. + +The final two entries of interest tell us something more about the content that +was transferred and how to locate it:: + + "resp_fuids": [ + "FBbQxG1GXLXgmWhbk9" + ], + "resp_mime_types": [ + "application/x-dosexec" + +The first entry provides a file identifier. This is similar to the connection +identifier in the :file:`conn.log`, except that we use the file identifier to +locate specific file contents when written to disk. + +The second entry shows that Zeek recognized the file content as +``application/x-dosexec``, which likely means that the client retrieved a +Windows executable file. + +Inspecting the :file:`files.log` +================================ + +Armed with the file identifier value, we can search any of our +:file:`files.log` repositories for matching values. By searching for the FUID +of ``FBbQxG1GXLXgmWhbk9`` we find the following entry. + +.. literal-emph:: + + { + "ts": 1596820191.969902, + **"fuid": "FBbQxG1GXLXgmWhbk9",** + "uid": "CzoFRWTQ6YIzfFXHk", + "id.orig_h": "192.168.4.37", + "id.orig_p": 58264, + "id.resp_h": "23.195.64.241", + "id.resp_p": 80, + "source": "HTTP", + "depth": 0, + "analyzers": [ + "EXTRACT", + "PE" + ], + **"mime_type": "application/x-dosexec",** + "duration": 0.015498876571655273, + "is_orig": false, + "seen_bytes": 179272, + "total_bytes": 179272, + "missing_bytes": 0, + "overflow_bytes": 0, + "timedout": false, + **"extracted": "HTTP-FBbQxG1GXLXgmWhbk9.exe",** + "extracted_cutoff": false + } + +Note that this :file:`files.log` entry also contains the UID we found in the +:file:`conn.log`, e.g., ``CzoFRWTQ6YIzfFXHk``. Theoretically we could have just +searched for that UID value and not bothered to locate the FUID in the +:file:`http.log`. However, I find that it makes sense to follow this sort of +progression, as we cannot rely on this same analytical workflow for all cases. + +In this :file:`files.log` data, we see that the ``EXTRACT`` and ``PE`` analyzer +events were activated. Zeek saw 179272 bytes transferred and does not appear to +have missed any bytes. Zeek extracted the file it saw as +``HTTP-FBbQxG1GXLXgmWhbk9.exe``, which means we should be able to locate that +file on disk. + +The ``is_orig`` field in a :file:`files.log` entry can be used to determine +which endpoint sent the file. When ``is_orig`` is ``false``, the responder of +the connection is sending the file. In the example above we can tell that +the HTTP server at ``23.195.64.241`` is sending the file and ``192.168.4.37`` +is receiving it. + +Inspecting the Extracted File +============================= + +The location for extracted files will vary depending on your Zeek +configuration. In my example, Zeek wrote extracted files to a directory called +:file:`extract_files/`. Here is the file in question: + +.. code-block:: console + + $ ls -al HTTP-FBbQxG1GXLXgmWhbk9.exe + +:: + + -rw-rw-r-- 1 zeek zeek 179272 Aug 7 17:23 HTTP-FBbQxG1GXLXgmWhbk9.exe + +Note the byte count, 179272, matches the value in the :file:`files.log`. + +Here is what the Linux file command thinks of this file. + +.. code-block:: console + + $ file HTTP-FBbQxG1GXLXgmWhbk9.exe + +:: + + HTTP-FBbQxG1GXLXgmWhbk9.exe: PE32 executable (GUI) Intel 80386, for MS Windows, MS CAB-Installer self-extracting archive + +This looks like a Windows executable. You can use the :program:`md5sum` utility to +generate a MD5 hash of the file. + +.. code-block:: console + + $ md5sum HTTP-FBbQxG1GXLXgmWhbk9.exe + +:: + + 6711727adf76599bf50c9426057a35fe HTTP-FBbQxG1GXLXgmWhbk9.exe + +We can search by the hash value on VirusTotal using the :program:`vt` command +line tool, provided we have registered and initialized :program:`vt` with our +free API key. + +.. code-block:: console + + $ ./vt file 6711727adf76599bf50c9426057a35fe + +:: + + - _id: "82f39086658ce80df4da6a49fef9d3062a00fd5795a4dd5042de32907bcb5b89" + _type: "file" + authentihash: "2a07d356273d32bf0c5aff83ea847351128fc3971b44052f92b6fb4f45c2272f" + creation_date: 1030609542 # 2002-08-29 08:25:42 +0000 UTC + first_submission_date: 1354191312 # 2012-11-29 12:15:12 +0000 UTC + last_analysis_date: 1592215708 # 2020-06-15 10:08:28 +0000 UTC + last_analysis_results: + ALYac: + category: "undetected" + engine_name: "ALYac" + engine_update: "20200615" + engine_version: "1.1.1.5" + method: "blacklist" + ...edited… + last_analysis_stats: + confirmed-timeout: 0 + failure: 0 + harmless: 0 + malicious: 0 + suspicious: 0 + timeout: 0 + type-unsupported: 2 + undetected: 74 + last_modification_date: 1592220693 # 2020-06-15 11:31:33 +0000 UTC + last_submission_date: 1539056691 # 2018-10-09 03:44:51 +0000 UTC + magic: "PE32 executable for MS Windows (GUI) Intel 80386 32-bit" + md5: "6711727adf76599bf50c9426057a35fe" + meaningful_name: "WEXTRACT.EXE" + names: + - "Wextract" + - "WEXTRACT.EXE" + - "wfetch.exe" + - "583526" + packers: + F-PROT: "CAB, ZIP" + PEiD: "Microsoft Visual C++ v6.0 SPx" + pe_info: + entry_point: 23268 + imphash: "1494de9b53e05fc1f40cb92afbdd6ce4" + import_list: + - imported_functions: + - "GetLastError" + - "IsDBCSLeadByte" + - "DosDateTimeToFileTime" + - "ReadFile" + - "GetStartupInfoA" + - "GetSystemInfo" + - "lstrlenA" + ...edited... + size: 179272 + ssdeep: "3072:BydJq5oyVzs+h0Jk5irDStDD5QOsP0CLRQq8ZZ3xlf/AQnFlFuKIUaKJH:UW2+AiDWOsPxQq8HHf/A07namH" + tags: + - "invalid-signature" + - "peexe" + - "signed" + - "overlay" + times_submitted: 33 + total_votes: + harmless: 1 + malicious: 0 + trid: + - file_type: "Microsoft Update - Self Extracting Cabinet" + probability: 46.3 + - file_type: "Win32 MS Cabinet Self-Extractor (WExtract stub)" + probability: 41.4 + - file_type: "Win32 Executable MS Visual C++ (generic)" + probability: 4.2 + - file_type: "Win64 Executable (generic)" + probability: 3.7 + - file_type: "Win16 NE executable (generic)" + probability: 1.9 + type_description: "Win32 EXE" + type_tag: "peexe" + unique_sources: 24 + vhash: " size: 179272 + ssdeep: "3072:BydJq5oyVzs+h0Jk5irDStDD5QOsP0CLRQq8ZZ3xlf/AQnFlFuKIUaKJH:UW2+AiDWOsPxQq8HHf/A07namH" + tags: + - "invalid-signature" + - "peexe" + - "signed" + - "overlay" + times_submitted: 33 + total_votes: + harmless: 1 + malicious: 0 + trid: + - file_type: "Microsoft Update - Self Extracting Cabinet" + probability: 46.3 + - file_type: "Win32 MS Cabinet Self-Extractor (WExtract stub)" + probability: 41.4 + - file_type: "Win32 Executable MS Visual C++ (generic)" + probability: 4.2 + - file_type: "Win64 Executable (generic)" + probability: 3.7 + - file_type: "Win16 NE executable (generic)" + probability: 1.9 + type_description: "Win32 EXE" + type_tag: "peexe" + unique_sources: 24 + vhash: "0150366d1570e013z1004cmz1f03dz" + +You can access the entire report `via the Web here +`_. + +It appears this is a harmless Windows executable. However, by virtue of having +it extracted from network traffic, analysts have many options for investigation +when the file is not considered benign. + +Conclusion +========== + +Zeek’s file extraction capabilities offer many advantages to analysts. +Administrators can configure Zeek to compute MD5 hashes of files that Zeek sees +in network traffic. Rather than computing a hash on a file written to disk, +Zeek could simply compute the hash as part of its inspection process. The +purpose of this section was to show some of the data in the :file:`files.log`, +how it relates to other Zeek logs, and how analysts might make use of it. diff --git a/doc/logs/ftp.rst b/doc/logs/ftp.rst new file mode 100644 index 0000000000..ec4ad6c69e --- /dev/null +++ b/doc/logs/ftp.rst @@ -0,0 +1,313 @@ +======= +ftp.log +======= + +Zeek’s :file:`ftp.log` summarizes activity using the File Transfer Protocol +(FTP). Similar to the :file:`http.log`, :file:`ftp.log` captures the essential +information an analyst would likely need to understand how a client and server +interact using FTP. + +FTP is an interesting protocol in the sense that it uses one TCP connection as +a control channel and a second TCP connection as a file transfer channel. The +control channel usually involves a FTP server listening on port 21 TCP. The +file transfer channel, however, depends on the choices made by the client and +server. With “passive FTP,” the server advertises a second TCP port to which +the client should connect, and the client connects to that TCP port to initiate +the file transfer. With “active FTP,” the server connects to a TCP port +advertised by the client, although the server uses a source port of 20 TCP. It +is more common to see passive FTP on the Internet today due to middleboxes +(such as firewalls or other filtering devices) interfering with active FTP +connections inbound to clients. + +For full details on each field in the :file:`ftp.log` file, please refer to +:zeek:see:`FTP::Info`. + +Finding the :file:`ftp.log` +=========================== + +In the following example, an analyst knows to look for Zeek logs on a specific +day bearing a specific UID. They search in the specified directory using the +:program:`zgrep` command and pipe the results to the Unix command +:program:`sed`, removing characters prior to the ``.gz:`` that would appear in +the output. This facilitates piping the results into the :program:`jq` utility +for easier viewing. + +.. code-block:: console + + $ zgrep "CLkXf2CMo11hD8FQ5" 2020-08-16/* | sed 's/.*gz://' | jq . + +:: + + { + "_path": "conn", + "_system_name": "ds61", + "_write_ts": "2020-08-16T06:26:10.266225Z", + "_node": "worker-01", + "ts": "2020-08-16T06:26:01.485394Z", + "uid": "CLkXf2CMo11hD8FQ5", + "id.orig_h": "192.168.4.76", + "id.orig_p": 53380, + "id.resp_h": "196.216.2.24", + "id.resp_p": 21, + "proto": "tcp", + "service": "ftp", + "duration": 3.780829906463623, + "orig_bytes": 184, + "resp_bytes": 451, + "conn_state": "SF", + "local_orig": true, + "local_resp": false, + "missed_bytes": 0, + "history": "ShAdDafF", + "orig_pkts": 20, + "orig_ip_bytes": 1232, + "resp_pkts": 17, + "resp_ip_bytes": 1343, + "ip_proto": 6, + "community_id": "1:lEESxqaSVYqFZvWNb4OccTa9sTs=" + } + { + "_path": "ftp", + "_system_name": "ds61", + "_write_ts": "2020-08-16T06:26:04.077276Z", + "_node": "worker-01", + "ts": "2020-08-16T06:26:03.553287Z", + "uid": "CLkXf2CMo11hD8FQ5", + "id.orig_h": "192.168.4.76", + "id.orig_p": 53380, + "id.resp_h": "196.216.2.24", + "id.resp_p": 21, + "user": "anonymous", + "password": "ftp@example.com", + "command": "EPSV", + "reply_code": 229, + "reply_msg": "Entering Extended Passive Mode (|||31746|).", + "data_channel.passive": true, + "data_channel.orig_h": "192.168.4.76", + "data_channel.resp_h": "196.216.2.24", + "data_channel.resp_p": 31746 + } + { + "_path": "ftp", + "_system_name": "ds61", + "_write_ts": "2020-08-16T06:26:05.117287Z", + "_node": "worker-01", + "ts": "2020-08-16T06:26:04.597290Z", + "uid": "CLkXf2CMo11hD8FQ5", + "id.orig_h": "192.168.4.76", + "id.orig_p": 53380, + "id.resp_h": "196.216.2.24", + "id.resp_p": 21, + "user": "anonymous", + "password": "ftp@example.com", + "command": "RETR", + "arg": "ftp://196.216.2.24/pub/stats/afrinic/delegated-afrinic-extended-latest.md5", + "file_size": 74, + "reply_code": 226, + "reply_msg": "Transfer complete.", + "fuid": "FueF95uKPrUuDnMc4" + } + +This output presents three log files. The first is a :file:`conn.log` entry for +the FTP control channel connection involving port 21 TCP. The second two +describe what happened during the FTP control channel. + +Before looking at the details, let’s see a reconstruction of the FTP control +channel. + +Reconstructing the FTP Control Channel +====================================== + +In the following example, we use the :program:`tcpflow` program introduced in +the :file:`http.log` section to reconstruct the FTP control channel. By using +the ``-c`` option, we can tell :program:`tcpflow`` to interleave the traffic +sent by both sides of the conversation. I pass it the port 53380 parameter to +be sure I reconstruct traffic involving that connection, which was the source +port for the FTP client. (If I chose something like 21 TCP instead, I could +have reconstructed numerous FTP sessions beyond the one in question here.) + +In this example, ``196.216.2.24`` is the FTP server, and ``192.168.4.76`` is +the FTP client. + +After the first two entries, I have manually edited the output for readability. + +.. code-block:: console + + $ tcpflow -c -r snort.log.1597554100-196.216.2.24.pcap port 53380 + +.. literal-emph:: + + 196.216.002.024.00021-192.168.004.076.53380 [**server** to client]: 220 ::::: Welcome to the AFRINIC FTP service :::::: + + 192.168.004.076.53380-196.216.002.024.00021 [**client** to server]: USER anonymous + + server: 331 Please specify the password. + + client: PASS ftp@example.com + + server: 230 Login successful. + + client: PWD + + server: 257 "/" + + client: CWD pub + + server: 250 Directory successfully changed. + + client: CWD stats + + server: 250 Directory successfully changed. + + client: CWD afrinic + + server: 250 Directory successfully changed. + + client: EPSV + + server: 229 Entering Extended Passive Mode (|||31746|). + + client: TYPE I + + server: 200 Switching to Binary mode. + + client: SIZE delegated-afrinic-extended-latest.md5 + + server: 213 74 + + client: RETR delegated-afrinic-extended-latest.md5 + + server: 150 Opening BINARY mode data connection for delegated-afrinic-extended-latest.md5 (74 bytes). + + server: 226 Transfer complete. + + client: QUIT + + server: 221 Goodbye. + +Reading this transcript, some important items include the following: + +* This is a FTP server that allows anonymous access. +* The data channel occurs using passive FTP. +* The FTP server opens port 31746 TCP to accept the FTP connection over which + it will transfer the requested file. +* The file transferred is ``delegated-afrinic-extended-latest.md5``, a 74 byte + file. + +With this understanding in place, let’s see how Zeek represents this activity. + +Inspecting the :file:`ftp.log` +============================== + +Let’s take a second look at the two :file:`ftp.log` entries. + +:: + + { + "_path": "ftp", + "_system_name": "ds61", + "_write_ts": "2020-08-16T06:26:04.077276Z", + "_node": "worker-01", + "ts": "2020-08-16T06:26:03.553287Z", + "uid": "CLkXf2CMo11hD8FQ5", + "id.orig_h": "192.168.4.76", + "id.orig_p": 53380, + "id.resp_h": "196.216.2.24", + "id.resp_p": 21, + "user": "anonymous", + "password": "ftp@example.com", + "command": "EPSV", + "reply_code": 229, + "reply_msg": "Entering Extended Passive Mode (|||31746|).", + "data_channel.passive": true, + "data_channel.orig_h": "192.168.4.76", + "data_channel.resp_h": "196.216.2.24", + "data_channel.resp_p": 31746 + } + +The first :file:`ftp.log` entry shows us that the FTP client logged in as user +``ftp@example.com``, requested a form of passive connection for its data +channel, and the server offered port 31746 TCP for that connection. + +:: + + { + "_path": "ftp", + "_system_name": "ds61", + "_write_ts": "2020-08-16T06:26:05.117287Z", + "_node": "worker-01", + "ts": "2020-08-16T06:26:04.597290Z", + "uid": "CLkXf2CMo11hD8FQ5", + "id.orig_h": "192.168.4.76", + "id.orig_p": 53380, + "id.resp_h": "196.216.2.24", + "id.resp_p": 21, + "user": "anonymous", + "password": "ftp@example.com", + "command": "RETR", + "arg": "ftp://196.216.2.24/pub/stats/afrinic/delegated-afrinic-extended-latest.md5", + "file_size": 74, + "reply_code": 226, + "reply_msg": "Transfer complete.", + "fuid": "FueF95uKPrUuDnMc4" + } + +The second :file:`ftp.log` entry gives details on the file retrieved from the +FTP server, such as the path on the server, its name, and the fact that the +file transfer completed. We also have a file identifier (``FueF95uKPrUuDnMc4``) +that we could use to find the file on disk, if we configured Zeek to extract +and save this sort of content. + +Finding the Data Channel +======================== + +For the sake of completeness, let’s take a look at the FTP data channel using +port 31746 TCP as our guide. I grep for the port number and the TCP protocol to +try to be more specific, although I could have added the source and destination +IP addresses too. + +.. code-block:: console + + $ zcat 2020-08-16/conn_20200816_06\:00\:00-07\:00\:00+0000.log.gz | grep 31746 | grep tcp | sed 's/.*gz://' | jq . + +:: + + { + "_path": "conn", + "_system_name": "ds61", + "_write_ts": "2020-08-16T06:26:09.771034Z", + "_node": "worker-01", + "ts": "2020-08-16T06:26:03.774520Z", + "uid": "CzLMFA3Eh8KBlY4kS7", + "id.orig_h": "192.168.4.76", + "id.orig_p": 60474, + "id.resp_h": "196.216.2.24", + "id.resp_p": 31746, + "proto": "tcp", + "service": "ftp-data", + "duration": 0.9965000152587891, + "orig_bytes": 0, + "resp_bytes": 74, + "conn_state": "SF", + "local_orig": true, + "local_resp": false, + "missed_bytes": 0, + "history": "ShAdfFa", + "orig_pkts": 4, + "orig_ip_bytes": 216, + "resp_pkts": 4, + "resp_ip_bytes": 290, + "ip_proto": 6, + "community_id": "1:DNwvGR6Ots6pISvsdXBUIaG8y3Q=" + } + +Zeek notes that this is a ``ftp-data`` service, which is another way we could +have used to find this connection. + +Conclusion +========== + +FTP is still in use, despite the fact that encrypted alternatives abound. +Zeek’s :file:`ftp.log` provides a compact way to summarize the salient features +of a FTP control channel, pointing out details of the control activity and how +to locate the data channel. diff --git a/doc/logs/http.rst b/doc/logs/http.rst new file mode 100644 index 0000000000..4a5e99cf71 --- /dev/null +++ b/doc/logs/http.rst @@ -0,0 +1,174 @@ +======== +http.log +======== + +The HyperText Transfer Protocol (HTTP) log, or :file:`http.log`, is another +core data source generated by Zeek. With the transition from clear-text HTTP to +encrypted HTTPS traffic, the :file:`http.log` is less active in many +environments. In some cases, however, organizations implement technologies or +practices to expose HTTPS as HTTP. Whether you’re looking at legacy HTTP on the +wire, or HTTPS that has been exposed as HTTP, Zeek’s :file:`http.log` offers +utility for examining normal, suspicious, and malicious activity. + +The Zeek scripting manual, derived from the Zeek source code, completely +explains the meaning of each field in the :file:`http.log` (and other logs). It +would be duplicative to manually recreate that information in another format +here. Therefore, this entry seeks to show how an analyst would make use of the +information in the :file:`http.log`. Those interested in getting details on +every element of the :file:`http.log` should refer to :zeek:see:`HTTP::Info`. + +Throughout the sections that follow, we will inspect Zeek logs in JSON format. + +Inspecting the :file:`http.log` +=============================== + +To inspect the :file:`http.log`, we will use the same techniques we learned +earlier in the manual. First, we have a JSON-formatted log file, either +collected by Zeek watching a live interface, or by Zeek processing stored +traffic. We use the :program:`jq` utility to review the contents. + +.. code-block:: console + + zeek@zeek:~/zeek-test/json$ jq . -c http.log + +:: + + {"ts":1591367999.512593,"uid":"C5bLoe2Mvxqhawzqqd","id.orig_h":"192.168.4.76","id.orig_p":46378,"id.resp_h":"31.3.245.133","id.resp_p":80,"trans_depth":1,"method":"GET","host":"testmyids.com","uri":"/","version":"1.1","user_agent":"curl/7.47.0","request_body_len":0,"response_body_len":39,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FEEsZS1w0Z0VJIb5x4"],"resp_mime_types":["text/plain"]} + +This is a very simple :file:`http.log`. With only one entry, it’s the simplest +possible entry. As before, we could see each field printed on its own line: + +.. code-block:: console + + zeek@zeek:~/zeek-test/json$ jq . http.log + +:: + + { + "ts": 1591367999.512593, + "uid": "C5bLoe2Mvxqhawzqqd", + "id.orig_h": "192.168.4.76", + "id.orig_p": 46378, + "id.resp_h": "31.3.245.133", + "id.resp_p": 80, + "trans_depth": 1, + "method": "GET", + "host": "testmyids.com", + "uri": "/", + "version": "1.1", + "user_agent": "curl/7.47.0", + "request_body_len": 0, + "response_body_len": 39, + "status_code": 200, + "status_msg": "OK", + "tags": [], + "resp_fuids": [ + "FEEsZS1w0Z0VJIb5x4" + ], + "resp_mime_types": [ + "text/plain" + ] + } + +HTTP is a protocol that was initially fairly simple. Over time it has become +increasingly complicated. It’s not the purpose of this manual to describe how +HTTP can be used and abused. Rather, we will take a brief look at the most +important elements of this :file:`http.log` entry, which is almost all of them. + +Understanding the :file:`http.log` Entry +======================================== + +Similar to the previous :file:`dns.log`, the :file:`http.log` is helpful +because it combines elements from the conversation between the source and +destination in one log entry. The most fundamental elements of the log answer +questions concerning who made a request, who responded, and the nature of the +request and response. + +In this entry, we see that ``192.168.4.76`` made a request to ``31.3.245.133``. +The originator made a HTTP version 1.1 GET request for the ``/`` or root of the +site ``testmyids.com`` hosted by the responder, passing a user agent of +``curl/7.47.0``. + +The responder replied with a 200 OK message, with a MIME (Multipurpose Internet +Mail Extensions) type of ``text/plain``. Zeek provides us a file ID (or +``fuid``) of ``FEEsZS1w0Z0VJIb5x4``. If we had configured Zeek to log files of +type ``text/plain``, we could look at the content returned by the responder. + +Finally, note the UID of ``C5bLoe2Mvxqhawzqqd``. This is the same UID found in +the :file:`conn.log` for this TCP connection. This allows us to link the +:file:`conn.log` entry with this :file:`http.log` entry. + +Reviewing the Original Traffic +============================== + +To better understand the original traffic, and how it relates to the Zeek +:file:`http.log`, let’s look at the contents manually. HTTP is a clear-text +protocol. Assuming the contents are also clear text, and not obfuscated or +encrypted, we can look at the contents. In the following example I use the +venerable program :program:`tcpflow` to create two files. One contains data +from the originator to the responder, while the second contains data from the +responder to the originator. + +.. code-block:: console + + zeek@zeek:~/zeek-test$ tcpflow -r tm1t.pcap port 80 + +Let’s first look at the data from the originator to the responder. + +.. code-block:: console + + zeek@zeek:~/zeek-test$ cat 192.168.004.076.46378-031.003.245.133.00080 + +:: + + GET / HTTP/1.1 + Host: testmyids.com + User-Agent: curl/7.47.0 + Accept: */* + +Here is the data from the responder to the originator. + +.. code-block:: console + + zeek@zeek:~/zeek-test$ cat 031.003.245.133.00080-192.168.004.076.46378 + +:: + + HTTP/1.1 200 OK + Server: nginx/1.16.1 + Date: Fri, 05 Jun 2020 14:40:07 GMT + Content-Type: text/html; charset=UTF-8 + Content-Length: 39 + Connection: keep-alive + Last-Modified: Fri, 10 Jan 2020 21:36:02 GMT + ETag: "27-59bcfe9932c32" + Accept-Ranges: bytes + + uid=0(root) gid=0(root) groups=0(root) + +As you can see, there are elements, particularly in the response, that do not +appear in the :file:`http.log`. For example, the Server type of +``nginx/1.16.1`` is not logged. If an analyst or administrator decided that he +or she wished to include that data in his or her :file:`http.log`, it is +possible to make adjustments. + +The data from the responder also shows the application payload it sent:: + + uid=0(root) gid=0(root) groups=0(root) + +This is the output of a Unix ``uname -a`` command. It is hosted at the server +``testmyids.com`` to trigger a “GPL ATTACK_RESPONSE id check returned root” +alert found in open source intrusion detection engine rule sets, such as that +supported by Suricata. Analysts sometimes use this site to test if their +intrusion detection engines are functioning properly. A more modern option with +many different tests can be found at https://github.com/0xtf/testmynids.org. + +Conclusion +========== + +Zeek’s :file:`http.log` is another important log that offers a great deal of +information on how systems are interacting with the Internet and each other. In +the example in this section we looked at a very simple interaction between an +originator and a responder. We could see the benefit of summarizing an HTTP +request and response in a single log entry. In the next section we will look +at other core Internet protocols. diff --git a/doc/logs/index.rst b/doc/logs/index.rst new file mode 100644 index 0000000000..6350dee59f --- /dev/null +++ b/doc/logs/index.rst @@ -0,0 +1,31 @@ +========= +Zeek Logs +========= + +.. toctree:: + :maxdepth: 1 + + analyzer + conn + dns + http + files + ftp + ssl + x509 + smtp + ssh + pe + dhcp + ntp + smb + irc + ldap + postgresql + quic + rdp + traceroute + tunnel + known-and-software + weird-and-notice + capture-loss-and-reporter diff --git a/doc/logs/irc.rst b/doc/logs/irc.rst new file mode 100644 index 0000000000..56fbe10fa2 --- /dev/null +++ b/doc/logs/irc.rst @@ -0,0 +1,475 @@ +======= +irc.log +======= + +Internet Relay Chat (IRC) is an older protocol that enables real time chat and +collaboration. The Zeek project hosted an IRC channel for many years to support +development and discussion. Some intruders eventually began using IRC to +control botnets, primarily for two reasons. First, as IRC had legitimate uses, +it may not have been suspicious or malicious to see IRC traffic on the wire. +Second, IRC enabled command-and-control, thanks to the ability for operators to +issue instructions to clients that controlled compromised systems. + +Traditionally, IRC clients connect via a clear-text TCP session to an IRC +server listening on port 6667. The commands and responses are text-based, +making it possible for an analyst to manually inspect them. More recent +implementations of IRC servers offer IRC over TLS, with the servers listening +on port 6697 TCP. However, for both unencrypted or encrypted sessions, IRC +servers can listen on any TCP port. + +For full details on each field in the :file:`irc.log` file, please see +:zeek:see:`IRC::Info`. + +Reconstructing an IRC Session +============================= + +Before examining the data provided by Zeek’s :file:`irc.log`, it might be +useful to see the contents of an IRC session. I generated the following +activity using the Hexchat IRC client. + +I have edited the transcript to focus on essential items. Text in bold was sent +by the IRC client. The server sent the remaining text. + +.. literal-emph:: + + **CAP LS 302** + :barjavel.freenode.net NOTICE * :*** Looking up your hostname... + **NICK zeektest** + **USER zeektest 0 * :realname** + :barjavel.freenode.net NOTICE * :*** Checking Ident + :barjavel.freenode.net NOTICE * :*** Found your hostname + :barjavel.freenode.net NOTICE * :*** No Ident response + :barjavel.freenode.net CAP * LS :account-notify away-notify cap-notify chghost extended-join identify-msg multi-prefix sasl tls + **CAP REQ :account-notify away-notify cap-notify chghost extended-join identify-msg multi-prefix** + **:barjavel.freenode.net CAP zeektest ACK :account-notify away-notify cap-notify chghost extended-join identify-msg multi-prefix ** + **CAP END** + :barjavel.freenode.net 001 zeektest :Welcome to the freenode Internet Relay Chat Network zeektest + :barjavel.freenode.net 002 zeektest :Your host is barjavel.freenode.net[195.154.200.232/6667], running version ircd-seven-1.1.9 + :barjavel.freenode.net 003 zeektest :This server was created Thu Dec 19 2019 at 20:10:02 UTC + :barjavel.freenode.net 004 zeektest barjavel.freenode.net ircd-seven-1.1.9 DOQRSZaghilopsuwz CFILMPQSbcefgijklmnopqrstuvz bkloveqjfI + :barjavel.freenode.net 005 zeektest CHANTYPES=# EXCEPTS INVEX CHANMODES=eIbq,k,flj,CFLMPQScgimnprstuz CHANLIMIT=#:120 PREFIX=(ov)@+ MAXLIST=bqeI:100 MODES=4 NETWORK=freenode STATUSMSG=@+ CALLERID=g CASEMAPPING=rfc1459 :are supported by this server + :barjavel.freenode.net 005 zeektest CHARSET=ascii NICKLEN=16 CHANNELLEN=50 TOPICLEN=390 DEAF=D FNC TARGMAX=NAMES:1,LIST:1,KICK:1,WHOIS:1,PRIVMSG:4,NOTICE:4,ACCEPT:,MONITOR: EXTBAN=$,ajrxz CLIENTVER=3.0 WHOX KNOCK CPRIVMSG :are supported by this server + :barjavel.freenode.net 005 zeektest CNOTICE ETRACE SAFELIST ELIST=CTU MONITOR=100 :are supported by this server + :barjavel.freenode.net 251 zeektest :There are 101 users and 82081 invisible on 31 servers + :barjavel.freenode.net 252 zeektest 43 :IRC Operators online + :barjavel.freenode.net 253 zeektest 45 :unknown connection(s) + :barjavel.freenode.net 254 zeektest 41982 :channels formed + :barjavel.freenode.net 255 zeektest :I have 3809 clients and 1 servers + :barjavel.freenode.net 265 zeektest 3809 5891 :Current local users 3809, max 5891 + :barjavel.freenode.net 266 zeektest 82182 90930 :Current global users 82182, max 90930 + :barjavel.freenode.net 250 zeektest :Highest connection count: 5892 (5891 clients) (1543159 connections received) + :barjavel.freenode.net 375 zeektest :- barjavel.freenode.net Message of the Day - + :barjavel.freenode.net 372 zeektest :- Welcome to barjavel.freenode.net in Paris, FR, EU. + ...edited… + :barjavel.freenode.net 372 zeektest :- Thank you for using freenode! + :barjavel.freenode.net 376 zeektest :End of /MOTD command. + :zeektest MODE zeektest :+i + **JOIN #freenode** + :zeektest!~zeektest@pool-XX-XXX-XXX-XX.washdc.fios.verizon.net JOIN #freenode * :realname + :barjavel.freenode.net 332 zeektest #freenode :Welcome to #freenode | Don't copy/paste spam | No politics. | Feel free to message staff at any time. You can find us using /stats p (shows immediately-available staff) or /who freenode/staff/* (shows all staff) + :barjavel.freenode.net 333 zeektest #freenode deadk 1604191950 + ...edited… + :ChanServ!ChanServ@services. NOTICE zeektest :+[#freenode] Please read the topic. + :services. 328 zeektest #freenode :https://freenode.net/ + **WHO #freenode %chtsunfra,152** + :barjavel.freenode.net 324 zeektest #freenode +CLPcntjf 5:10 #freenode-overflow + ...edited… + **PING LAG641756037** + :barjavel.freenode.net PONG barjavel.freenode.net :LAG641756037 + :willcl_ark!~quassel@cpc123780-trow7-2-0-cust177.18-1.cable.virginm.net AWAY :Away + :EGH!~EGH@79.142.76.202 JOIN #freenode EGH :Erik + **PRIVMSG #freenode :One more test... thanks everyone.** + **QUIT :Leaving** + :zeektest!~zeektest@pool-XX-XXX-XXX-XX.washdc.fios.verizon.net QUIT :Client Quit + ERROR :Closing Link: pool-XX-XXX-XXX-XX.washdc.fios.verizon.net (Client Quit) + +As you can see, there is a lot of detail about the IRC server and the channels +and users it supports. The client uses the nickname ``zeektest`` and joins the +``#freenode`` channel. It issues one message. ``One more test… thanks +everyone``, and then quits. + +I captured this traffic by manually setting disabling TLS. Otherwise, the +protocol exchange would have been opaque to Zeek (and other NSM tools). + +With this basic background on IRC, let’s see how Zeek renders this activity. + +Port 6667 :file:`conn.log` +========================== + +Zeek generated the following :file:`conn.log` entry for the example traffic. + +.. literal-emph:: + + { + "ts": 1607009493.558305, + "uid": "CDsHGC2ZJuJh10XNbk", + "id.orig_h": "192.168.4.142", + "id.orig_p": 52856, + "id.resp_h": "195.154.200.232", + **"id.resp_p": 6667,** + **"proto": "tcp",** + **"service": "irc",** + "duration": 55.26594305038452, + "orig_bytes": 311, + "resp_bytes": 239330, + "conn_state": "RSTO", + "missed_bytes": 0, + "history": "ShADadfR", + "orig_pkts": 41, + "orig_ip_bytes": 1963, + "resp_pkts": 185, + "resp_ip_bytes": 246742, + "ip_proto": 6 + } + +We see that Zeek correctly identified this traffic as IRC. We can expect to see +an :file:`irc.log` entry. + +Port 6667 :file:`irc.log` +========================= + +Zeek generated the following three :file:`irc.log` entries: + +.. literal-emph:: + + { + "ts": 1607009493.733304, + "uid": "CDsHGC2ZJuJh10XNbk", + "id.orig_h": "192.168.4.142", + "id.orig_p": 52856, + "id.resp_h": "195.154.200.232", + "id.resp_p": 6667, + **"command": "NICK",** + **"value": "zeektest"** + } + { + "ts": 1607009493.733304, + "uid": "CDsHGC2ZJuJh10XNbk", + "id.orig_h": "192.168.4.142", + "id.orig_p": 52856, + "id.resp_h": "195.154.200.232", + "id.resp_p": 6667, + **"nick": "zeektest",** + **"command": "USER",** + **"value": "zeektest",** + "addl": "0 * realname" + } + { + "ts": 1607009514.481161, + "uid": "CDsHGC2ZJuJh10XNbk", + "id.orig_h": "192.168.4.142", + "id.orig_p": 52856, + "id.resp_h": "195.154.200.232", + "id.resp_p": 6667, + **"nick": "zeektest",** + **"user": "zeektest",** + **"command": "JOIN",** + **"value": "#freenode",** + "addl": "" + } + +We see that Zeek collected information on three aspects of the IRC activity. It +captured the setting of the NICK and USER values, as well as a JOIN command. + +Looking at the Zeek scripting reference, it looks like Zeek will also track +Direct Client-to-Client (or Direct Client Connection, also known as DCC) +activity, usually used to exchange files via IRC. + +Now that we know what a traditional unencrypted IRC session looks like, let’s +see how a modern TLS-encrypted IRC session appears. + +Port 6697 :file:`conn.log` +========================== + +Running Zeek against a capture of IRC over TLS, Zeek produces the following +:file:`conn.log` entry. + +.. literal-emph:: + + { + "ts": 1607009173.307125, + "uid": "CxLRXG3BJ8KYCW6flg", + "id.orig_h": "192.168.4.142", + "id.orig_p": 59423, + "id.resp_h": "185.30.166.38", + **"id.resp_p": 6697,** + **"proto": "tcp",** + **"service": "ssl",** + "duration": 80.66936779022217, + "orig_bytes": 1162, + "resp_bytes": 251941, + "conn_state": "RSTR", + "missed_bytes": 0, + "history": "ShADadfr", + "orig_pkts": 49, + "orig_ip_bytes": 3134, + "resp_pkts": 197, + "resp_ip_bytes": 259833 + } + +Here we see that Zeek only knows that it is looking at a TLS session. + +Port 6697 :file:`ssl.log` and :file:`x509.log` +============================================== + +Because this traffic is encrypted via TLS, Zeek produced :file:`ssl.log` and +:file:`x509.log` entries. + +First, let’s look at :file:`ssl.log`: + +.. literal-emph:: + + { + "ts": 1607009173.826036, + "uid": "CxLRXG3BJ8KYCW6flg", + "id.orig_h": "192.168.4.142", + "id.orig_p": 59423, + "id.resp_h": "185.30.166.38", + "id.resp_p": 6697, + "version": "TLSv12", + "cipher": "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "curve": "secp256r1", + **"server_name": "chat.freenode.net",** + "resumed": false, + "established": true, + "cert_chain_fuids": [ + "F6pDkA4niQwyXPxugf", + "F1JGJ81fmUN17LOYnk" + ], + "client_cert_chain_fuids": [], + **"subject": "CN=verne.freenode.net",** + "issuer": "CN=Let's Encrypt Authority X3,O=Let's Encrypt,C=US" + } + +The references to Freenode and ``chat`` can help clue an analyst to the +likelihood that the client is engaging in IRC sessions. + +Now let’s look at the :file:`x509.log`: + +.. literal-emph:: + + { + "ts": 1607009173.828159, + "id": "F6pDkA4niQwyXPxugf", + "certificate.version": 3, + "certificate.serial": "040831FAE9EF9E4D666A4B9EDE996878C79B", + "certificate.subject": "CN=verne.freenode.net", + "certificate.issuer": "CN=Let's Encrypt Authority X3,O=Let's Encrypt,C=US", + "certificate.not_valid_before": 1605501336, + "certificate.not_valid_after": 1613277336, + "certificate.key_alg": "rsaEncryption", + "certificate.sig_alg": "sha256WithRSAEncryption", + "certificate.key_type": "rsa", + "certificate.key_length": 4096, + "certificate.exponent": "65537", + "san.dns": [ + **"chat.au.freenode.com",** + **"chat.au.freenode.net",** + **"chat.au.freenode.org",** + **"chat.eu.freenode.com",** + **"chat.eu.freenode.net",** + **"chat.eu.freenode.org",** + **"chat.freenode.com",** + **"chat.freenode.net",** + **"chat.freenode.org",** + **"chat.ipv4.freenode.com",** + **"chat.ipv4.freenode.net",** + **"chat.ipv4.freenode.org",** + **"chat.ipv6.freenode.com",** + **"chat.ipv6.freenode.net",** + **"chat.ipv6.freenode.org",** + **"chat.us.freenode.com",** + **"chat.us.freenode.net",** + **"chat.us.freenode.org",** + **"ipv6.chat.freenode.net",** + **"ipv6.irc.freenode.net",** + **"irc.au.freenode.com",** + **"irc.au.freenode.net",** + **"irc.au.freenode.org",** + **"irc.eu.freenode.com",** + **"irc.eu.freenode.net",** + **"irc.eu.freenode.org",** + **"irc.freenode.com",** + **"irc.freenode.net",** + **"irc.freenode.org",** + **"irc.ipv4.freenode.com",** + **"irc.ipv4.freenode.net",** + **"irc.ipv4.freenode.org",** + **"irc.ipv6.freenode.com",** + **"irc.ipv6.freenode.net",** + **"irc.ipv6.freenode.org",** + **"irc.us.freenode.com",** + **"irc.us.freenode.net",** + **"irc.us.freenode.org",** + **"verne.freenode.net"** + ], + "basic_constraints.ca": false + } + { + "ts": 1607009173.828159, + "id": "F1JGJ81fmUN17LOYnk", + "certificate.version": 3, + "certificate.serial": "0A0141420000015385736A0B85ECA708", + "certificate.subject": "CN=Let's Encrypt Authority X3,O=Let's Encrypt,C=US", + "certificate.issuer": "CN=DST Root CA X3,O=Digital Signature Trust Co.", + "certificate.not_valid_before": 1458232846, + "certificate.not_valid_after": 1615999246, + "certificate.key_alg": "rsaEncryption", + "certificate.sig_alg": "sha256WithRSAEncryption", + "certificate.key_type": "rsa", + "certificate.key_length": 2048, + "certificate.exponent": "65537", + "basic_constraints.ca": true, + "basic_constraints.path_len": 0 + } + +The bolded entries containing strings with “IRC”, “chat”, and Freenode are +again clues that IRC is in play here. + +Port 31337 :file:`conn.log` +=========================== + +Here is a different session where port 31337 TCP offered clear-text IRC +connections. Zeek produced three :file:`conn.log` entries, involving clients +with IP addresses of ``10.240.0.3``, ``10.240.0.4``, and ``10.240.0.5``. Here +is an entry for the client ``10.240.0.5``. + +.. literal-emph:: + + { + "ts": 1461774814.057057, + "uid": "Cs0hwm3slMw4IBDU0h", + "id.orig_h": "10.240.0.5", + "id.orig_p": 42277, + "id.resp_h": "10.240.0.2", + **"id.resp_p": 31337,** + **"proto": "tcp",** + **"service": "irc",** + "duration": 787.9501581192017, + "orig_bytes": 1026, + "resp_bytes": 10425, + "conn_state": "SF", + "missed_bytes": 0, + "history": "ShADadfF", + "orig_pkts": 95, + "orig_ip_bytes": 5974, + "resp_pkts": 87, + "resp_ip_bytes": 14957 + } + +Zeek identified the protocol as IRC by using its dynamic port detection +functionality. It did not need to see IRC on port 6667 TCP in order to +recognize the protocol. + +Port 31337 :file:`irc.log` +========================== + +Zeek produced many entries in the :file:`irc.log` for this activity, so I +extracted the key values. + +.. code-block:: console + + $ jq -c '[."id.orig_h", ."nick", ."user", ."command", ."value", ."addl"]' irc.log + +:: + + ["10.240.0.3",null,null,"NICK","Matir",null] + ["10.240.0.3","Matir",null,"USER","root-poppopret","root-poppopret 10.240.0.2 matir"] + ["10.240.0.3","Matir","root-poppopret","JOIN","#ctf",""] + ["10.240.0.4",null,null,"NICK","andrewg",null] + ["10.240.0.4","andrewg",null,"USER","root-poppopret","root-poppopret 10.240.0.2 andrewg"] + ["10.240.0.4","andrewg","root-poppopret","JOIN","#ctf",""] + ["10.240.0.5",null,null,"NICK","itsl0wk3y",null] + ["10.240.0.5","itsl0wk3y",null,"USER","root-poppopret","root-poppopret 10.240.0.2 l0w"] + ["10.240.0.5","itsl0wk3y","root-poppopret","JOIN","#ctf",""] + +As with the previous :file:`irc.log`, you can see elements like the nickname, +username, commands, and additional data for the connections. You do not see any +details of what users said to each other. + +Botnet IRC Traffic +================== + +The following example is an excerpt from a case provided by the Malware Capture +Facility, a sister project to the Stratosphere IPS Project. The case is +CTU-IoT-Malware-Capture-3-1, located here: + +https://mcfp.felk.cvut.cz/publicDatasets/IoTDatasets/CTU-IoT-Malware-Capture-3-1/ + +The case includes IRC traffic caused by systems compromised and under the +control of the Muhstihk botnet. More details are available in this blog post: + +https://blog.netlab.360.com/botnet-muhstik-is-actively-exploiting-drupal-cve-2018-7600-in-a-worm-style-en/ + +Here is a summary of the :file:`conn.log` for the malicious IRC traffic. + +.. code-block:: console + + $ jq -c '[."id.orig_h", ."id.resp_h", ."id.resp_p", ."proto", ."service"]' conn.log + +:: + + ["192.168.2.5","111.230.241.23",2407,"tcp","irc"] + ["192.168.2.5","51.38.81.99",2407,"tcp","irc"] + ["192.168.2.5","185.61.149.22",2407,"tcp",null] + ["192.168.2.5","54.39.23.28",2407,"tcp","irc"] + ["192.168.2.5","54.39.23.28",2407,"tcp","irc"] + ["192.168.2.5","185.47.129.56",2407,"tcp",null] + ["213.140.50.114","192.168.2.5",1,"icmp",null] + ["192.168.2.5","111.230.241.23",2407,"tcp","irc"] + ["192.168.2.5","54.39.23.28",2407,"tcp","irc"] + +We see the victim, ``192.168.2.5``, connecting to multiple IRC servers on port +2407 TCP. Note that Zeek does not recognize all of the IRC traffic using its +IRC protocol analyzer. Zeek does see six IRC sessions that it parses in the +:file:`irc.log`. + +Here is a summary of the :file:`irc.log` for the IRC traffic created by this +botnet client. + +.. code-block:: console + + $ jq -c '[."id.orig_h", ."id.resp_h", ."nick", ."user", ."command", ."value", ."addl"]' irc.log + +:: + + ["192.168.2.5","111.230.241.23",null,null,"NICK","A5|1|5358668|black-pe",null] + ["192.168.2.5","111.230.241.23","A5|1|5358668|black-pe",null,"USER","muhstik","localhost localhost muhstik-11052018"] + ["192.168.2.5","51.38.81.99",null,null,"NICK","A5|1|5358668|black-pe",null] + ["192.168.2.5","51.38.81.99","A5|1|5358668|black-pe",null,"USER","muhstik","localhost localhost muhstik-11052018"] + ["192.168.2.5","51.38.81.99","A5|1|5358668|black-pe","muhstik","JOIN","#a925d765"," with channel key: ':8974'"] + ["192.168.2.5","54.39.23.28",null,null,"NICK","A5|1|5358668|black-pe",null] + ["192.168.2.5","54.39.23.28","A5|1|5358668|black-pe",null,"USER","muhstik","localhost localhost muhstik-11052018"] + ["192.168.2.5","54.39.23.28","A5|1|5358668|black-pe","muhstik","JOIN","#a925d765"," with channel key: ':8974'"] + ["192.168.2.5","54.39.23.28",null,null,"NICK","A5|1|5358668|black-pe",null] + ["192.168.2.5","54.39.23.28","A5|1|5358668|black-pe",null,"USER","muhstik","localhost localhost muhstik-11052018"] + ["192.168.2.5","54.39.23.28","A5|1|5358668|black-pe","muhstik","JOIN","#a925d765"," with channel key: ':8974'"] + ["192.168.2.5","111.230.241.23",null,null,"NICK","A5|1|5358668|black-pe",null] + ["192.168.2.5","111.230.241.23","A5|1|5358668|black-pe",null,"USER","muhstik","localhost localhost muhstik-11052018"] + ["192.168.2.5","111.230.241.23","A5|1|5358668|black-pe","muhstik","JOIN","#a925d765"," with channel key: ':8974'"] + ["192.168.2.5","54.39.23.28",null,null,"NICK","A5|1|5358668|black-pe",null] + ["192.168.2.5","54.39.23.28","A5|1|5358668|black-pe",null,"USER","muhstik","localhost localhost muhstik-11052018"] + ["192.168.2.5","54.39.23.28","A5|1|5358668|black-pe","muhstik","JOIN","#a925d765"," with channel key: ':8974'"] + +Here is an example transcript for one of the IRC sessions: + +.. literal-emph:: + + **NICK A5|1|5358668|black-pe** + **USER muhstik localhost localhost :muhstik-11052018** + PING :A2A5630 + **PONG :A2A5630** + :x4.tipu 010 A5|1|5358668|black-pe x4.tipu 0 + :x4.tipu 010 A5|1|5358668|black-pe pomf 6667 + ERROR :Closing Link: A5|1|5358668|black-pe[109.81.208.168] (This server is full.) + +Thankfully for the analyst, it declares itself using the easily-searchable name +``muhstik``. This makes it easy to do open source research and identify the +malicious nature of the activity. + +Conclusion +========== + +Security analysts may still encounter IRC when botnets and other malware use it +for command-and-control. As other forms of modern collaboration and chat have +become prevalent, the normality of IRC has become a remnant of a bygone era. diff --git a/doc/logs/known-and-software.rst b/doc/logs/known-and-software.rst new file mode 100644 index 0000000000..f2187f683c --- /dev/null +++ b/doc/logs/known-and-software.rst @@ -0,0 +1,137 @@ +============================ +known_*.log and software.log +============================ + +Zeek produces several logs that help summarize certain aspects of the network +it monitors. These logs track a few aspects of the local network, such as +SSL/TLS certificates, host IP addresses, services, and applications. + +The sections which follow will present examples of entries in +:file:`known_certs.log`, :file:`known_hosts.log`, :file:`known_services.log`, +and :file:`software.log` files collected on live networks. + +For full details on each field of those log files, see +:zeek:see:`Known::CertsInfo`, :zeek:see:`Known::HostsInfo`, +:zeek:see:`Known::ServicesInfo`, and :zeek:see:`Software::Info`. + +:file:`known_certs.log` +======================= + +The :file:`known_certs.log` captures information about SSL/TLS certificates +seen on the local network. Here is one example:: + + { + "ts": "2020-12-31T15:15:53.690221Z", + "host": "192.168.4.1", + "port_num": 443, + "subject": "L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US", + "issuer_subject": "L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US", + "serial": "98D0AD47D748CDD6" + } + +This example shows a device offering a TLS server on port 443 TCP, with a +certificate associated with Ubiquiti Networks. + +:file:`known_hosts.log` +======================= + +The :file:`known_hosts.log` simply records a timestamp and an IP address when +Zeek observes a new system on the local network. + +:: + + {"ts":"2021-01-03T01:19:26.260073Z","host":"192.168.4.25"} + {"ts":"2021-01-03T01:19:27.353353Z","host":"192.168.4.29"} + {"ts":"2021-01-03T01:19:32.488179Z","host":"192.168.4.43"} + {"ts":"2021-01-03T01:19:58.792683Z","host":"192.168.4.142"} + ...edited... + {"ts":"2021-01-03T12:17:22.496004Z","host":"192.168.4.115"} + +This edited example shows how this log could be part of an IP address inventory +program. + +:file:`known_services.log` +========================== + +The :file:`known_services.log` records a timestamp, IP, port number, protocol, +and service (if available) when Zeek observes a system offering a new service +on the local network. Here is what a single entry looks like:: + + { + "ts": "2021-01-03T01:19:36.242774Z", + "host": "192.168.4.1", + "port_num": 53, + "port_proto": "udp", + "service": [ + "DNS" + ] + } + +For the following list, I used the :program:`jq` utility to remove the +timestamp but show the other log values. + +:: + + ["192.168.4.43",51472,"tcp",[]] + ["192.168.4.1",443,"tcp",["SSL"]] + ["192.168.4.1",80,"tcp",["HTTP"]] + ["192.168.4.1",22,"tcp",["SSH"]] + ["192.168.4.1",53,"tcp",["DNS"]] + ["192.168.4.1",123,"udp",["NTP"]] + ["192.168.4.50",49745,"tcp",[]] + ["192.168.4.158",4500,"udp",[]] + ["192.168.4.159",53032,"tcp",[]] + ["192.168.4.142",36807,"udp",[]] + ["192.168.4.1",53,"udp",["DNS"]] + ["192.168.4.149",8080,"tcp",["HTTP"]] + ["192.168.4.1",67,"udp",["DHCP"]] + ["192.168.4.43",64744,"tcp",[]] + ["192.168.4.43",52793,"tcp",[]] + ["192.168.4.29",52827,"tcp",[]] + ["192.168.4.43",64807,"tcp",[]] + ["192.168.4.43",64752,"tcp",[]] + ["192.168.4.149",3478,"udp",[]] + +Note how many of the services do not have names associated with them. + +:file:`software.log` +==================== + +Zeek’s :file:`software.log` collects details on applications operated by the +hosts it sees on the local network. The log captures information like the +following:: + + { + "ts": "2021-01-03T00:16:22.694616Z", + "host": "192.168.4.25", + "software_type": "HTTP::BROWSER", + "name": "Windows-Update-Agent", + "version.major": 10, + "version.minor": 0, + "version.minor2": 10011, + "version.minor3": 16384, + "version.addl": "Client", + "unparsed_version": "Windows-Update-Agent/10.0.10011.16384 Client-Protocol/2.0" + } + +It is amazing in 2021 that so many modern applications still use clear text +protocols subject to collection and analysis by software like Zeek. + +Services beyond HTTP may also reveal interesting details. Consider these three +entries:: + + ["192.168.4.1","SSH::SERVER","OpenSSH",6,6,1,null,"p1","OpenSSH_6.6.1p1 Debian-4~bpo70+1"] + ["192.168.4.37","SSH::CLIENT","OpenSSH",6,6,1,null,"p1","OpenSSH_6.6.1p1 Debian-4~bpo70+1"] + ["192.168.4.37","SSH::CLIENT","OpenSSH",7,6,null,null,"p1","OpenSSH_7.6p1"] + +These examples show an SSH server and two different SSH clients. + +Conclusion +========== + +Details recorded in :file:`known_certs.log`, :file:`known_hosts.log`, +:file:`known_services.log`, and :file:`software.log` files can help network and +security analysts better understand the nature of the activity in their +environment. Some of this information relies on capturing clear text, while +other aspects are based solely on the presence of the services and hosts on the +network. diff --git a/doc/logs/ldap.rst b/doc/logs/ldap.rst new file mode 100644 index 0000000000..d8e073a76b --- /dev/null +++ b/doc/logs/ldap.rst @@ -0,0 +1,140 @@ +============================ +ldap.log and ldap_search.log +============================ + +.. versionadded:: 6.1 + +The Lightweight Directory Access Protocol (LDAP) is a +widely observed protocol commonly used for authenticating, directory lookups, +centralizing organisational information and accessing client information on +email servers. Accordingly, the protocol attracts significant attention from +those with adversarial intention. + + +LDAP Protocol Overview +====================== + +LDAP communicates using a client-server model. The LDAP server contains the +directory information and the LDAP client performs operations against this +information. This is a quick overview of how the protocol works: + + Sessions: An LDAP session begins with a client connecting to an LDAP server, + optionally securing the connection with encryption, and then binding to the + server by providing credentials. + + Queries: Clients search for entries in the LDAP directory using LDAP + queries, which consist of a base Distinguished Name (DN), a scope (such + as one level or the entire subtree), and a filter to match entries. Queries + are read only. + + Operations: Clients with the correct privileges can perform a variety of + operations; in addition to search, they can add, delete or modify. + + Data Format: LDAP data entries are formatted as records consisting of a + DN and a set of attributes. Each attribute has a name and one or more values. + +The LDAP analyzer outputs two LDAP related logs. :file:`ldap.log` contains +details about the LDAP session except those related to searches. +:file:`ldap_search.log` contains information related to LDAP searches. + +For details on every element of the :file:`ldap.log` and :file:`ldap_search.log` +refer to :zeek:see:`LDAP::MessageInfo` and :zeek:see:`LDAP::SearchInfo`, respectively. +Below is an inspection of the :file:`ldap.log` and :file:`ldap_search.log` in JSON format. + +:file:`ldap.log` +================ + +An example of an :file:`ldap.log`. + +.. code-block:: console + + zeek@zeek-6.1:~ zeek -C LogAscii::use_json=T LDAP::default_log_search_attributes=T -r ldap-simpleauth.pcap + zeek@zeek-6.1:~ jq . ldap.log + +:: + + { + "ts": 1463256456.051759, + "uid": "ChD43F3guxAmJ5f2aj", + "id.orig_h": "10.0.0.1", + "id.orig_p": 25936, + "id.resp_h": "10.0.0.2", + "id.resp_p": 3268, + "message_id": 3, + "version": 3, + "opcode": "bind simple", + "result": "success", + "object": "CN=xxxxxxxx,OU=Users,OU=Accounts,DC=xx,DC=xxx,DC=xxxxx,DC=net", + "argument": "REDACTED" + } + + +:file:`ldap_search.log` +======================= + +An example of an :file:`ldap_search.log`. Note the default for +:zeek:see:`LDAP::default_log_search_attributes` is F, excluding attributes +from the log. + +.. code-block:: console + + zeek@zeek-6.1:~ zeek -C LogAscii::use_json=T LDAP::default_log_search_attributes=T -r ldap-simpleauth.pcap + zeek@zeek-6.1:~ jq . ldap_search.log + +:: + + { + "ts": 1463256456.047579, + "uid": "CAOF1l3FR8UzQ7mIb8", + "id.orig_h": "10.0.0.1", + "id.orig_p": 25936, + "id.resp_h": "10.0.0.2", + "id.resp_p": 3268, + "message_id": 2, + "scope": "tree", + "deref_aliases": "always", + "base_object": "DC=xx,DC=xxx,DC=xxxxx,DC=net", + "result_count": 1, + "result": "success", + "filter": "(&(objectclass=*)(sAMAccountName=xxxxxxxx))", + "attributes": [ + "sAMAccountName" + ] + } + + +StartTLS +======== + +.. versionadded:: 7.0 + +Zeek's LDAP analyzer supports the +`extended StartTLS `_ +operation, handing off analysis to Zeek's TLS analyzer. The following shows an +example :file:`ldap.log` entry for the StartTLS request. + +.. code-block:: console + + $ zeek -C LogAscii::use_json=T -r ldap-starttls.pcap + $ jq < ldap.log + { + "ts": 1721218680.158341, + "uid": "CW0qzo9A3QsrCWL4k", + "id.orig_h": "127.0.0.1", + "id.orig_p": 45936, + "id.resp_h": "127.0.1.1", + "id.resp_p": 389, + "message_id": 1, + "opcode": "extended", + "result": "success", + "object": "1.3.6.1.4.1.1466.20037 (StartTLS)" + } + +The :file:`conn.log`'s history field will contain ``ssl`` and ``ldap`` in +the ``service`` field. + +Conclusion +========== + +The Zeek LDAP logs provide additional insights that help improve observability +into this protocol. diff --git a/doc/logs/ntp.rst b/doc/logs/ntp.rst new file mode 100644 index 0000000000..73bd570280 --- /dev/null +++ b/doc/logs/ntp.rst @@ -0,0 +1,294 @@ +======= +ntp.log +======= + +Network Time Protocol (NTP) is another core protocol found in IP networks. NTP +is a mechanism by which clients can adjust their local clocks to more closely +match those of NTP servers. Many devices ship with NTP clients already +configured to contact public NTP servers. Administrators can use Zeek logs to +identify NTP clients and servers, and determine if they are operating as +expected. + +As with all entries in this chapter, see :zeek:see:`NTP::Info` for full +explanation of each field in the log. + +NTP via :program:`tcpdump` +========================== + +NTP is a request-response protocol, as demonstrated by the following exchange +decoded by :program:`tcpdump`:: + + 00:29:07.927672 IP 192.168.4.49.38461 > 208.79.89.249.123: NTPv4, Client, length 48 + 00:29:07.995844 IP 208.79.89.249.123 > 192.168.4.49.38461: NTPv4, Server, length 48 + +Using the verbose feature, we see the following details:: + + 00:29:07.927672 IP (tos 0x10, ttl 64, id 3186, offset 0, flags [DF], proto UDP (17), length 76) + 192.168.4.49.38461 > 208.79.89.249.123: [udp sum ok] NTPv4, length 48 + Client, Leap indicator: (0), Stratum 0 (unspecified), poll 0 (1s), precision 0 + Root Delay: 0.000000, Root dispersion: 0.000000, Reference-ID: (unspec) + Reference Timestamp: 0.000000000 + Originator Timestamp: 0.000000000 + Receive Timestamp: 0.000000000 + Transmit Timestamp: 3811105747.215585991 (2020/10/08 00:29:07) + Originator - Receive Timestamp: 0.000000000 + Originator - Transmit Timestamp: 3811105747.215585991 (2020/10/08 00:29:07) + + 00:29:07.995844 IP (tos 0x0, ttl 56, id 18045, offset 0, flags [DF], proto UDP (17), length 76) + 208.79.89.249.123 > 192.168.4.49.38461: [udp sum ok] NTPv4, length 48 + Server, Leap indicator: (0), Stratum 2 (secondary reference), poll 3 (8s), precision -24 + Root Delay: 0.009216, Root dispersion: 0.021224, Reference-ID: 127.67.113.92 + Reference Timestamp: 3811105455.942204197 (2020/10/08 00:24:15) + Originator Timestamp: 3811105747.215585991 (2020/10/08 00:29:07) + Receive Timestamp: 3811105747.964280626 (2020/10/08 00:29:07) + Transmit Timestamp: 3811105747.964314032 (2020/10/08 00:29:07) + Originator - Receive Timestamp: +0.748694635 + Originator - Transmit Timestamp: +0.748728040 + +A look at :rfc:`5905`, explaining NTPv4, helps us understand the timestamps +shown in the decoded output:: + + LI Leap Indicator (leap): 2-bit integer warning of an impending leap second + to be inserted or deleted in the last minute of the current month with values + defined in Figure 9. + + +-------+----------------------------------------+ + | Value | Meaning | + +-------+----------------------------------------+ + | 0 | no warning | + | 1 | last minute of the day has 61 seconds | + | 2 | last minute of the day has 59 seconds | + | 3 | unknown (clock unsynchronized) | + +-------+----------------------------------------+ + + Figure 9: Leap Indicator + + VN Version Number (version): 3-bit integer representing the NTP version + number, currently 4. + + Mode (mode): 3-bit integer representing the mode, with values defined in + Figure 10. + + +-------+--------------------------+ + | Value | Meaning | + +-------+--------------------------+ + | 0 | reserved | + | 1 | symmetric active | + | 2 | symmetric passive | + | 3 | client | + | 4 | server | + | 5 | broadcast | + | 6 | NTP control message | + | 7 | reserved for private use | + +-------+--------------------------+ + + Figure 10: Association Modes + + Stratum (stratum): 8-bit integer representing the stratum, with values + defined in Figure 11. + + +--------+-----------------------------------------------------+ + | Value | Meaning | + +--------+-----------------------------------------------------+ + | 0 | unspecified or invalid | + | 1 | primary server (e.g., equipped with a GPS receiver) | + | 2-15 | secondary server (via NTP) | + | 16 | unsynchronized | + | 17-255 | reserved | + +--------+-----------------------------------------------------+ + + Figure 11: Packet Stratum + + Poll: 8-bit signed integer representing the maximum interval between + successive messages, in log2 seconds. + + Precision: 8-bit signed integer representing the precision of the system + clock, in log2 seconds. For instance, a value of -18 corresponds to a + precision of about one microsecond. + + Root Delay (rootdelay): Total round-trip delay to the reference clock, in NTP + short format. + + Root Dispersion (rootdisp): Total dispersion to the reference clock, in NTP + short format. + + Reference ID (refid): 32-bit code identifying the particular server or + reference clock. + + Reference Timestamp: Time when the system clock was last set or corrected, in + NTP timestamp format. + + Origin Timestamp (org): Time at the client when the request departed for the + server, in NTP timestamp format. + + Receive Timestamp (rec): Time at the server when the request arrived from the + client, in NTP timestamp format. + + Transmit Timestamp (xmt): Time at the server when the response left for the + client, in NTP timestamp format. + + Destination Timestamp (dst): Time at the client when the reply arrived from + the server, in NTP timestamp format. + +It makes sense that the reference, originator, and receive timestamps would be +zero in the client request, but non-zero in the server reply. + +NTP via :program:`tcpdump` and :program:`tshark` +================================================ + +Let’s look at :program:`tshark`’s decode for the NTP-specific data, to see if +:program:`tcpdump` missed anything:: + + Client to server: + + Network Time Protocol (NTP Version 4, client) + Flags: 0x23, Leap Indicator: no warning, Version number: NTP Version 4, Mode: client + 00.. .... = Leap Indicator: no warning (0) + ..10 0... = Version number: NTP Version 4 (4) + .... .011 = Mode: client (3) + Peer Clock Stratum: unspecified or invalid (0) + Peer Polling Interval: invalid (0) + Peer Clock Precision: 1.000000 sec + Root Delay: 0 seconds + Root Dispersion: 0 seconds + Reference ID: NULL + Reference Timestamp: Jan 1, 1970 00:00:00.000000000 UTC + Origin Timestamp: Jan 1, 1970 00:00:00.000000000 UTC + Receive Timestamp: Jan 1, 1970 00:00:00.000000000 UTC + Transmit Timestamp: Oct 8, 2020 00:29:07.215585991 UTC + + Server to client: + + Network Time Protocol (NTP Version 4, server) + Flags: 0x24, Leap Indicator: no warning, Version number: NTP Version 4, Mode: server + 00.. .... = Leap Indicator: no warning (0) + ..10 0... = Version number: NTP Version 4 (4) + .... .100 = Mode: server (4) + Peer Clock Stratum: secondary reference (2) + Peer Polling Interval: invalid (3) + Peer Clock Precision: 0.000000 sec + Root Delay: 0.00921630859375 seconds + Root Dispersion: 0.0212249755859375 seconds + Reference ID: 127.67.113.92 + Reference Timestamp: Oct 8, 2020 00:24:15.942204197 UTC + Origin Timestamp: Oct 8, 2020 00:29:07.215585991 UTC + Receive Timestamp: Oct 8, 2020 00:29:07.964280626 UTC + Transmit Timestamp: Oct 8, 2020 00:29:07.964314032 UTC + +It does not appear that :program:`tshark` reveals any details that +:program:`tcpdump` did not. One difference is that for the client reference, +origin, and receive timestamps, Tshark renders the 0 values as the Unix epoch, +i.e., ``Jan 1, 1970 00:00:00.000000000 UTC``. + +NTP via Zeek +============ + +Here is how Zeek summarizes this NTP activity: + +.. literal-emph:: + + { + "ts": "2020-10-08T00:29:07.977170Z", + "uid": "CqlPpF1AQVLMPgGiL5", + "id.orig_h": "192.168.4.49", + "id.orig_p": 38461, + "id.resp_h": "208.79.89.249", + "id.resp_p": 123, + "version": 4, + **"mode": 3,** + "stratum": 0, + "poll": 1, + "precision": 1, + "root_delay": 0, + "root_disp": 0, + "ref_id": "\\x00\\x00\\x00\\x00", + "ref_time": "1970-01-01T00:00:00.000000Z", + "org_time": "1970-01-01T00:00:00.000000Z", + "rec_time": "1970-01-01T00:00:00.000000Z", + "xmt_time": "2020-10-08T00:29:07.215586Z", + "num_exts": 0 + } + + { + "ts": "2020-10-08T00:29:08.081209Z", + "uid": "CqlPpF1AQVLMPgGiL5", + "id.orig_h": "192.168.4.49", + "id.orig_p": 38461, + "id.resp_h": "208.79.89.249", + "id.resp_p": 123, + "version": 4, + **"mode": 4,** + "stratum": 2, + "poll": 8, + "precision": 5.960464477539063e-08, + "root_delay": 0.00921630859375, + "root_disp": 0.0212249755859375, + "ref_id": "127.67.113.92", + "ref_time": "2020-10-08T00:24:15.942204Z", + "org_time": "2020-10-08T00:29:07.215586Z", + "rec_time": "2020-10-08T00:29:07.964281Z", + "xmt_time": "2020-10-08T00:29:07.964314Z", + "num_exts": 0 + } + +By looking at the mode field in each log, we see that the first entry is a NTP +client request (mode 3), and the second is the server’s reply (mode 4). + +These log entries make an interesting comparison with those for DHCP. Zeek’s +DHCP logs seek to summarize potentially up to four individual datagrams (for +the DORA exchange) into one log entry. In contrast, Zeek’s NTP logs create an +entry for each NTP message. + +Identifying NTP Servers +======================= + +As with DHCP servers, Zeek can help identify NTP servers used by clients. The +following query shows a subset of systems and the NTP servers they have +queried: + +.. code-block:: console + + $ find . -name "ntp**.gz" | while read -r file; do zcat -f "$file"; done | jq -c '[."id.orig_h", ."id.resp_h"]' | sort | uniq -c | sort -nr | head -10 + +:: + + 570 ["192.168.4.48","193.0.0.229"] + 271 ["192.168.4.76","91.189.91.157"] + 271 ["192.168.4.76","216.229.0.50"] + 270 ["192.168.4.76","74.6.168.73"] + 270 ["192.168.4.76","72.30.35.88"] + 270 ["192.168.4.76","38.229.71.1"] + 216 ["192.168.4.149","84.16.73.33"] + 206 ["192.168.4.48","50.205.244.21"] + 164 ["192.168.4.57","216.239.35.12"] + 162 ["192.168.4.57","216.239.35.8"] + +The following query summarizes only the NTP servers seen by Zeek: + +.. code-block:: console + + $ find . -name "ntp**.gz" | while read -r file; do zcat -f "$file"; done | jq -c '[."id.resp_h"]' | sort | uniq -c | sort -nr | head -10 + +:: + + 570 ["193.0.0.229"] + 470 ["17.253.20.253"] + 468 ["17.253.20.125"] + 357 ["91.189.91.157"] + 287 ["216.229.0.50"] + 286 ["74.6.168.73"] + 276 ["72.30.35.88"] + 270 ["38.229.71.1"] + 221 ["84.16.73.33"] + 206 ["50.205.244.21"] + +Security and network administrators can use queries like this to identify +systems that are polling unauthorized NTP servers. + +Conclusion +========== + +NTP is an important protocol for modern network administration. Without +accurate clocks, many systems will not be able to complete cryptographic +exchanges. Be sure systems are kept up to date using the NTP servers you expect +them to query. diff --git a/doc/logs/pe.rst b/doc/logs/pe.rst new file mode 100644 index 0000000000..9c4f4090a6 --- /dev/null +++ b/doc/logs/pe.rst @@ -0,0 +1,506 @@ +====== +pe.log +====== + +Earlier we looked at the data provided by Zeek’s :file:`files.log`. In this +section we will take a step further for one type of log -- Zeek’s +:file:`pe.log`. In this instance, “pe” stands for portable executable, a format +associated with Microsoft binaries. + +For more details on the specifics of the format, please refer to +:zeek:see:`PE::Info`. + +Starting with :file:`conn.log` +============================== + +This example starts with the :file:`conn.log`. It’s not strictly necessary to +explain the :file:`pe.log`, although I wanted to include a recent example +of a modern application conducting activities via HTTP. + +.. literal-emph:: + + { + "ts": "2020-09-23T00:24:31.210053Z", + "uid": "Cq2b9jR12c4lqZafg", + **"id.orig_h": "192.168.4.152",** + "id.orig_p": 59125, + **"id.resp_h": "63.88.73.83",** + **"id.resp_p": 80,** + "proto": "tcp", + **"service": "http",** + "duration": 25.614583015441895, + "orig_bytes": 5753, + "resp_bytes": 1975717, + "conn_state": "SF", + "local_orig": true, + "local_resp": false, + "missed_bytes": 0, + "history": "ShADadttFf", + "orig_pkts": 521, + "orig_ip_bytes": 29041, + "resp_pkts": 1367, + "resp_ip_bytes": 2030409, + "ip_proto": 6 + } + +This example shows a host, ``192.168.4.152``, conducting a HTTP session with +``63.88.73.83`` over port 80 TCP. The server sends 2 MB of content to the +client. + +Continuing with :file:`http.log` +================================ + +The :file:`http.log` entries associated with UID ``Cq2b9jR12c4lqZafg`` are +fascinating. There are multiple entries. I have reproduced a sample of them +below. + +.. literal-emph:: + + { + "ts": "2020-09-23T00:24:31.235201Z", + "uid": "Cq2b9jR12c4lqZafg", + **"id.orig_h": "192.168.4.152",** + "id.orig_p": 59125, + **"id.resp_h": "63.88.73.83",** + **"id.resp_p": 80,** + "trans_depth": 1, + **"method": "HEAD",** + **"host": "r8---sn-8xgp1vo-p5ql.gvt1.com",** + **"uri": "/edgedl/release2/chrome/SAWXCyZhLAbPfxC5kv_Fkw_85.0.4183.121/85.0.4183.121_85.0.4183.102_chrome_updater.exe?cms_redirect=yes&mh=t-&mip=-public-ip-edited-&mm=28&mn=sn-8xgp1vo-p5ql&ms=nvh&mt=1600820539&mv=m&mvi=8&pl=19&shardbypass=yes",** + "version": "1.1", + **"user_agent": "Microsoft BITS/7.8",** + "request_body_len": 0, + "response_body_len": 0, + **"status_code": 200,** + **"status_msg": "OK",** + "tags": [] + } + +The first entry shown above provides details on a HEAD request for a binary +titled ``85.0.4183.121_85.0.4183.102_chrome_updater.exe``. The user agent is +the Microsoft Background Intelligent Transfer Service (BITS). The server +responses with a successful message, 200 OK. Note that I have inserted +``-public-ip-edited-`` in the URI rather than expose the public IP address of +the system requesting this file. + +The fact that the BITS client provides the public IP address in the URI +indicates that either the server is sending this information to the client, or +that the client is requesting this information from an Internet-residing +system. There is no native way for this client to know its public IP address +when it is sitting behind a network address (port) translation device. + +This aspect of the URI could help administrators better understand their +networks, as it can sometimes be difficult to map private IP addresses (like +``192.168.4.152``) to their public representations (here +``-public-ip-edited-``). + +Also note the value for the host field showing +``r8---sn-8xgp1vo-p5ql.gvt1.com``. I resolved the odd name to see the +following: + +.. code-block:: console + + $ host r8---sn-8xgp1vo-p5ql.gvt1.com + +:: + + r8---sn-8xgp1vo-p5ql.gvt1.com is an alias for r8.sn-8xgp1vo-p5ql.gvt1.com. + r8.sn-8xgp1vo-p5ql.gvt1.com has address 63.88.73.83 + r8.sn-8xgp1vo-p5ql.gvt1.com has IPv6 address 2600:803:f00:1::13 + +Let’s look at the next :file:`http.log` entry. + +.. literal-emph:: + + { + "ts": "2020-09-23T00:24:31.334435Z", + "uid": "Cq2b9jR12c4lqZafg", + **"id.orig_h": "192.168.4.152",** + "id.orig_p": 59125, + **"id.resp_h": "63.88.73.83",** + **"id.resp_p": 80,** + "trans_depth": 2, + **"method": "GET",** + **"host": "r8---sn-8xgp1vo-p5ql.gvt1.com",** + **"uri": "/edgedl/release2/chrome/SAWXCyZhLAbPfxC5kv_Fkw_85.0.4183.121/85.0.4183.121_85.0.4183.102_chrome_updater.exe?cms_redirect=yes&mh=t-&mip=-public-ip-edited-&mm=28&mn=sn-8xgp1vo-p5ql&ms=nvh&mt=1600820539&mv=m&mvi=8&pl=19&shardbypass=yes",** + "version": "1.1", + **"user_agent": "Microsoft BITS/7.8",** + "request_body_len": 0, + "response_body_len": 1392, + **"status_code": 206,** + **"status_msg": "Partial Content",** + "tags": [], + "resp_fuids": [ + **"FGYKX64SkXc4OcvlFf"** + ] + } + +In the previous :file:`http.log` entry we see that the BITS client has made a +GET request for the same file. The server is providing it via “partial +content”, represented by the 206 status code. + +Also note we now have a file UID present in the :file:`http.log`: +``FGYKX64SkXc4OcvlFf``. + +The next :file:`http.log` entry is similar, although the amount of data sent is +different. + +.. literal-emph:: + + { + "ts": "2020-09-23T00:24:35.247333Z", + "uid": "Cq2b9jR12c4lqZafg", + "id.orig_h": "192.168.4.152", + "id.orig_p": 59125, + "id.resp_h": "63.88.73.83", + "id.resp_p": 80, + "trans_depth": 3, + "method": "GET", + "host": "r8---sn-8xgp1vo-p5ql.gvt1.com", + "uri": "/edgedl/release2/chrome/SAWXCyZhLAbPfxC5kv_Fkw_85.0.4183.121/85.0.4183.121_85.0.4183.102_chrome_updater.exe?cms_redirect=yes&mh=t-&mip=-public-ip-edited-&mm=28&mn=sn-8xgp1vo-p5ql&ms=nvh&mt=1600820539&mv=m&mvi=8&pl=19&shardbypass=yes", + "version": "1.1", + "user_agent": "Microsoft BITS/7.8", + "request_body_len": 0, + **"response_body_len": 1995,** + "status_code": 206, + "status_msg": "Partial Content", + "tags": [] + } + +I have removed the half a dozen or so intervening messages as they are very +similar to the preceding entries. I include the last one for reference. It is +similar to the previous entries, although the response body length shows much +more data was sent. + +.. literal-emph:: + + { + "ts": "2020-09-23T00:24:46.547359Z", + "uid": "Cq2b9jR12c4lqZafg", + "id.orig_h": "192.168.4.152", + "id.orig_p": 59125, + "id.resp_h": "63.88.73.83", + "id.resp_p": 80, + "trans_depth": 12, + "method": "GET", + "host": "r8---sn-8xgp1vo-p5ql.gvt1.com", + "uri": "/edgedl/release2/chrome/SAWXCyZhLAbPfxC5kv_Fkw_85.0.4183.121/85.0.4183.121_85.0.4183.102_chrome_updater.exe?cms_redirect=yes&mh=t-&mip=-public-ip-edited-&mm=28&mn=sn-8xgp1vo-p5ql&ms=nvh&mt=1600820539&mv=m&mvi=8&pl=19&shardbypass=yes", + "version": "1.1", + "user_agent": "Microsoft BITS/7.8", + "request_body_len": 0, + **"response_body_len": 652148,** + "status_code": 206, + "status_msg": "Partial Content", + "tags": [] + } + +That concludes the relevant :file:`http.log` entries. Using the file UID we can +search the :file:`files.log` next. + +Continuing with :file:`files.log` +================================= + +The relevant :file:`files.log` entry contains the following: + +.. literal-emph:: + + { + "ts": "2020-09-23T00:24:31.334435Z", + "fuid": "FGYKX64SkXc4OcvlFf", + "uid": "Cq2b9jR12c4lqZafg", + "id.orig_h": "192.168.4.152", + "id.orig_p": 59125, + "id.resp_h": "63.88.73.83", + "id.resp_p": 80, + **"source": "HTTP",** + "depth": 0, + "analyzers": [ + "MD5", + **"PE",** + "SHA1", + "EXTRACT" + ], + **"mime_type": "application/x-dosexec",** + "duration": 15.468528032302856, + "local_orig": false, + "is_orig": false, + "seen_bytes": 1967360, + "total_bytes": 1967360, + "missing_bytes": 0, + "overflow_bytes": 0, + "timedout": false, + **"md5": "a5843bd951f148e99b7265e5bd159fb7",** + "sha1": "fc8b8deb5b34fec1f3f094e579667b2bddee0b21", + **"extracted": "/nsm/zeek/extracted/HTTP-FGYKX64SkXc4OcvlFf.exe",** + "extracted_cutoff": false + } + +This :file:`files.log` entry shows that the content returned by the BITS server +included a Windows executable. Zeek calculates MD5 and SHA1 hashes, and also +shows the location on disk for the extracted file. + +Do you remember a similar entry from the Zeek documentation on +:file:`files.log`? + +:: + + "analyzers": [ + "EXTRACT", + "PE" + ], + +In that example, we have active extract and PE analyzers. + +In the current :file:`files.log`, we have additional analyzers present: + +.. literal-emph:: + + "analyzers": [ + "MD5", + **"PE",** + "SHA1", + "EXTRACT" + ], + +Thanks to these analyzers, we have the MD5 and SHA1 hashes, along with a +:file:`pe.log` entry and an extracted file. + +Continuing with :file:`pe.log` +============================== + +Finally we come to the :file:`pe.log`. We are able to connect it with the +appropriate activity using the file UID ``FGYKX64SkXc4OcvlFf``. + +.. literal-emph:: + + { + "ts": "2020-09-23T00:24:36.395445Z", + **"id": "FGYKX64SkXc4OcvlFf",** + "machine": "AMD64", + **"compile_ts": "2020-09-19T00:10:08.000000Z",** + **"os": "Windows XP x64 or Server 2003",** + **"subsystem": "WINDOWS_GUI",** + **"is_exe": true,** + **"is_64bit": true,** + "uses_aslr": true, + "uses_dep": true, + "uses_code_integrity": false, + "uses_seh": true, + "has_import_table": true, + "has_export_table": false, + "has_cert_table": true, + "has_debug_data": true, + "section_names": [ + ".text", + ".rdata", + ".data", + ".pdata", + ".00cfg", + ".rsrc", + ".reloc" + ] + } + +The compile time is one of the more interesting details for analysts. This is a +freshly compiled Windows executable. + +Reviewing the Extracted Binary +============================== + +As we did in the :file:`files.log` documentation, we can analyze our extracted +file using the command line version of VirusTotal. + +Here is the extracted file on disk. Notice the filename includes the file UID +calculated by Zeek, i.e., ``FGYKX64SkXc4OcvlFf``. + +.. code-block:: console + + $ file /nsm/zeek/extracted/HTTP-FGYKX64SkXc4OcvlFf.exe + +:: + + /nsm/zeek/extracted/HTTP-FGYKX64SkXc4OcvlFf.exe: PE32+ executable (GUI) x86-64, for MS Windows + +We use the Linux :program:`md5sum` utility to calculate the MD5 hash. + +.. code-block:: console + + $ md5sum /nsm/zeek/extracted/HTTP-FGYKX64SkXc4OcvlFf.exe + +:: + + a5843bd951f148e99b7265e5bd159fb7 /nsm/zeek/extracted/HTTP-FGYKX64SkXc4OcvlFf.exe + +Note the MD5 hash matches the one provided by Zeek in the :file:`files.log` +entry. + +Next we submit the hash, not the binary, to VirusTotal for analysis. Whenever +possible, submit hashes to cloud file analysis engines. This preserves the +confidentiality of your sample. + +The output is edited for readability. + +.. code-block:: console + + $ vt file a5843bd951f148e99b7265e5bd159fb7 + +.. literal-emph:: + + - _id: "14a1b9947b77174244a6f6bfd2cd7e1b1c860a09b3b5d74f07b81e45b5548de4" + _type: "file" + authentihash: "a4a6a1011bb3e33af37a1dce19bd41b72d5360dc4175d570ec7260d1d9815747" + **creation_date: 1600474208 # 2020-09-19 00:10:08 +0000 UTC** + **first_submission_date: 1600711798 # 2020-09-21 18:09:58 +0000 UTC** + **last_analysis_date: 1600840562 # 2020-09-23 05:56:02 +0000 UTC** + last_analysis_results: + ALYac: + category: "undetected" + engine_name: "ALYac" + engine_update: "20200923" + engine_version: "1.1.1.5" + method: "blacklist" + ...edited... + eGambit: + category: "undetected" + engine_name: "eGambit" + engine_update: "20200923" + method: "blacklist" + last_analysis_stats: + confirmed-timeout: 0 + failure: 0 + harmless: 0 + malicious: 0 + suspicious: 0 + timeout: 0 + type-unsupported: 4 + undetected: 69 + last_modification_date: 1600878930 # 2020-09-23 16:35:30 +0000 UTC + last_submission_date: 1600830769 # 2020-09-23 03:12:49 +0000 UTC + magic: "PE32+ executable for MS Windows (GUI) Mono/.Net assembly" + md5: "a5843bd951f148e99b7265e5bd159fb7" + **meaningful_name: "mini_installer"** + names: + **- "85.0.4183.121_85.0.4183.102_chrome_updater.exe"** + - "mini_installer" + **- "HTTP-FjcOYuaXbbQFV1cJj.exe"** + pe_info: + entry_point: 4096 + imphash: "ec06ab323a50409817b4a6a54b98f157" + import_list: + - imported_functions: + - "CommandLineToArgvW" + library_name: "SHELL32.dll" + - imported_functions: + - "GetLastError" + - "GetVolumePathNameW" + ...edited... + - "GetEnvironmentVariableW" + library_name: "KERNEL32.dll" + machine_type: 34404 + overlay: + chi2: 1124223.375 + entropy: 4.492208003997803 + filetype: "binary Computer Graphics Metafile" + md5: "ddc7adbbc3760a81d8510e57fedbe055" + offset: 1951232 + size: 16128 + resource_details: + - chi2: 286.0988464355469 + entropy: 7.999892711639404 + filetype: "Data" + lang: "ENGLISH US" + sha256: "133ccfebc6cebb05333ed1677bb419716a8ad00b39417f2f4fa6ee45bdbb92df" + type: "B7" + ...edited... + timestamp: 1600474208 + reputation: 0 + sha1: "fc8b8deb5b34fec1f3f094e579667b2bddee0b21" + sha256: "14a1b9947b77174244a6f6bfd2cd7e1b1c860a09b3b5d74f07b81e45b5548de4" + signature_info: + copyright: "Copyright 2020 Google LLC. All rights reserved." + counter signers: "TIMESTAMP-SHA256-2019-10-15; DigiCert SHA2 Assured ID Timestamping CA; DigiCert" + counter signers details: + - algorithm: "sha256RSA" + cert issuer: "DigiCert SHA2 Assured ID Timestamping CA" + name: "TIMESTAMP-SHA256-2019-10-15" + serial number: "04 CD 3F 85 68 AE 76 C6 1B B0 FE 71 60 CC A7 6D" + status: "Valid" + thumbprint: "0325BD505EDA96302DC22F4FA01E4C28BE2834C5" + valid from: "12:00 AM 10/01/2019" + valid to: "12:00 AM 10/17/2030" + valid usage: "Timestamp Signing" + - algorithm: "sha256RSA" + cert issuer: "DigiCert Assured ID Root CA" + name: "DigiCert SHA2 Assured ID Timestamping CA" + serial number: "0A A1 25 D6 D6 32 1B 7E 41 E4 05 DA 36 97 C2 15" + status: "Valid" + thumbprint: "3BA63A6E4841355772DEBEF9CDCF4D5AF353A297" + valid from: "12:00 PM 01/07/2016" + valid to: "12:00 PM 01/07/2031" + valid usage: "Timestamp Signing" + - algorithm: "sha1RSA" + cert issuer: "DigiCert Assured ID Root CA" + name: "DigiCert" + serial number: "0C E7 E0 E5 17 D8 46 FE 8F E5 60 FC 1B F0 30 39" + status: "Valid" + thumbprint: "0563B8630D62D75ABBC8AB1E4BDFB5A899B24D43" + valid from: "12:00 AM 11/10/2006" + valid to: "12:00 AM 11/10/2031" + valid usage: "Client Auth, Code Signing, Email Protection, Server Auth, Timestamp Signing" + **description: "Google Chrome Installer"** + **file version: "85.0.4183.121"** + **internal name: "mini_installer"** + **product: "Google Chrome Installer"** + signers: "Google LLC; DigiCert SHA2 Assured ID Code Signing CA; DigiCert" + signers details: + - algorithm: "sha256RSA" + cert issuer: "DigiCert SHA2 Assured ID Code Signing CA" + name: "Google LLC" + serial number: "0C 15 BE 4A 15 BB 09 03 C9 01 B1 D6 C2 65 30 2F" + status: "Valid" + thumbprint: "CB7E84887F3C6015FE7EDFB4F8F36DF7DC10590E" + valid from: "12:00 AM 11/07/2018" + valid to: "12:00 PM 11/17/2021" + valid usage: "Code Signing" + ...edited... + ssdeep: "49152:zS2WLLoAgkZlbpkJDy5KrwM4wN9UT90hZv6AFV56vt9IWA:m2WvgSbpkFAKrwMpTZJV5kgW" + tags: + - "peexe" + - "assembly" + - "overlay" + - "runtime-modules" + - "signed" + - "64bits" + - "trusted" + times_submitted: 2 + total_votes: + harmless: 0 + malicious: 0 + trid: + - file_type: "OS/2 Executable (generic)" + probability: 33.6 + - file_type: "Generic Win/DOS Executable" + probability: 33.1 + - file_type: "DOS Executable Generic" + probability: 33.1 + **trusted_verdict:** + **filename: "85.0.4183.121_85.0.4183.102_chrome_updater.exe"** + **link: "https://dl.google.com/dl/release2/chrome/SAWXCyZhLAbPfxC5kv_Fkw_85.0.4183.121/85.0.4183.121_85.0.4183.102_chrome_updater.exe"** + **organization: "Google"** + **verdict: "goodware"** + type_description: "Win32 EXE" + type_tag: "peexe" + unique_sources: 2 + vhash: "016076651d151515751az36hz1lz" + +This file appears to be a component of the Google Chrome Installer. It is not +malicious software. + +Conclusion +========== + +Although the :file:`pe.log` was only part of this section, I wanted to show an +integrated set of Zeek logs for this example, beginning with the +:file:`conn.log`, continuing with the :file:`http.log` and :file:`files.log`, +and concluding with the :file:`pe.log`. This is recent activity and shows that +modern software still uses HTTP in some cases! diff --git a/doc/logs/postgresql.rst b/doc/logs/postgresql.rst new file mode 100644 index 0000000000..6477c01809 --- /dev/null +++ b/doc/logs/postgresql.rst @@ -0,0 +1,97 @@ +.. _PostgreSQL protocol: https://www.postgresql.org/docs/current/protocol.html + +============== +postgresql.log +============== + +.. versionadded:: 7.1 + +Overview +======== + +Zeek contains a basic spicy-based `PostgreSQL protocol`_ analyzer. + +Example +======= + +An example of :file:`postgresql.log`. + +.. code-block:: console + + $ zeek -C LogAscii::use_json=T -r psql-create-insert-select-delete-drop.pcap + $ jq < postgresql.log + { + "ts": 1725368066.79174, + "uid": "C68Wxi3EStaTmxaUVl", + "id.orig_h": "127.0.0.1", + "id.orig_p": 40190, + "id.resp_h": "127.0.0.1", + "id.resp_p": 5432, + "user": "postgres", + "database": "postgres", + "application_name": "psql", + "frontend": "simple_query", + "frontend_arg": "CREATE TABLE IF NOT EXISTS t (i int, s varchar, t time);", + "success": true, + "rows": 0 + } + { + "ts": 1725368066.80694, + "uid": "C68Wxi3EStaTmxaUVl", + "id.orig_h": "127.0.0.1", + "id.orig_p": 40190, + "id.resp_h": "127.0.0.1", + "id.resp_p": 5432, + "user": "postgres", + "database": "postgres", + "application_name": "psql", + "frontend": "simple_query", + "frontend_arg": "INSERT INTO t VALUES (42, 'forty-two', now());", + "success": true, + "rows": 0 + } + + +:zeek:see:`PostgreSQL::Info` provides further details about the current output of the +:file:`postgresql.log`. + +TLS +=== + +The PostgreSQL protocol provides a mechanism to upgrade client-server connections +to TLS. The analyzer detects this mechanism and hands off analysis to Zeek's +TLS analyzer. The :file:`postgresql.log` and :file:`conn.log` files will look +as follows: + +.. code-block:: console + + $ zeek -C LogAscii::use_json=T -r testing/btest/Traces/postgresql/psql-aws-ssl-preferred.pcap + $ jq < postgresql.log + { + "ts": 1670520068.267888, + "uid": "CAcbxM1ou0N1V2cGpe", + "id.orig_h": "192.168.123.132", + "id.orig_p": 39910, + "id.resp_h": "52.200.36.167", + "id.resp_p": 5432, + "frontend": "ssl_request", + "backend": "ssl_reply", + "backend_arg": "S", + "success": true + } + + $ jq < conn.log + { + "ts": 1670520068.15752, + "uid": "CAcbxM1ou0N1V2cGpe", + "id.orig_h": "192.168.123.132", + "id.orig_p": 39910, + "id.resp_h": "52.200.36.167", + "id.resp_p": 5432, + "proto": "tcp", + "service": "postgresql,ssl", + "duration": 0.931433916091919, + "orig_bytes": 786, + "resp_bytes": 4542, + ... + } diff --git a/doc/logs/quic.rst b/doc/logs/quic.rst new file mode 100644 index 0000000000..4bcd21ea3c --- /dev/null +++ b/doc/logs/quic.rst @@ -0,0 +1,88 @@ +======== +quic.log +======== + +.. versionadded:: 6.1 + +Overview +======== + +The QUIC protocol integrates encryption, stream multiplexing and flow control at +the transport layer. QUIC uses TLS 1.3 by default. Zeek's QUIC analyzer +provides greater observability into the protocol's TLS handshake. + + +Example +======= + +An example of a :file:`quic.log`. + +.. code-block:: console + + zeek@zeek-6.1:~ zeek -C LogAscii::use_json=T -r chromium-115.0.5790.110-api-cirrus-com.pcap + zeek@zeek-6.1:~ jq . quic.log + +:: + + { + "ts": 1692198386.837988, + "uid": "CA482y1XJVd3d0RYI7", + "id.orig_h": "82.239.54.117", + "id.orig_p": 53727, + "id.resp_h": "110.213.53.115", + "id.resp_p": 443, + "version": "1", + "client_initial_dcid": "95412c47018cdfe8", + "server_scid": "d5412c47018cdfe8", + "server_name": "api.cirrus-ci.com", + "client_protocol": "h3", + "history": "ISisH" + } + + +:zeek:see:`QUIC::Info` provides further details on the current output of the +:file:`quic.log`. Current fields include: + +- **version**: A string interpretation of the QUIC version number, usually "1" + or "quicv2". + +- **client_initial_dcid**: When QUIC initiates a connection it uses Random + Number Generators to create the first Destination Connection ID (DCID). This + DCID is subsequently used for routing and packet protection by client and + server. + +- **server_scid**: A QUIC-supported server responds to a DCID by selecting a + Source Connection ID (SCID). This usually occurs within the server’s first + ``INITIAL`` packet. This is typically used by the client in subsequent + packets, although the SCID can change to adapt to new network conditions. + +- **client_protocol**: If the ``ClientHello`` packet is successfully extracted + and contains the ALPN extension, the extension's first entry is placed in + ``client_protocol``. + +- **history**: Provides a history of QUIC protocol activity in a connection, + similar to the history fields in conn.log and ssh.log. See the + :zeek:see:`QUIC::Info` documentation for details. In the example above, + the history outlines: + + + An initial packet from the client (I) - a new connection + + + An TLS ``ClientHello`` from the client (S) - the start of a + TLS handshake + + + An initial packet from the server (i) - an acknowledgement + from the server of the new connection + + + A TLS ServerHello response from the server (s) - the + selection of a cipher suite from the options provided by the + client + + + A handshake packet from the client (H) + + +Conclusion +========== + +The QUIC analyzer provides some observability into QUIC network traffic, +particularly around connection establishment. Introduced in version 6.1, it's +one of Zeek's newer parsers, so feedback is particularly welcome. diff --git a/doc/logs/rdp.rst b/doc/logs/rdp.rst new file mode 100644 index 0000000000..cfe17bf0ab --- /dev/null +++ b/doc/logs/rdp.rst @@ -0,0 +1,216 @@ +======= +rdp.log +======= + +Remote Desktop Protocol (RDP) is a protocol Microsoft developed to enable +remote graphical communication. RDP implementations exist for other operating +systems, but RDP is most popular on systems running Windows NT 4.0 and newer. + +Older versions of RDP are unencrypted, while newer versions offer SSL and TLS +encryption. + +Standard RDP servers listen on port 3389 TCP. Administrators can configure the +service to listen on any port, however. The following material investigates the +process by which a simulated intruder gains access to a system via RDP. First +he makes many connections to the RDP server, testing usernames and passwords. +Following the correct guessing of a username and password, he connects and +briefly interacts with the system offering access via RDP. + +For full details on each field in the :file:`rdp.log` file, please refer to +:zeek:see:`RDP::Info`. + +:file:`conn.log` +================ + +Let’s start with the :file:`conn.log` for the activity in question. I’ve broken +it into two sets of activities. The first is the reconnaissance and the second +is the interactive session. + +I’ve summarized the first set of :file:`conn.log` entries using the following syntax: + +.. code-block:: console + + $ jq -c '[."id.orig_h", ."id.resp_h", ."id.resp_p", ."service", ."orig_bytes", ."resp_bytes"]' conn.log | sort | uniq -c + +:: + + 38 ["192.168.4.160","192.168.4.161",3389,"ssl",1392,1238] + 1 ["192.168.4.160","192.168.4.161",3389,"ssl",3365,4855] + +We see 38 sessions which contain the same number of bytes sent and received by +the client and server, and 1 session which contains a different number of +bytes. That could indicate a successful connection. Port 3389 TCP is the +destination, but remember that any TCP port could host a RDP server. Also note +Zeek reports the service as SSL, because this RDP session is encrypted by TLS. + +The second set of :file:`conn.log` entries contains the following session: + +.. literal-emph:: + + { + "ts": 1607353272.790635, + "uid": "CFdEZNjN5MtPzGMS8", + **"id.orig_h": "192.168.4.160",** + "id.orig_p": 59758, + **"id.resp_h": "192.168.4.161",** + **"id.resp_p": 3389,** + "proto": "tcp", + **"service": "ssl",** + "duration": 109.49137687683105, + **"orig_bytes": 66747,** + **"resp_bytes": 1823511,** + "conn_state": "RSTR", + "missed_bytes": 0, + "history": "ShADdaFr", + "orig_pkts": 2913, + "orig_ip_bytes": 183287, + "resp_pkts": 2250, + "resp_ip_bytes": 1913523 + } + +This activity is similar to the previous, except that the client and server +have sent many more bytes of data. + +:file:`rdp.log` +=============== + +The following syntax summarizes the relevant content in the first set of Zeek +:file:`rdp.log` entries, caused by the simulated intruder’s RDP reconnaissance: + +.. code-block:: console + + $ jq -c '[."id.orig_h", ."id.resp_h", ."id.resp_p", ."cookie", ."result", ."security_protocol", ."cert_count"]' rdp.log | sort | uniq -c + +:: + + 39 ["192.168.4.160","192.168.4.161",3389,"test","encrypted","HYBRID",0] + +There is nothing in these logs to indicate whether the session was successful +or not. However, Zeek was able to determine that RDP was in use, based on its +recognition of the protocol. + +Here is the entire :file:`rdp.log` entry for the interactive RDP session: + +.. literal-emph:: + + { + "ts": 1607353272.791158, + "uid": "CFdEZNjN5MtPzGMS8", + **"id.orig_h": "192.168.4.160",** + "id.orig_p": 59758, + **"id.resp_h": "192.168.4.161",** + **"id.resp_p": 3389,** + "cookie": "test", + "result": "encrypted", + "security_protocol": "HYBRID", + "cert_count": 0 + } + +As before, there is nothing stating that this is an interactive session. + +:file:`ssl.log` and :file:`x509.log` +==================================== + +The Zeek logs associated with TLS-encrypted sessions might tell us a bit about +the RDP server. Here is a :file:`ssl.log` entry for the interactive session: + +.. literal-emph:: + + { + "ts": 1607353272.79572, + "uid": "CFdEZNjN5MtPzGMS8", + **"id.orig_h": "192.168.4.160",** + "id.orig_p": 59758, + **"id.resp_h": "192.168.4.161",** + **"id.resp_p": 3389,** + **"version": "TLSv12",** + **"cipher": "TLS_RSA_WITH_AES_256_GCM_SHA384",** + **"server_name": "192.168.4.161",** + "resumed": false, + "established": true, + "cert_chain_fuids": [ + **"FWesoX2H43hXhuqoGb"** + ], + "client_cert_chain_fuids": [], + **"subject": "CN=WinDev2010Eval",** + **"issuer": "CN=WinDev2010Eval"** + } + +From this information it looks like the target is a Windows development server. + +Here is the corresponding :file:`x509.log` entry. We match it to the preceding +:file:`ssl.log` entry using the ``id`` field. + +.. literal-emph:: + + { + "ts": 1607353272.79572, + **"id": "FWesoX2H43hXhuqoGb",** + "certificate.version": 3, + "certificate.serial": "5578FF9983F26AA6442533AB6AD54C72", + **"certificate.subject": "CN=WinDev2010Eval",** + **"certificate.issuer": "CN=WinDev2010Eval",** + "certificate.not_valid_before": 1602434171, + "certificate.not_valid_after": 1618245371, + "certificate.key_alg": "rsaEncryption", + "certificate.sig_alg": "sha256WithRSAEncryption", + "certificate.key_type": "rsa", + "certificate.key_length": 2048, + "certificate.exponent": "65537" + } + +While this might have some significance in other investigations, here it is not +as important. + +Running the Test +================ + +For those who might want to simulate this activity themselves, I wanted to +share how I conducted this experiment. + +.. code-block:: console + + $ hydra -t 1 -V -f -l test -P wordlist.txt rdp://192.168.4.161 + +.. literal-emph:: + + Hydra v9.1 (c) 2020 by van Hauser/THC & David Maciejak - Please do not use in military or secret service organizations, or for illegal purposes (this is non-binding, these *** ignore laws and ethics anyway). + + Hydra (https://github.com/vanhauser-thc/thc-hydra) starting at 2020-12-07 09:46:30 + [WARNING] the rdp module is experimental. Please test, report - and if possible, fix. + [DATA] max 1 task per 1 server, overall 1 task, 4999 login tries (l:1/p:4999), ~4999 tries per task + [DATA] attacking rdp://192.168.4.161:3389/ + [ATTEMPT] target 192.168.4.161 - login "test" - pass "123456" - 1 of 4999 [child 0] (0/0) + [ATTEMPT] target 192.168.4.161 - login "test" - pass "12345" - 2 of 4999 [child 0] (0/0) + [ATTEMPT] target 192.168.4.161 - login "test" - pass "123456789" - 3 of 4999 [child 0] (0/0) + [ATTEMPT] target 192.168.4.161 - login "test" - pass "password" - 4 of 4999 [child 0] (0/0) + ...edited... + [ATTEMPT] target 192.168.4.161 - login "test" - pass "liverpool" - 38 of 4999 [child 0] (0/0) + **[ATTEMPT] target 192.168.4.161 - login "test" - pass "football" - 39 of 4999 [child 0] (0/0)** + **[3389][rdp] host: 192.168.4.161 login: test password: football** + [STATUS] attack finished for 192.168.4.161 (valid pair found) + **1 of 1 target successfully completed, 1 valid password found** + Hydra (https://github.com/vanhauser-thc/thc-hydra) finished at 2020-12-07 09:46:53 + +I used the reconnaissance tool THC-Hydra by van Hauser/THC & David Maciejak. I +provided a word list that had a password that I had enabled on a test account +on the Windows RDP server at ``192.168.4.161``. I ran Hydra from a Kali Linux +virtual machine against a Windows 10 development virtual machine and captured +the traffic on Kali Linux. I then processed it with Zeek to produce the logs in +this section. + +Conclusion +========== + +When processing unencrypted RDP sessions, Zeek can provide a bit more +information than that provided here. However, in my experience Zeek is most +helpful for identifying systems which should or should not be offering RDP +services. Zeek will also generate records for interactive sessions, helping +analysts identify when authorized or unauthorized users access systems via RDP. + +For more information on analyzing RDP in context of vulnerabilities that +appeared in 2020, please see the following blog posts: + +https://corelight.blog/2019/05/23/how-to-use-corelight-and-zeek-logs-to-mitigate-rds-rdp-vulnerabilities/ + +https://corelight.blog/2020/05/13/analyzing-encrypted-rdp-connections/ diff --git a/doc/logs/smb.rst b/doc/logs/smb.rst new file mode 100644 index 0000000000..7241168585 --- /dev/null +++ b/doc/logs/smb.rst @@ -0,0 +1,1825 @@ + +.. _zkg package manager: https://docs.zeek.org/projects/package-manager/en/stable/ + +======================================= +SMB Logs (plus DCE-RPC, Kerberos, NTLM) +======================================= + +Server Message Block (SMB) is a protocol most commonly associated with +Microsoft Windows enterprise administration. While there are implementations +for other operating systems, such as Linux, Mac OS, FreeBSD, and the like, many +security and network analysts seek information on SMB due to its use in Windows +environments. + +Introduction +============ + +For the most part, the log analyses in this section address a single +Zeek log, such as :file:`conn.log` or :file:`dns.log`. When Zeek encounters SMB +protocol usage, it usually creates multiple logs of varying types. In addition +to the ubiquitous :file:`conn.log`, Zeek may generate :file:`dce_rpc.log`, +:file:`kerberos.log`, :file:`ntlm.log`, :file:`smb_cmd.log`, +:file:`smb_files.log`, :file:`smb_mapping.log`, :file:`pe.log`, and even +:file:`notice.log` entries. + +This section will build upon a paper by Nate Marx published December 20, 2017 +titled “An Introduction to SMB for Network Security Analysts.” The paper +analyzes a set of packet captures that contain activity in a simulated +compromised Windows environment. + +The paper is available here: + +https://401trg.github.io/pages/an-introduction-to-smb-for-network-security-analysts.html + +The packet captures are available here: + +https://github.com/401trg/detections/tree/master/pcaps + +Thorough documentation of several versions of SMB are available online thanks +to Microsoft. + +SMB version 1 is posted here: + +https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-smb/f210069c-7086-4dc2-885e-861d837df688 + +SMB versions 2 and 3 are posted here: + +https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-smb2/5606ad47-5ee0-437a-817e-70c366052962 + +For information on the individual field values in these SMB-affiliated logs, +please refer to :zeek:see:`DCE_RPC::Info`, :zeek:see:`KRB::Info`, +:zeek:see:`NTLM::Info`, :zeek:see:`SMB::CmdInfo`, :zeek:see:`SMB::FileInfo`, +and :zeek:see:`SMB::TreeInfo`. + +When presenting information in this section, my general convention is to bold +commands and items of interest in the resulting output. + +Leveraging BZAR +=============== + +Before looking at individual logs associated with SMB, it’s helpful to first +consider adding the BZAR package to your repertoire. + +BZAR stands for Bro/Zeek ATT&CK-based Analytics and Reporting. Mark Fernandez +and others from MITRE and the Zeek community wrote BZAR to generate +:file:`notice.log` entries when certain patterns of activity appear in some SMB +logs. + +You can learn more about BZAR at https://github.com/mitre-attack/bzar and install +it via the `zkg package manager`_ by saying + +.. literal-emph:: + + zkg install bzar + +I suggest using BZAR when one first begins looking at SMB logs. Without BZAR, +it could be difficult to know what might be worth investigating and what might +be normal. However, even with BZAR, it is no easy feat to differentiate among +normal, suspicious, and malicious SMB activity. Still, leveraging the BZAR +policy script for Zeek will give analysts a place to begin their +investigations. + +Running the ``net user`` Command +================================ + +Let’s start our investigation of SMB logs with the case labelled “RPC” in Nate +Marx’s paper. The relevant packet capture file is titled +:file:`20171220_smb_net_user.pcap`. + +If we process the packet capture with Zeek and BZAR, the following files appear: + +* :file:`conn.log` +* :file:`dce_rpc.log` +* :file:`kerberos.log` +* :file:`notice.log` +* :file:`packet_filter.log` +* :file:`smb_mapping.log` + +Let’s look at the :file:`conn.log` first to get a general overview of the +traffic. + +.. literal-emph:: + + { + "ts": 1507562478.10937, + "uid": "CzgIrZ31Lh5vCHioWi", + **"id.orig_h": "192.168.10.31",** + "id.orig_p": 49282, + **"id.resp_h": "192.168.10.10",** + **"id.resp_p": 445,** + "proto": "tcp", + "service": "gssapi,smb,dce_rpc,krb", + "duration": 0.22932004928588867, + "orig_bytes": 16271, + "resp_bytes": 13720, + "conn_state": "S1", + "missed_bytes": 0, + "history": "ShADda", + "orig_pkts": 78, + "orig_ip_bytes": 19403, + "resp_pkts": 77, + "resp_ip_bytes": 16812, + "ip_proto": 6 + } + +We see that ``192.168.10.31`` initiated a connection to ``192.168.10.10``. The +destination port is 445 TCP, which is associated with SMB activity. Note that +Zeek observed the services on this connection as ``gssapi,smb,dce_rpc,krb``, +which represents Generic Security Service Application Programming Interface, +Server Message Block, Distributed Computing Environment Remote Procedure Call, +and Kerberos. + +The GSS-API reference likely relates to authentication, as noted in the Windows +protocol guide for SMB versions 2 and 3. It does not produce any logs named +``gssapi``. SMB is expected as we are looking for it in this case, and will +create smb-named logs. DCE-RPC is a protocol associated with Windows networking +and command execution between machines, and will likely create a +:file:`dce_rpc.log` entry. Kerberos is an authentication protocol that will +likely create a :file:`kerberos.log` entry. + +:file:`notice.log` +------------------ + +Let’s see what the :file:`notice.log` has to say about this activity. + +.. literal-emph:: + + { + "ts": 1507562478.117387, + **"note": "ATTACK::Discovery",** + **"msg": "Detected activity from host 192.168.10.31, total attempts 5 within timeframe 5.0 mins",** + "actions": [ + "Notice::ACTION_LOG" + ], + "suppress_for": 3600 + } + { + "ts": 1507562478.124176, + **"note": "ATTACK::Discovery",** + **"msg": "Detected activity from host 192.168.10.31, total attempts 10 within timeframe 5.0 mins",** + "actions": [ + "Notice::ACTION_LOG" + ], + "suppress_for": 3600 + } + { + "ts": 1507562478.138992, + **"note": "ATTACK::Discovery",** + **"msg": "Detected activity from host 192.168.10.31, total attempts 15 within timeframe 5.0 mins",** + "actions": [ + "Notice::ACTION_LOG" + ], + "suppress_for": 3600 + } + +These three entries all indicate the same sort of activity: ``192.168.10.31`` +is doing some sort of “discovery” action. We do not know the nature of the +reconnaissance nor do we know the target. However, when combined with the +:file:`conn.log` we saw previously, we can assume that ``192.168.10.10`` is the +target. + +:file:`dce_rpc.log` +------------------- + +The :file:`notice.log` alerted us to suspicious or malicious activity from +``192.168.10.31``. Perhaps the :file:`dce_rpc.log` can help us understand what +is happening? + +Let’s look at the first entry in :file:`dce_rpc.log`. + +.. literal-emph:: + + { + "ts": 1507562478.112879, + "uid": "CzgIrZ31Lh5vCHioWi", + **"id.orig_h": "192.168.10.31",** + "id.orig_p": 49282, + **"id.resp_h": "192.168.10.10",** + **"id.resp_p": 445,** + "rtt": 0.0003020763397216797, + **"named_pipe": "\\pipe\\lsass",** + **"endpoint": "samr",** + **"operation": "SamrConnect5"** + } + +This entry shows that ``192.168.10.31`` connected to ``192.168.10.10`` via a +named pipe titled ``lsass``. Microsoft’s documentation says “a pipe is a +section of shared memory that processes use for communication. The process that +creates a pipe is the pipe server. A process that connects to a pipe is a pipe +client… Named pipes can be used to provide communication between processes on +the same computer or between processes on different computers across a +network.” + +Ref: https://docs.microsoft.com/en-us/windows/win32/ipc/pipes + +The lsass named pipe refers to the Local Security Authority Subsystem Service +(LSASS). The endpoint, ``samr``, refers to the Security Accounts Manager. +Microsoft’s documentation says “the SamrConnect5 method obtains a handle to a +server object.” + +Ref: https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-samr/c842a897-0a42-4ca5-a607-2afd05271dae + +Even if you do not fully understand all of these details (and who does!), it +appears that ``192.168.10.31`` is trying to remotely access ``192.168.10.10`` +in a way that requires security authentication on the client, via DCE-RPC over +SMB. + +All of the entries in the :file:`dce_rpc.log` have the same source and +destination addresses and ports. We can summarize them by extracting only the +relevant fields using :program:`jq`: + +If we look at every one of the 46 entries in the :file:`dce_rpc.log`, we will +see repeats of some commands. These do not add to our general understanding of +what is happening. To show a reduced set of commands, I invoke :program:`jq` +and pipe the output through uniq to only show unique outputs: + +.. code-block:: console + + $ jq -c '[."named_pipe", ."endpoint", ."operation"]' dce_rpc.log | uniq + +.. literal-emph:: + + ["\\pipe\\lsass","samr","SamrConnect5"] + ["\\pipe\\lsass","samr","**SamrEnumerateDomainsInSamServer**"] + ["\\pipe\\lsass","samr","SamrLookupDomainInSamServer"] + ["\\pipe\\lsass","samr","SamrOpenDomain"] + ["\\pipe\\lsass","samr","**SamrLookupNamesInDomain**"] + ["\\pipe\\lsass","samr","SamrOpenUser"] + ["\\pipe\\lsass","samr","**SamrQueryInformationUser**"] + ["\\pipe\\lsass","samr","SamrQuerySecurityObject"] + ["\\pipe\\lsass","samr","**SamrGetGroupsForUser**"] + ["\\pipe\\lsass","samr","SamrGetAliasMembership"] + ["\\pipe\\lsass","samr","SamrCloseHandle"] + ["\\pipe\\lsass","samr","SamrConnect5"] + ["\\pipe\\lsass","samr","SamrEnumerateDomainsInSamServer"] + ["\\pipe\\lsass","samr","SamrLookupDomainInSamServer"] + ["\\pipe\\lsass","samr","SamrOpenDomain"] + ["\\pipe\\lsass","samr","SamrQueryInformationDomain"] + ["\\pipe\\lsass","samr","SamrCloseHandle"] + ["\\pipe\\lsass","lsarpc","LsarOpenPolicy2"] + ["\\pipe\\lsass","lsarpc","LsarQueryInformationPolicy"] + ["\\pipe\\lsass","samr","SamrConnect5"] + ["\\pipe\\lsass","samr","SamrOpenDomain"] + ["\\pipe\\lsass","samr","SamrCloseHandle"] + ["\\pipe\\lsass","lsarpc","LsarLookupNames3"] + ["\\pipe\\lsass","samr","SamrGetAliasMembership"] + ["\\pipe\\lsass","samr","SamrCloseHandle"] + ["\\pipe\\lsass","lsarpc","LsarClose"] + ["\\pipe\\lsass","samr","SamrConnect5"] + ["\\pipe\\lsass","samr","SamrEnumerateDomainsInSamServer"] + ["\\pipe\\lsass","samr","SamrLookupDomainInSamServer"] + ["\\pipe\\lsass","samr","SamrOpenDomain"] + ["\\pipe\\lsass","samr","SamrLookupNamesInDomain"] + ["\\pipe\\lsass","samr","SamrOpenUser"] + ["\\pipe\\lsass","samr","SamrGetGroupsForUser"] + ["\\pipe\\lsass","samr","SamrLookupIdsInDomain"] + ["\\pipe\\lsass","samr","SamrCloseHandle"] + +The bolded entries indicate that ``192.168.10.31`` is performing some sort of +user enumeration against ``192.168.10.10``. Again, we don’t necessarily know +exactly what all of this means, but if there is no reason from +``192.168.10.31`` to be performing this action, then it’s worth investigating! + +:file:`kerberos.log` and :file:`smb_mapping.log` +------------------------------------------------ + +Let’s see if the :file:`kerberos.log` has anything new to add to our +investigation. + +.. literal-emph:: + + { + "ts": 1507562478.110863, + "uid": "CzgIrZ31Lh5vCHioWi", + **"id.orig_h": "192.168.10.31",** + "id.orig_p": 49282, + **"id.resp_h": "192.168.10.10",** + **"id.resp_p": 445** + } + +These are the same details we found through the :file:`conn.log`, but it +confirms that Zeek identified Kerberos authentication in use. + +The :file:`smb_mapping.log` offers one entry as well: + +.. literal-emph:: + + { + "ts": 1507562478.111677, + "uid": "CzgIrZ31Lh5vCHioWi", + "id.orig_h": "192.168.10.31", + "id.orig_p": 49282, + "id.resp_h": "192.168.10.10", + "id.resp_p": 445, + **"path": "\\\\DC1.contoso.local\\IPC$",** + "share_type": "PIPE" + } + +Here we see the first mention of the ``IPC$`` share. As noted in Mr. Marx’s +paper, Windows uses the ``IPC$`` share as a means to enable remote procedure +calls. We knew this was the case when we reviewed the :file:`dce_rpc.log`. It’s +possible that the ``DC1`` in the path value for this log means that +``192.168.10.10`` is a domain controller. It’s likely that there is user +reconnaissance occurring. + +If we look at the explanation for this activity noted in Mr. Marx’s paper, he +says that a simulated intruder on ``192.168.10.31`` executed the ``net user`` +command against ``192.168.10.10``. The intruder took this action to enumerate +the user list on the target. + +In the next two cases we will see what it looks like when simulated intruders move files from one system to another. + +Connecting to a SMB Share and Uploading a File +============================================== + +We continue our exploration of SMB logs by reviewing the first case discussed +in Mr. Marx’s paper. The relevant packet capture file is titled +:file:`20171220_smb_mimikatz_copy.pcap`. Mr. Marx’s discussion appears in the +section “The Basics” in his paper. + +If we process the packet capture with Zeek and BZAR, the following files appear: + +* :file:`conn.log` +* :file:`extract_files/` +* :file:`files.log` +* :file:`kerberos.log` +* :file:`notice.log` +* :file:`packet_filter.log` +* :file:`pe.log` +* :file:`smb_files.log` +* :file:`smb_mapping.log` + +Let’s look at the :file:`conn.log` first to get a general overview of the +traffic. + +:file:`conn.log` +---------------- + +The :file:`conn.log` has two entries: + +.. literal-emph:: + + { + "ts": 1507565438.203425, + "uid": "CR7Vww4LuLkMzi4jMd", + **"id.orig_h": "192.168.10.31",** + "id.orig_p": 49238, + **"id.resp_h": "192.168.10.30",** + **"id.resp_p": 445,** + "proto": "tcp", + **"service": "krb,smb,gssapi",** + "duration": 1.1398930549621582, + "orig_bytes": 814051, + "resp_bytes": 11657, + "conn_state": "S1", + "missed_bytes": 0, + "history": "ShADda", + "orig_pkts": 66, + "orig_ip_bytes": 816703, + "resp_pkts": 91, + "resp_ip_bytes": 15309, + "ip_proto": 6 + } + { + "ts": 1507565425.183882, + "uid": "CyeWAg1QrRKQL0HHMi", + "id.orig_h": "192.168.10.30", + "id.orig_p": 138, + **"id.resp_h": "192.168.10.255",** + **"id.resp_p": 138,** + "proto": "udp", + "conn_state": "S0", + "missed_bytes": 0, + "history": "D", + "orig_pkts": 1, + "orig_ip_bytes": 207, + "resp_pkts": 0, + "resp_ip_bytes": 0, + "ip_proto": 17 + } + +The first entry shows a connection initiated by ``192.168.10.31`` to +``192.168.10.30``. + +The second entry is likely a SMB-related Windows broadcast, as seen by the +destination IP address of ``192.168.10.255``. According to a Wireshark decode +of that datagram, it’s a Windows Browser Protocol message, namely a "Become +backup browser" command with the "browser to promote" being "VICTIM-PC". +“Browser” in this case does not refer to a Web browser; it’s about accessing +resources on the local network. + +Let’s next turn to the :file:`notice.log`. + +:file:`notice.log` +------------------ + +I have selected examples of the two unique log types appearing in +:file:`notice.log`. + +.. literal-emph:: + + { + "ts": 1507565439.130425, + **"uid": "CR7Vww4LuLkMzi4jMd",** + **"id.orig_h": "192.168.10.31",** + "id.orig_p": 49238, + **"id.resp_h": "192.168.10.30",** + "id.resp_p": 445, + "proto": "tcp", + **"note": "ATTACK::Lateral_Movement",** + **"msg": "Detected SMB::FILE_WRITE to admin file share '\\\\admin-pc\\c$temp\\mimikatz.exe'",** + **"sub": "T1021.002 Remote Services: SMB/Windows Admin Shares + T1570 Lateral Tool Transfer",** + **"src": "192.168.10.31",** + **"dst": "192.168.10.30",** + "p": 445, + "actions": [ + "Notice::ACTION_LOG" + ], + "suppress_for": 3600 + } + + { + "ts": 1507565439.343318, + "uid": "CR7Vww4LuLkMzi4jMd", + "id.orig_h": "192.168.10.31", + "id.orig_p": 49238, + "id.resp_h": "192.168.10.30", + "id.resp_p": 445, + "fuid": "FwVZpk12AKBjE11UNg", + "file_mime_type": "application/x-dosexec", + "file_desc": "temp", + "proto": "tcp", + **"note": "ATTACK::Lateral_Movement_Extracted_File",** + **"msg": "Saved a copy of the file written to SMB admin file share",** + **"sub": "CR7Vww4LuLkMzi4jMd_FwVZpk12AKBjE11UNg__admin-pc_c$temp_mimikatz.exe",** + **"src": "192.168.10.31",** + **"dst": "192.168.10.30",** + "p": 445, + "actions": [ + "Notice::ACTION_LOG" + ], + "suppress_for": 3600 + } + +My processing of the packet capture produced 13 of the first entry and 1 of the +second entry. + +These two entries in the :file:`notice.log` tell us a lot, but also provide +material for additional investigation. + +First, the note, msg, and sub entries of each log provide useful information. + +Both notes relate to “lateral movement.” If a new analyst is not familiar with +that term, the sub field in the first log entry provides a reference to “T1570 +Lateral Tool Transfer.” T1570 refers to the MITRE ATT&CK technique number 1570, +which is described here: + +https://attack.mitre.org/techniques/T1570/ + +The ATT&CK Web site explains Lateral Tool Transfer thus: + + “**Adversaries may transfer tools or other files between systems in a + compromised environment**. Files may be copied from one system to another to + stage adversary tools or other files over the course of an operation. + Adversaries may copy files laterally between internal victim systems to + support lateral movement using inherent file sharing protocols such as file + sharing over **SMB** to connected network shares or with authenticated + connections with **SMB/Windows Admin Shares** or Remote Desktop Protocol. Files + can also be copied over on Mac and Linux with native tools like scp, rsync, + and sftp.” (emphasis added) + +With this understanding, the msg from the first log makes more sense:: + + Detected SMB::FILE_WRITE to admin file share '\\\\admin-pc\\c$temp\\mimikatz.exe' + +Zeek is trying to tell us that the BZAR script detected a transfer of a file +called ``mikikatz.exe``. + +The details from the second log tell us what actions Zeek took when it noticed +this activity:: + + "msg": "Saved a copy of the file written to SMB admin file share", + "sub": "CR7Vww4LuLkMzi4jMd_FwVZpk12AKBjE11UNg__admin-pc_c$temp_mimikatz.exe", + +This means we should be able to look in a directory associated with our run of +Zeek to find an extracted copy of this file. + +Finally, as with many Zeek logs, we have an id (in this case, +``CR7Vww4LuLkMzi4jMd``), and IP addresses which we can use to pivot through other +Zeek data. Note the src and dst entries in both logs indicate that +``192.168.10.31`` copied a file to ``192.168.10.30``. + +:file:`extract_files/`, :file:`files.log`, and :file:`pe.log`, and VirusTotal +----------------------------------------------------------------------------- + +Next, let’s look for the extracted file. We can use the Linux :program:`file` +command to get some details: + +.. code-block:: console + + $ file extract_files/CR7Vww4LuLkMzi4jMd_FwVZpk12AKBjE11UNg__admin-pc_c\$temp_mimikatz.exe + +:: + + extract_files/CR7Vww4LuLkMzi4jMd_FwVZpk12AKBjE11UNg__admin-pc_c$temp_mimikatz.exe: PE32+ executable (console) x86-64, for MS Windows + +As we learned in the :file:`files.log` documentation, we can look in that data +for similar information on extracted files: + +.. literal-emph:: + + { + "ts": 1507565439.130425, + "fuid": "FwVZpk12AKBjE11UNg", + "uid": "CR7Vww4LuLkMzi4jMd", + "id.orig_h": "192.168.10.31", + "id.orig_p": 49238, + "id.resp_h": "192.168.10.30", + "id.resp_p": 445 + "source": "SMB", + "depth": 0, + "analyzers": [ + "SHA1", + "SHA256", + "PE", + "MD5", + "EXTRACT" + ], + **"mime_type": "application/x-dosexec",** + **"filename": "temp\\mimikatz.exe",** + "duration": 0.0034439563751220703, + "is_orig": true, + "seen_bytes": 804352, + "missing_bytes": 0, + "overflow_bytes": 0, + "timedout": true, + **"md5": "2c527d980eb30daa789492283f9bf69e",** + "sha1": "d007f64dae6bc5fdfe4ff30fe7be9b7d62238012", + "sha256": "fb55414848281f804858ce188c3dc659d129e283bd62d58d34f6e6f568feab37", + "extracted": "CR7Vww4LuLkMzi4jMd_FwVZpk12AKBjE11UNg__admin-pc_c$temp_mimikatz.exe", + "extracted_cutoff": false + } + +Here I highlighted the MIME type, showing a Windows executable, as well as the +filename, which includes a directory. + +Let’s take a quick look at the :file:`pe.log` entry: + +.. literal-emph:: + + { + "ts": 1507565439.130425, + "id": "FwVZpk12AKBjE11UNg", + "machine": "AMD64", + **"compile_ts": 1502638084,** + "os": "Windows XP x64 or Server 2003", + "subsystem": "WINDOWS_CUI", + "is_exe": true, + "is_64bit": true, + "uses_aslr": true, + "uses_dep": true, + "uses_code_integrity": false, + "uses_seh": true, + "has_import_table": true, + "has_export_table": false, + "has_cert_table": false, + "has_debug_data": false, + "section_names": [ + ".text", + ".rdata", + ".data", + ".pdata", + ".rsrc", + ".reloc" + ] + } + +There’s some interesting information in this log, like the compile time. We can +convert it to a human readable form using the Linux :program:`date` command. + + +.. code-block:: console + + $ date -d @1502638084 + +:: + + Sun Aug 13 15:28:04 UTC 2017 + +Finally, we can use the md5 from the :file:`file.log` entry to query +VirusTotal, as we also did previously: + +.. code-block:: console + + $ vt file "2c527d980eb30daa789492283f9bf69e" + +:: + + - _id: "fb55414848281f804858ce188c3dc659d129e283bd62d58d34f6e6f568feab37" + _type: "file" + authentihash: "02c86c9977c85a08f18ac1dae02f1cdda569eaba51ec6d17aed6f4ebc2adaf21" + creation_date: 1502638084 # 2017-08-13 15:28:04 +0000 UTC + crowdsourced_yara_results: + - description: "mimikatz" + rule_name: "mimikatz" + ruleset_id: "00043243d1" + ruleset_name: "gen_mimikatz" + source: "https://github.com/Neo23x0/signature-base" + - description: "Detects Mimikatz strings" + rule_name: "Mimikatz_Strings" + ruleset_id: "00043243d1" + ruleset_name: "gen_mimikatz" + source: "https://github.com/Neo23x0/signature-base" + - description: "Detects Mimikatz SkeletonKey in Memory" + rule_name: "HKTL_Mimikatz_SkeletonKey_in_memory_Aug20_1" + ruleset_id: "00043243d1" + ruleset_name: "gen_mimikatz" + source: "https://github.com/Neo23x0/signature-base" + - description: "Detects Powerkatz - a Mimikatz version prepared to run in memory via Powershell (overlap with other Mimikatz versions is possible)" + rule_name: "Powerkatz_DLL_Generic" + ruleset_id: "000d2a7a67" + ruleset_name: "gen_powerkatz" + source: "https://github.com/Neo23x0/signature-base" + - description: "Detects Mimikatz by using some special strings" + rule_name: "Mimikatz_Gen_Strings" + ruleset_id: "000be577b3" + ruleset_name: "thor-hacktools" + source: "https://github.com/Neo23x0/signature-base" + first_submission_date: 1502652611 # 2017-08-13 19:30:11 +0000 UTC + last_analysis_date: 1602435563 # 2020-10-11 16:59:23 +0000 UTC + +I reproduced the first set of results generated by VirusTotal’s +crowdsourced_yara_results to show that this is indeed a copy of Mimikatz, the +ubiquitous credential-dumping tool used for lateral movement in Windows +environments. + +:file:`kerberos.log`, :file:`smb_mapping.log`, and :file:`smb_files.log` +------------------------------------------------------------------------ + +We have learned that ``192.168.10.31`` copied :file:`mimikatz.exe` to +``192.168.10.30``. This is probably the most important aspect of the activity, +and it is based on BZAR’s interpretation of the SMB logs. Let’s take a quick +look at those logs to see if we can glean anything more from them. + +The :file:`kerberos.log` has a single short entry: + +:: + + { + "ts": 1507565438.204785, + "uid": "CR7Vww4LuLkMzi4jMd", + "id.orig_h": "192.168.10.31", + "id.orig_p": 49238, + "id.resp_h": "192.168.10.30", + "id.resp_p": 445 + } + +This indicates that Kerberos, an authentication measure used by Windows, had a +role in this connection. + +The :file:`smb_mapping.log` also has a single short entry: + +.. literal-emph:: + + { + "ts": 1507565438.205583, + "uid": "CR7Vww4LuLkMzi4jMd", + "id.orig_h": "192.168.10.31", + "id.orig_p": 49238, + "id.resp_h": "192.168.10.30", + "id.resp_p": 445, + **"path": "\\\\admin-pc\\c$",** + "share_type": "DISK" + } + +We see evidence of connecting to the administrative file share on +``192.168.10.30``. + +The :file:`smb_files.log` has many entries. The first looks like this: + +.. literal-emph:: + + { + "ts": 1507565438.205868, + "uid": "CR7Vww4LuLkMzi4jMd", + "id.orig_h": "192.168.10.31", + "id.orig_p": 49238, + "id.resp_h": "192.168.10.30", + "id.resp_p": 445, + **"action": "SMB::FILE_OPEN",** + **"path": "\\\\admin-pc\\c$",** + **"name": "",** + "size": 4096, + "times.modified": 1507316839.5820882, + "times.accessed": 1507316839.5820882, + "times.created": 1247539136.5268176, + "times.changed": 1507316839.5820882 + } + +All of the entries have the same ``uid``, ``id.orig_h``, ``id.orig_p``, +``id.resp_h``, and ``id.resp_p``. The ``size`` and ``times`` entries aren’t +especially interesting here. + +I include the specific :program:`jq` syntax in case you’ve forgotten how to +tell :program:`jq` what fields you want to see: + +.. code-block:: console + + $ jq -c '[."action", ."path", ."name"]' smb_files.log + +:: + + ["SMB::FILE_OPEN","\\\\admin-pc\\c$",""] + ["SMB::FILE_OPEN","\\\\admin-pc\\c$","temp"] + ["SMB::FILE_OPEN","\\\\admin-pc\\c$","temp"] + ["SMB::FILE_OPEN","\\\\admin-pc\\c$","temp\\mimikatz.exe"] + ["SMB::FILE_WRITE","\\\\admin-pc\\c$","temp\\mimikatz.exe"] + ["SMB::FILE_WRITE","\\\\admin-pc\\c$","temp\\mimikatz.exe"] + ["SMB::FILE_OPEN","\\\\admin-pc\\c$","temp\\mimikatz.exe"] + ["SMB::FILE_OPEN","\\\\admin-pc\\c$","temp"] + ["SMB::FILE_OPEN","\\\\admin-pc\\c$","temp\\mimikatz.exe"] + +These results do not tell us anything we did not know from the entries the BZAR +script made in the :file:`notice.log`. However, I include them here to help +show how BZAR decided to write in the :file:`notice.log` that it detected +lateral movement via the copy of the file :file:`mimikatz.exe` from +``192.168.10.31`` to ``192.168.10.30``. + +Connecting to a SMB Share and Downloading a File +================================================ + +We continue our exploration of SMB logs by reviewing the second case discussed +in Nate Marx’s paper. The relevant packet capture file is titled +:file:`20171220_smb_mimikatz_copy_to_host.pcap`. Mr. Marx’s discussion appears +at the end of the section titled “The Basics” in his paper. + +If we process the packet capture with Zeek and BZAR, the following files appear: + +* :file:`conn.log` +* :file:`files.log` +* :file:`kerberos.log` +* :file:`packet_filter.log` +* :file:`pe.log` +* :file:`smb_files.log` +* :file:`smb_mapping.log` + +Note that this time we do not have an :file:`extract_files/` directory nor a +:file:`notice.log`! + +We’ll start with the :file:`conn.log` as we did with the previous case. + +:file:`conn.log` +---------------- + +The :file:`conn.log` for this case has only one entry: + +.. literal-emph:: + + { + "ts": 1512585460.295445, + "uid": "C4j5Ds3VyExc2ZAOh9", + **"id.orig_h": "192.168.10.31",** + "id.orig_p": 1112, + **"id.resp_h": "192.168.10.30",** + **"id.resp_p": 445,** + "proto": "tcp", + "service": "krb,gssapi,smb", + "duration": 13.435487985610962, + "orig_bytes": 5762, + "resp_bytes": 812728, + "conn_state": "S1", + "missed_bytes": 0, + "history": "ShADda", + "orig_pkts": 74, + "orig_ip_bytes": 8734, + "resp_pkts": 575, + "resp_ip_bytes": 835740, + "ip_proto": 6 + } + +We see the same pattern: ``192.168.10.31`` initiated a connection to +``192.168.10.30``, to port 445 TCP. In the previous case and the current case, +``192.168.10.31`` connected to a Windows share on ``192.168.10.30``. What +happened next was different. + +In the first case, ``192.168.10.31`` uploaded a file to ``192.168.10.30``. + +In the second case, ``192.168.10.31`` downloaded a file from ``192.168.10.30``. + +Now let’s look at the :file:`files.log` and :file:`pe.log`, as we do not have a +:file:`notice.log` to check. + +:file:`files.log` and :file:`pe.log` +------------------------------------ + +We see one entry in :file:`files.log`: + +.. literal-emph:: + + { + "ts": 1512585460.300969, + "fuid": "FNMweB3f2OvTZ4UZLe", + "uid": "CR7Vww4LuLkMzi4jMd", + **"id.orig_h": "192.168.10.31",** + "id.orig_p": 49238, + **"id.resp_h": "192.168.10.30",** + "id.resp_p": 445 + "source": "SMB", + "source": "SMB", + "depth": 0, + "analyzers": [ + "PE" + ], + "mime_type": "application/x-dosexec", + **"filename": "temp\\mimikatz.exe",** + "duration": 0.010069131851196289, + **"is_orig": false**, + "seen_bytes": 804352, + "total_bytes": 804352, + "missing_bytes": 0, + "overflow_bytes": 0, + "timedout": false + } + +This :file:`files.log` entry is similar to that seen in the previous case, +except the ``is_orig`` value is ``false``. This +indicates that ``192.168.10.30`` sent a file titled :file:`mimikatz.exe` to +``192.168.10.31``, or, said differently, ``192.168.10.31`` downloaded a file +from ``192.168.10.30``. + +With either language, the file started at ``192.168.10.30`` (the responder) +and ended up on ``192.168.10.31`` (the originator). + +This is the reverse of the previous case. + +Here is the :file:`pe.log`: + +.. literal-emph:: + + { + "ts": 1512585460.300969, + "id": "FNMweB3f2OvTZ4UZLe", + "machine": "AMD64", + **"compile_ts": 1502638084,** + "os": "Windows XP x64 or Server 2003", + "subsystem": "WINDOWS_CUI", + "is_exe": true, + "is_64bit": true, + "uses_aslr": true, + "uses_dep": true, + "uses_code_integrity": false, + "uses_seh": true, + "has_import_table": true, + "has_export_table": false, + "has_cert_table": false, + "has_debug_data": false, + "section_names": [ + ".text", + ".rdata", + ".data", + ".pdata", + ".rsrc", + ".reloc" + ] + } + +This output is the same as the previous case, to include the compile time. +There is a different id field because this file was transferred in a different +connection. + +:file:`kerberos.log`, :file:`smb_mapping`.log, and :file:`smb_files.log` +------------------------------------------------------------------------ + +Let’s see what the other relevant files say. + +The :file:`kerberos.log` has one entry: + +:: + + { + "ts": 1512585460.296744, + "uid": "C4j5Ds3VyExc2ZAOh9", + "id.orig_h": "192.168.10.31", + "id.orig_p": 1112, + "id.resp_h": "192.168.10.30", + "id.resp_p": 445 + } + +This is very similar to the previous :file:`kerberos.log` entry, because the +direction of the connection and the authentication is the same. + +The :file:`smb_mapping.log` has one entry: + +:: + + { + "ts": 1512585460.297722, + "uid": "C4j5Ds3VyExc2ZAOh9", + "id.orig_h": "192.168.10.31", + "id.orig_p": 1112, + "id.resp_h": "192.168.10.30", + "id.resp_p": 445, + "path": "\\\\admin-pc\\c$", + "share_type": "DISK" + } + +This is also very similar to the previous :file:`smb_mapping.log` entry, +because the direction of the connection and the share access is the same. + +The :file:`smb_files.log` only has two entries: + +:: + + { + "ts": 1512585460.298136, + "uid": "C4j5Ds3VyExc2ZAOh9", + "id.orig_h": "192.168.10.31", + "id.orig_p": 1112, + "id.resp_h": "192.168.10.30", + "id.resp_p": 445, + "action": "SMB::FILE_OPEN", + "path": "\\\\admin-pc\\c$", + "name": "temp\\mimikatz.exe", + "size": 804352, + "times.modified": 1512171135.77705, + "times.accessed": 1512585399.9219997, + "times.created": 1512585399.9219997, + "times.changed": 1512585399.9376247 + } + { + "ts": 1512585460.299373, + "uid": "C4j5Ds3VyExc2ZAOh9", + "id.orig_h": "192.168.10.31", + "id.orig_p": 1112, + "id.resp_h": "192.168.10.30", + "id.resp_p": 445, + "action": "SMB::FILE_OPEN", + "path": "\\\\admin-pc\\c$", + "name": "temp", + "size": 0, + "times.modified": 1512585399.9219997, + "times.accessed": 1512585399.9219997, + "times.created": 1512585360.2032497, + "times.changed": 1512585399.9219997 + } + +These entries are similar to those from the previous case, at least as far as +the ``id.orig_h`` and ``id.resp_h`` IP addresses and the ``id.resp_p`` port +values. + +Summarizing these two logs, as we did for the previous case, yields these +values: + +.. code-block:: console + + $ jq -c '[."action", ."path", ."name"]' smb_files.log + +:: + + ["SMB::FILE_OPEN","\\\\admin-pc\\c$","temp\\mimikatz.exe"] + ["SMB::FILE_OPEN","\\\\admin-pc\\c$","temp"] + +Looking at these logs, I would not as an analyst be able to tell exactly what +is happening here, other than to say it looks like :file:`mimikatz.exe` is +being transferred. Only the :file:`files.log` entry makes it possible to see +the direction of the transfer: + +The file started at ``192.168.10.30`` and ended up on ``192.168.10.31``. This +conclusion is drawn from the originator and responder information and the +``is_orig`` value for the given entry being ``false``. + +In the next section we will look at how someone might execute a file once it is +present on a target. + +Scheduling Mimikatz via the At Service +====================================== + +The following analysis is based on the :file:`20171220_smb_at_schedule.pcap` +and appears near the end of the RPC section of Mr. Marx’s paper. + +After processing the packet capture with Zeek and BZAR, we have the following +logs: + +* :file:`conn.log` +* :file:`files.log` +* :file:`packet_filter.log` +* :file:`smb_files.log` + +This is a short set of logs to analyze. We will start with the :file:`conn.log`. + +:file:`conn.log` +---------------- + +Looking at the :file:`conn.log`, we see one entry: + +.. literal-emph:: + + { + "ts": 1508525002.992213, + "uid": "Cirxt14nybZjVhpOAk", + **"id.orig_h": "192.168.10.31",** + "id.orig_p": 49266, + **"id.resp_h": "192.168.10.30",** + **"id.resp_p": 445,** + "proto": "tcp", + **"service": "dce_rpc,smb",** + "duration": 12.397327899932861, + "orig_bytes": 1155, + "resp_bytes": 1037, + "conn_state": "OTH", + "missed_bytes": 0, + "history": "DdAR", + "orig_pkts": 11, + "orig_ip_bytes": 1595, + "resp_pkts": 9, + "resp_ip_bytes": 1397, + "ip_proto": 6 + } + +We see ``192.168.10.31`` initiated a connection to ``192.168.10.30``, port 445 +TCP. Zeek recognized this as DCE RPC and SMB traffic. Note that for some +reason Zeek did not create a :file:`dce_rpc.log` for this activity. + +:file:`smb_files.log` +--------------------- + +The :file:`smb_files.log` holds the next clue to this activity. It contains +three entries: + +.. literal-emph:: + + { + "ts": 1508525002.992213, + "uid": "Cirxt14nybZjVhpOAk", + "id.orig_h": "192.168.10.31", + "id.orig_p": 49266, + "id.resp_h": "192.168.10.30", + "id.resp_p": 445, + **"action": "SMB::FILE_OPEN",** + **"name": "atsvc",** + "size": 0 + } + { + "ts": 1508525002.992213, + "uid": "Cirxt14nybZjVhpOAk", + "id.orig_h": "192.168.10.31", + "id.orig_p": 49266, + "id.resp_h": "192.168.10.30", + "id.resp_p": 445, + **"action": "SMB::FILE_WRITE",** + **"name": "atsvc",** + "size": 0, + "data_offset_req": 0, + "data_len_req": 160 + } + { + "ts": 1508525002.992213, + "uid": "Cirxt14nybZjVhpOAk", + "id.orig_h": "192.168.10.31", + "id.orig_p": 49266, + "id.resp_h": "192.168.10.30", + "id.resp_p": 445, + **"fuid": "Fw42Pp34N0CC79C5Ua",** + **"action": "SMB::FILE_WRITE",** + **"name": "atsvc",** + "size": 0, + "data_offset_req": 0, + "data_len_req": 160 + } + +We see SMB ``FILE_OPEN`` and ``FILE_WRITE`` messages to the ``atsvc``. This +indicates that ``192.168.10.31`` is accessing the Windows At service, used for +scheduling processes on Windows. Note that Windows and hence Zeek treats the At +service as a “file,” even though it is a service offered by Windows. + +:file:`files.log` +----------------- + +An odd result of Windows providing the At service as a “file” is that Zeek +creates a :file:`files.log` entry for it. Here is that entry: + +.. literal-emph:: + + { + "ts": 1508525002.992817, + "fuid": "Fw42Pp34N0CC79C5Ua", + "uid": "Cirxt14nybZjVhpOAk", + "id.orig_h": "192.168.10.31", + "id.orig_p": 49266, + "id.resp_h": "192.168.10.30", + "id.resp_p": 445, + **"source": "SMB",** + "depth": 0, + "analyzers": [], + **"filename": "atsvc",** + "duration": 0.00038909912109375, + "is_orig": true, + "seen_bytes": 160, + "missing_bytes": 0, + "overflow_bytes": 0, + "timedout": false + } + +This file does not tell us anything we did not already know. Zeek did not +extract a file either, because the “file” in this instance is an abstraction +used to represent the At service on the Windows target. + +Reviewing the Packet Capture with :program:`tshark` +=================================================== + +If administrators are authorized to use the At service to schedule jobs, from +the indicated source to the indicated destination, then it may not be possible +for a security analyst to identify this as malicious activity. We might be able +to learn a bit more about the activity by looking at the packet capture +directly. + +To create the following output, I told :program:`tshark` to only display the +source IP address, the protocol, and the information field for each frame. I +also specified that it look at SMB version 2 traffic. + +.. code-block:: console + + $ tshark -r 20171220_smb_at_schedule.pcap -T fields -e _ws.col.No. -e _ws.col.Source -e _ws.col.Protocol -e _ws.col.Info -Y smb2 + +.. literal-emph:: + + **1 192.168.10.31 SMB2 Create Request File: atsvc** + 2 192.168.10.30 SMB2 Create Response File: atsvc + 3 192.168.10.31 SMB2 GetInfo Request FILE_INFO/SMB2_FILE_STANDARD_INFO File: atsvc + 4 192.168.10.30 SMB2 GetInfo Response + 5 192.168.10.31 DCERPC Bind: call_id: 2, Fragment: Single, 3 context items: ATSVC V1.0 (32bit NDR), ATSVC V1.0 (64bit NDR), ATSVC V1.0 (6cb71c2c-9812-4540-0300-000000000000) + 6 192.168.10.30 SMB2 Write Response + 7 192.168.10.31 SMB2 Read Request Len:1024 Off:0 File: atsvc + 8 192.168.10.30 DCERPC Bind_ack: call_id: 2, Fragment: Single, max_xmit: 4280 max_recv: 4280, 3 results: Provider rejection, Acceptance, Negotiate ACK + **9 192.168.10.31 ATSVC JobAdd request** + 10 192.168.10.30 SMB2 Ioctl Response, Error: STATUS_PENDING + 11 192.168.10.30 ATSVC JobAdd response + 13 192.168.10.31 SMB2 Close Request File: atsvc + 14 192.168.10.30 SMB2 Close Response + 16 192.168.10.31 SMB2 Tree Disconnect Request + 17 192.168.10.30 SMB2 Tree Disconnect Response + 18 192.168.10.31 SMB2 Session Logoff Request + 19 192.168.10.30 SMB2 Session Logoff Response + +Right away in frame 1 we see the request to create a “file” for the ``atsvc``. + +Frame 9 might have the details of the Atsvc request. We can look at the details +using :program:`tshark`. The -O (capital letter O) command specifies which +layer of the decode we want to see. + +.. code-block:: console + + $ tshark -r 20171220_smb_at_schedule.pcap -V -Y frame.number==9 -O atsvc + +.. literal-emph:: + + Frame 9: 338 bytes on wire (2704 bits), 338 bytes captured (2704 bits) + Ethernet II, Src: 08:00:27:7f:b5:8b, Dst: 08:00:27:a1:27:e8 + Internet Protocol Version 4, Src: 192.168.10.31, Dst: 192.168.10.30 + Transmission Control Protocol, Src Port: 49266, Dst Port: 445, Seq: 636, Ack: 541, Len: 284 + NetBIOS Session Service + SMB2 (Server Message Block Protocol version 2) + Distributed Computing Environment / Remote Procedure Call (DCE/RPC) Request, Fragment: Single, FragLen: 160, Call: 2, Ctx: 1 + Microsoft AT-Scheduler Service, JobAdd + Operation: JobAdd (0) + Pointer to Servername (uint16): \\admin-pc + Referent ID: 0x0000000000020000 + Max Count: 11 + Offset: 0 + Actual Count: 11 + Server: \\admin-pc + Pointer to Job Info (atsvc_JobInfo) + JobInfo + Job Time: 47100000 + Days Of Month: 0x00000000: (No values set) + .... .... .... .... .... .... .... ...0 = First: First is NOT SET + .... .... .... .... .... .... .... ..0. = Second: Second is NOT SET + .... .... .... .... .... .... .... .0.. = Third: Third is NOT SET + .... .... .... .... .... .... .... 0... = Fourth: Fourth is NOT SET + .... .... .... .... .... .... ...0 .... = Fifth: Fifth is NOT SET + .... .... .... .... .... .... ..0. .... = Sixth: Sixth is NOT SET + .... .... .... .... .... .... .0.. .... = Seventh: Seventh is NOT SET + .... .... .... .... .... .... 0... .... = Eight: Eight is NOT SET + .... .... .... .... .... ...0 .... .... = Ninth: Ninth is NOT SET + .... .... .... .... .... ..0. .... .... = Tenth: Tenth is NOT SET + .... .... .... .... .... .0.. .... .... = Eleventh: Eleventh is NOT SET + .... .... .... .... .... 0... .... .... = Twelfth: Twelfth is NOT SET + .... .... .... .... ...0 .... .... .... = Thitteenth: Thitteenth is NOT SET + .... .... .... .... ..0. .... .... .... = Fourteenth: Fourteenth is NOT SET + .... .... .... .... .0.. .... .... .... = Fifteenth: Fifteenth is NOT SET + .... .... .... .... 0... .... .... .... = Sixteenth: Sixteenth is NOT SET + .... .... .... ...0 .... .... .... .... = Seventeenth: Seventeenth is NOT SET + .... .... .... ..0. .... .... .... .... = Eighteenth: Eighteenth is NOT SET + .... .... .... .0.. .... .... .... .... = Ninteenth: Ninteenth is NOT SET + .... .... .... 0... .... .... .... .... = Twentyth: Twentyth is NOT SET + .... .... ...0 .... .... .... .... .... = Twentyfirst: Twentyfirst is NOT SET + .... .... ..0. .... .... .... .... .... = Twentysecond: Twentysecond is NOT SET + .... .... .0.. .... .... .... .... .... = Twentythird: Twentythird is NOT SET + .... .... 0... .... .... .... .... .... = Twentyfourth: Twentyfourth is NOT SET + .... ...0 .... .... .... .... .... .... = Twentyfifth: Twentyfifth is NOT SET + .... ..0. .... .... .... .... .... .... = Twentysixth: Twentysixth is NOT SET + .... .0.. .... .... .... .... .... .... = Twentyseventh: Twentyseventh is NOT SET + .... 0... .... .... .... .... .... .... = Twentyeighth: Twentyeighth is NOT SET + ...0 .... .... .... .... .... .... .... = Twentyninth: Twentyninth is NOT SET + ..0. .... .... .... .... .... .... .... = Thirtieth: Thirtieth is NOT SET + .0.. .... .... .... .... .... .... .... = Thirtyfirst: Thirtyfirst is NOT SET + Days Of Week: 0x00: (No values set) + .... ...0 = DAYSOFWEEK MONDAY: DAYSOFWEEK_MONDAY is NOT SET + .... ..0. = DAYSOFWEEK TUESDAY: DAYSOFWEEK_TUESDAY is NOT SET + .... .0.. = DAYSOFWEEK WEDNESDAY: DAYSOFWEEK_WEDNESDAY is NOT SET + .... 0... = DAYSOFWEEK THURSDAY: DAYSOFWEEK_THURSDAY is NOT SET + ...0 .... = DAYSOFWEEK FRIDAY: DAYSOFWEEK_FRIDAY is NOT SET + ..0. .... = DAYSOFWEEK SATURDAY: DAYSOFWEEK_SATURDAY is NOT SET + .0.. .... = DAYSOFWEEK SUNDAY: DAYSOFWEEK_SUNDAY is NOT SET + Flags: 0x00: (No values set) + .... ...0 = JOB RUN PERIODICALLY: JOB_RUN_PERIODICALLY is NOT SET + .... ..0. = JOB EXEC ERROR: JOB_EXEC_ERROR is NOT SET + .... .0.. = JOB RUNS TODAY: JOB_RUNS_TODAY is NOT SET + .... 0... = JOB ADD CURRENT DATE: JOB_ADD_CURRENT_DATE is NOT SET + ...0 .... = JOB NONINTERACTIVE: JOB_NONINTERACTIVE is NOT SET + **Pointer to Command (uint16): c:\mimikatz.exe** + **Referent ID: 0x0000000000020000** + **Max Count: 16** + **Offset: 0** + **Actual Count: 16** + **Command: c:\mimikatz.exe** + +Once you get past the spelling errors in the “Days of Month” section, we see in +the “Pointer to Command” section a reference to :file:`c:\mimikatz.exe`. This +detail was not available in the Zeek logs, but this additional information +helps us recognize this activity as being likely malicious. + +We can look to see if the command succeeded by reviewing the details of frame +11. + +.. code-block:: console + + $ tshark -r 20171220_smb_at_schedule.pcap -V -Y frame.number==11 -O atsvc + +.. literal-emph:: + + Frame 11: 202 bytes on wire (1616 bits), 202 bytes captured (1616 bits) + Ethernet II, Src: 08:00:27:a1:27:e8, Dst: 08:00:27:7f:b5:8b + Internet Protocol Version 4, Src: 192.168.10.30, Dst: 192.168.10.31 + Transmission Control Protocol, Src Port: 445, Dst Port: 49266, Seq: 618, Ack: 920, Len: 148 + NetBIOS Session Service + SMB2 (Server Message Block Protocol version 2) + Distributed Computing Environment / Remote Procedure Call (DCE/RPC) Response, Fragment: Single, FragLen: 32, Call: 2, Ctx: 1, [Req: #9] + Microsoft AT-Scheduler Service, JobAdd + Operation: JobAdd (0) + [Request in frame: 9] + Pointer to Job Id (uint32) + Job Id: 2 + **NT Error: STATUS_SUCCESS (0x00000000)** + +The ``NT Error`` message shows ``STATUS_SUCCESS``, which indicates that the job +was scheduled via the At service. + +In the next section we will introduce another capability associated with +Windows lateral movement. + +Using PsExec to Retrieve a File from a Target +============================================= + +Microsoft describes PsExec in the following terms: + + “PsExec is a light-weight telnet-replacement that lets you execute processes + on other systems, complete with full interactivity for console applications, + without having to manually install client software. PsExec's most powerful + uses include launching interactive command-prompts on remote systems and + remote-enabling tools like IpConfig that otherwise do not have the ability to + show information about remote systems.” + +Ref: https://docs.microsoft.com/en-us/sysinternals/downloads/psexec + +Intruders are fond of PsExec for the very capabilities that Microsoft +describes. + +The following analysis is based on the +:file:`20171220_smb_psexec_mimikatz_ticket_dump.pcap` file described in the +PsExec section of Nate Marx’s paper. + +Zeek creates the following output for this packet capture, along with an +:file:`extract_files/` directory. I use the :program:`wc` command to show how +many lines appear in each file. + +.. code-block:: console + + $ wc -l *.log + +:: + + 9 conn.log + 20 dce_rpc.log + 9 dns.log + 1 files.log + 2 kerberos.log + 8 notice.log + 1 packet_filter.log + 1 pe.log + 5 smb_files.log + 2 smb_mapping.log + +We’ll start with the :file:`conn.log` but move to the :file:`notice.log` +quickly thereafter. + +:file:`conn.log` +---------------- + +Because we saw that there were 9 entries in the :file:`conn.log`, I’m going to +summarize them using the following command: + +.. code-block:: console + + $ jq -c '[."uid", ."id.orig_h", ."id.resp_h", ."id.resp_p", ."proto", ."service"]' conn.log + +:: + + ["CT7qITytKtae83Tyi","192.168.10.31","192.168.10.10",88,"tcp","krb_tcp"] + ["CBFaLB1HJivXnb9Jw2","192.168.10.31","192.168.10.30",135,"tcp","dce_rpc"] + ["CqgZIa4KYnX4cNHJo8","192.168.10.31","192.168.10.30",49155,"tcp","dce_rpc"] + ["C95D4lsjb4GjGbBq2","192.168.10.31","192.168.10.255",137,"udp","dns"] + ["CEcy2LEJUZQrLwO4b","192.168.10.31","192.168.10.10",53,"udp","dns"] + ["CPlgJVWL9yrKdUsX8","192.168.10.31","192.168.10.10",53,"udp","dns"] + ["C6zoLD2QgM71nvWdX5","192.168.10.30","192.168.10.255",137,"udp","dns"] + ["C6HQVsDf8VCu0XTJe","192.168.10.31","192.168.10.30",445,"tcp","smb,krb,gssapi"] + ["Cishox1cH3JLghxiV8","192.168.10.31","192.168.10.10",3,"icmp",null] + +The 4 TCP connections likely are the sessions we want to investigate in this +case. However, because we have a :file:`notice.log` for this activity, it’s +smartest to look at those entries next. + +:file:`notice.log` +------------------ + +The :file:`notice.log` for this activity has 8 entries. I tried to distill them +to the bare minimum required to convey what is happening, according to Zeek and +BZAR. + +.. code-block:: console + + $ jq -c '[."uid", ."note", ."msg", ."sub", ."src", ."dst"]' notice.log | uniq + +.. literal-emph:: + + ["C6HQVsDf8VCu0XTJe","ATTACK::Lateral_Movement","Detected SMB::FILE_WRITE to admin file share '\\\\admin-pc\\ADMIN$PSEXESVC.exe'","T1021.002 Remote Services: SMB/Windows Admin Shares + **T1570 Lateral Tool Transfer**","192.168.10.31","192.168.10.30"] + + ["C6HQVsDf8VCu0XTJe","ATTACK::Lateral_Movement_Extracted_File","**Saved a copy of the file written to SMB admin file share**","C6HQVsDf8VCu0XTJe_FtIFnm3ZqI1s96P74l__admin-pc_ADMIN$**PSEXESVC.exe**","192.168.10.31","192.168.10.30"] + + ["CqgZIa4KYnX4cNHJo8","ATTACK::Execution","svcctl::CreateServiceWOW64W","T1569.002 **System Services: Service Execution**","192.168.10.31","192.168.10.30"] + + [null,"ATTACK::Lateral_Movement_and_Execution","**Detected activity against host 192.168.10.30**, total score 1004 within timeframe 10.0 mins",null,null,null] + + ["CqgZIa4KYnX4cNHJo8","ATTACK::Execution","svcctl::StartServiceW","T1569.002 System Services: **Service Execution**","192.168.10.31","192.168.10.30"] + +The highlighted fields indicate suspicious or malicious activity. We see +evidence of lateral tool transfer to ``192.168.10.30`` via SMB of a file named +:file:`psexecsvc.exe`, then service execution. + +:file:`dce_rpc.log` +------------------- + +Let’s see if the :file:`dce_rpc.log` adds any useful details. We saw earlier +that this log has 20 entries. The first two shows us the pattern that occupies +all 20 entries. + +.. literal-emph:: + + { + "ts": 1507565599.588936, + "uid": "CBFaLB1HJivXnb9Jw2", + "id.orig_h": "192.168.10.31", + "id.orig_p": 49240, + **"id.resp_h": "192.168.10.30",** + **"id.resp_p": 135,** + "rtt": 0.0002448558807373047, + "named_pipe": "135", + **"endpoint": "epmapper",** + "operation": "ept_map" + } + + { + "ts": 1507565599.601632, + "uid": "CqgZIa4KYnX4cNHJo8", + "id.orig_h": "192.168.10.31", + "id.orig_p": 49241, + **"id.resp_h": "192.168.10.30",** + **"id.resp_p": 49155,** + "rtt": 0.0003237724304199219, + "named_pipe": "49155", + "endpoint": "svcctl", + "operation": "OpenSCManagerW" + } + +The first entry shows a call to the Windows endpoint mapper, ``epmapper``, on +port 135 TCP on ``192.168.10.30``. The response from this service directs the +client ``192.168.10.31`` to port 49155 TCP on ``192.168.10.30``. The second and +subsequent :file:`dce_rpc.log` entries involve port 49155 TCP on the target, which is +offering ``svcctrl``. + +We see the target IP address is ``192.168.10.30``, confirming the activity in +the :file:`notice.log`. As we did with a previous :file:`dce_rpc.log`, we can +simplify this one into the following entries: + +.. code-block:: console + + $ jq -c '[."named_pipe", ."endpoint", ."operation"]' dce_rpc.log | uniq + +:: + + ["135","epmapper","ept_map"] + ["49155","svcctl","OpenSCManagerW"] + ["49155","svcctl","CreateServiceWOW64W"] + ["49155","svcctl","CloseServiceHandle"] + ["49155","svcctl","OpenServiceW"] + ["49155","svcctl","StartServiceW"] + ["49155","svcctl","QueryServiceStatus"] + ["49155","svcctl","CloseServiceHandle"] + ["49155","svcctl","OpenSCManagerW"] + ["49155","svcctl","OpenServiceW"] + ["49155","svcctl","ControlService"] + ["49155","svcctl","QueryServiceStatus"] + ["49155","svcctl","CloseServiceHandle"] + ["49155","svcctl","OpenServiceW"] + ["49155","svcctl","DeleteService"] + ["49155","svcctl","CloseServiceHandle"] + +We see some sort of successful interaction with the ``svcctrl`` service on the target. + +Incidentally, we can’t see much more using a protocol analyzer like +:program:`tshark`, either: + +.. code-block:: console + + $ tshark -r 20171220_smb_psexec_mimikatz_ticket_dump.pcap -V -Y frame.number==76 -O svcctl + +.. literal-emph:: + + Frame 76: 258 bytes on wire (2064 bits), 258 bytes captured (2064 bits) + Ethernet II, Src: 08:00:27:7f:b5:8b, Dst: 08:00:27:a1:27:e8 + Internet Protocol Version 4, Src: 192.168.10.31, Dst: 192.168.10.30 + Transmission Control Protocol, Src Port: 49241, Dst Port: 49155, Seq: 1945, Ack: 366, Len: 204 + Distributed Computing Environment / Remote Procedure Call (DCE/RPC) Request, Fragment: Single, FragLen: 204, Call: 2, Ctx: 0 + Microsoft Service Control, OpenSCManagerW + Operation: OpenSCManagerW (15) + **Encrypted stub data: 02353eb074e7e350b9632e05b550f725c99d41d419165110...** + +As Mr. Marx notes in his paper, the content of these exchanges are encrypted +within the Microsoft Service Control layer. + +:file:`kerberos.log` +-------------------- + +The :file:`kerberos.log` contains two entries: + +.. literal-emph:: + + { + "ts": 1507565599.590346, + "uid": "CT7qITytKtae83Tyi", + **"id.orig_h": "192.168.10.31",** + "id.orig_p": 49242, + **"id.resp_h": "192.168.10.10",** + **"id.resp_p": 88,** + "request_type": "TGS", + **"client": "RonHD/CONTOSO.LOCAL",** + **"service": "HOST/admin-pc",** + "success": true, + "till": 2136422885, + "cipher": "aes256-cts-hmac-sha1-96", + "forwardable": true, + "renewable": true + } + { + "ts": 1507565599.575721, + "uid": "C6HQVsDf8VCu0XTJe", + "id.orig_h": "192.168.10.31", + "id.orig_p": 49239, + **"id.resp_h": "192.168.10.30",** + "id.resp_p": 445 + } + +The first entry includes the acronym TGS, which means Ticket Granting service. +The system ``192.168.10.10`` appears to be a domain controller, as we saw in an +earlier case. We gather some information on the intruder’s system, namely that +it is ``RonHD`` in the ``CONTOSO.LOCAL`` domain. + +The second entry shows that the aggressor ``192.168.10.31`` used Kerberos to +authenticate to the target ``192.168.10.30``. + +:file:`smb_mapping.log` +----------------------- + +The :file:`smb_mapping.log` contains two entries: + +.. literal-emph:: + + { + "ts": 1507565599.576613, + "uid": "C6HQVsDf8VCu0XTJe", + "id.orig_h": "192.168.10.31", + "id.orig_p": 49239, + "id.resp_h": "192.168.10.30", + "id.resp_p": 445, + **"path": "\\\\admin-pc\\ADMIN$",** + "share_type": "DISK" + } + { + "ts": 1507565599.729707, + "uid": "C6HQVsDf8VCu0XTJe", + "id.orig_h": "192.168.10.31", + "id.orig_p": 49239, + "id.resp_h": "192.168.10.30", + "id.resp_p": 445, + **"path": "\\\\admin-pc\\IPC$",** + "share_type": "PIPE" + } + +As we learned earlier, connections to the ``ADMIN$`` and ``IPC$`` shares on a +target system are suspicious or malicious if they are not already authorized. + +:file:`smb_files.log` +--------------------- + +There are many entries in the :file:`smb_files.log`. The first looks like +this: + +.. literal-emph:: + + { + "ts": 1507565599.576942, + "uid": "C6HQVsDf8VCu0XTJe", + **"id.orig_h": "192.168.10.31",** + "id.orig_p": 49239, + **"id.resp_h": "192.168.10.30",** + **"id.resp_p": 445,** + "action": "SMB::FILE_OPEN", + "path": "\\\\admin-pc\\ADMIN$", + **"name": "PSEXESVC.exe",** + "size": 0, + "times.modified": 1507565599.607777, + "times.accessed": 1507565599.607777, + "times.created": 1507565599.607777, + "times.changed": 1507565599.607777 + } + +As we noted earlier, use of :file:`psexecsvc.exe` is likely malicious as +intruders use it to run :program:`PsExec` on remote systems. + +We can summarize all of the entries in :file:`smb_files.log` with the following +syntax: + +.. code-block:: console + + $ jq -c '[."action", ."path", ."name"]' smb_files.log + +:: + + ["SMB::FILE_OPEN","\\\\admin-pc\\ADMIN$","PSEXESVC.exe"] + ["SMB::FILE_WRITE","\\\\admin-pc\\ADMIN$","PSEXESVC.exe"] + ["SMB::FILE_WRITE","\\\\admin-pc\\ADMIN$","PSEXESVC.exe"] + ["SMB::FILE_OPEN","\\\\admin-pc\\ADMIN$","PSEXESVC.exe"] + ["SMB::FILE_DELETE","\\\\admin-pc\\ADMIN$","PSEXESVC.exe"] + +This does not give us any more context but it shows the sorts of data in the +:file:`smb_files.log`. + +:file:`extract_files/`, :file:`files.log`, and :file:`pe.log`, and VirusTotal +----------------------------------------------------------------------------- + +As we did in a previous case, we can look into the files that Zeek and BZAR +captured for this activity. + +The :file:`extract_files/` directory contains one executable file:: + + extract_files/C6HQVsDf8VCu0XTJe_FtIFnm3ZqI1s96P74l__admin-pc_ADMIN$PSEXESVC.exe: PE32 executable (console) Intel 80386, for MS Windows + +Zeek’s :file:`files.log` says the following about it: + +.. literal-emph:: + + { + "ts": 1507565599.578328, + "fuid": "FtIFnm3ZqI1s96P74l", + "uid": "C6HQVsDf8VCu0XTJe", + "id.orig_h": "192.168.10.31", + "id.orig_p": 49239, + "id.resp_h": "192.168.10.30", + "id.resp_p": 445, + "source": "SMB", + "depth": 0, + "analyzers": [ + "MD5", + "SHA1", + "PE", + "EXTRACT", + "SHA256" + ], + "mime_type": "application/x-dosexec", + **"filename": "PSEXESVC.exe",** + "duration": 0.0006651878356933594, + "is_orig": true, + "seen_bytes": 145568, + "missing_bytes": 0, + "overflow_bytes": 0, + "timedout": false, + "md5": "75b55bb34dac9d02740b9ad6b6820360", + "sha1": "a17c21b909c56d93d978014e63fb06926eaea8e7", + "sha256": "141b2190f51397dbd0dfde0e3904b264c91b6f81febc823ff0c33da980b69944", + "extracted": "C6HQVsDf8VCu0XTJe_FtIFnm3ZqI1s96P74l__admin-pc_ADMIN$PSEXESVC.exe", + "extracted_cutoff": false + } + +Zeek’s :file:`pe.log` says the following: + +.. literal-emph:: + + { + "ts": 1507565599.578328, + "id": "FtIFnm3ZqI1s96P74l", + "machine": "I386", + **"compile_ts": 1467139314,** + "os": "Windows XP", + "subsystem": "WINDOWS_CUI", + "is_exe": true, + "is_64bit": false, + "uses_aslr": true, + "uses_dep": true, + "uses_code_integrity": false, + "uses_seh": true, + "has_import_table": true, + "has_export_table": false, + "has_cert_table": true, + "has_debug_data": false, + "section_names": [ + ".text", + ".rdata", + ".data", + ".rsrc", + ".reloc" + ] + } + +The compile time translates to human readable format as this: + +.. code-block:: console + + $ date -d @1467139314 + +:: + + Tue Jun 28 18:41:54 UTC 2016 + +We can also check VirusTotal using the MD5 hash: + +.. code-block:: console + + $ vt file "75b55bb34dac9d02740b9ad6b6820360" + +.. literal-emph:: + + - _id: "141b2190f51397dbd0dfde0e3904b264c91b6f81febc823ff0c33da980b69944" + _type: "file" + authentihash: "62287971b29db5858ceaf92e9db310862e9082608f9dd3ac7f5ed3f71c7cfc38" + **creation_date: 1467139314 # 2016-06-28 18:41:54 +0000 UTC** + **first_seen_itw_date: 1463443155 # 2016-05-16 23:59:15 +0000 UTC** + **first_submission_date: 1467293310 # 2016-06-30 13:28:30 +0000 UTC** + **last_analysis_date: 1606108041 # 2020-11-23 05:07:21 +0000 UTC** + last_analysis_results: + ALYac: + category: "undetected" + engine_name: "ALYac" + engine_update: "20201123" + engine_version: "1.1.1.5" + method: "blacklist" + ...truncated… + +The various dates for this copy of :program:`PsExecSvc` are interesting. + +I am not sure how to account for a first seen in the wild date that precedes +the creation date. I think it’s interesting that only a few hours before I +worked with this sample, someone else was doing the same thing, but via +uploading the executable! + +After this analysis, all we know is that :program:`PsExecSvc` is being used +successfully against ``192.168.10.31``. Mr. Marx’s paper notes that his +activity involved retrieving a file from the target. We cannot tell that from +these logs. This is an example of using Zeek logs to identify suspicious or +malicious activity, and then pivoting to host-centric data to determine exactly +what is happening. + +:file:`ntlm.log` +---------------- + +One log we have not seen in any of these cases is the :file:`ntlm.log`. This +log captures old-style Windows NT Lan Manager (NTLM) authentication details. +The packet capture :file:`smb-on-windows-10.pcapng` provided by the Wireshark +project produces a :file:`ntlm.log` when Zeek processes it. + +Ref: https://wiki.wireshark.org/SMB2 + +.. literal-emph:: + + { + "ts": 1476605364.033848, + "uid": "CNicnvp8Qdqbqm96a", + "id.orig_h": "192.168.199.133", + "id.orig_p": 49672, + "id.resp_h": "192.168.199.1", + "id.resp_p": 139, + "hostname": "DESKTOP-V1FA0UQ", + "server_nb_computer_name": "SCV", + "server_dns_computer_name": "SCV", + **"success": true** + } + { + "ts": 1476605590.442053, + "uid": "CLVEN87g2bfZgXqP5", + "id.orig_h": "192.168.199.132", + "id.orig_p": 49670, + "id.resp_h": "192.168.199.133", + "id.resp_p": 445, + "username": "user", + "hostname": "DESKTOP-2AEFM7G", + "domainname": "DESKTOP-2AEFM7G", + "server_nb_computer_name": "DESKTOP-V1FA0UQ", + "server_dns_computer_name": "DESKTOP-V1FA0UQ" + } + { + "ts": 1476605590.474118, + "uid": "C74tDzQl0ttE8v813", + "id.orig_h": "192.168.199.132", + "id.orig_p": 49671, + "id.resp_h": "192.168.199.133", + "id.resp_p": 445, + "username": "user", + "hostname": "DESKTOP-2AEFM7G", + "domainname": "DESKTOP-2AEFM7G", + "server_nb_computer_name": "DESKTOP-V1FA0UQ", + "server_dns_computer_name": "DESKTOP-V1FA0UQ" + } + { + "ts": 1476605590.484196, + "uid": "CzLJgJ2nrXGMxvnXze", + "id.orig_h": "192.168.199.132", + "id.orig_p": 49672, + "id.resp_h": "192.168.199.133", + "id.resp_p": 445, + "username": "user", + "hostname": "DESKTOP-2AEFM7G", + "domainname": "DESKTOP-2AEFM7G", + "server_nb_computer_name": "DESKTOP-V1FA0UQ", + "server_dns_computer_name": "DESKTOP-V1FA0UQ" + } + { + "ts": 1476605590.496004, + "uid": "Ct46uQ2dOQuqnp5YPj", + "id.orig_h": "192.168.199.132", + "id.orig_p": 49673, + "id.resp_h": "192.168.199.133", + "id.resp_p": 445, + "username": "user", + "hostname": "DESKTOP-2AEFM7G", + "domainname": "DESKTOP-2AEFM7G", + "server_nb_computer_name": "DESKTOP-V1FA0UQ", + "server_dns_computer_name": "DESKTOP-V1FA0UQ" + } + { + "ts": 1476605609.93236, + "uid": "CQorcF2L5fLEA4EImh", + "id.orig_h": "192.168.199.132", + "id.orig_p": 49674, + "id.resp_h": "192.168.199.133", + "id.resp_p": 445, + "username": "Tim Tester", + "hostname": "DESKTOP-2AEFM7G", + "domainname": "DESKTOP-2AEFM7G", + "server_nb_computer_name": "DESKTOP-V1FA0UQ", + "server_dns_computer_name": "DESKTOP-V1FA0UQ" + } + { + "ts": 1476605761.4297, + "uid": "CBbRT6X875vQPAgJj", + "id.orig_h": "192.168.199.132", + "id.orig_p": 49675, + "id.resp_h": "192.168.199.133", + "id.resp_p": 445, + "username": "Willi Wireshark", + "hostname": "DESKTOP-2AEFM7G", + "domainname": "DESKTOP-2AEFM7G", + "server_nb_computer_name": "DESKTOP-V1FA0UQ", + "server_dns_computer_name": "DESKTOP-V1FA0UQ", + **"success": true** + } + +This pcap produces a lot of Zeek logs, so I wanted to only show these entries. +Analysts would probably take two investigative steps. First, should +``192.168.199.132`` be trying to access these other systems? Second, should the +authentication have succeeded, as denoted by the two “true” results? + +Conclusion +========== + +This has been a large section, but the goal was to present a set of cases and +show how Zeek and BZAR (when available) made sense of them. I recommend reading +Mr. Marx’s paper for more details as well. diff --git a/doc/logs/smtp.rst b/doc/logs/smtp.rst new file mode 100644 index 0000000000..1e190480c0 --- /dev/null +++ b/doc/logs/smtp.rst @@ -0,0 +1,847 @@ +======== +smtp.log +======== + +In the section discussing the :file:`http.log`, we noted that most HTTP traffic +is now encrypted and transmitted as HTTPS. We face a similar situation with +Simple Mail Transfer Protocol (SMTP). For a protocol with “simple” in its name, +modern instantiations of SMTP are surprisingly complex. + +For the purpose of this article, it’s sufficient to recognize that a mail user +agent (MUA) seeking to submit email via SMTP will contact a mail submission +agent (MSA). Modern implementations will use ports 587 or 465 TCP, which is +encrypted using TLS. Unencrypted implementations will use port 25 TCP. + +Because SMTP traffic on ports 587 or 465 TCP is encrypted, we will not see +individual emails when observing traffic using those protocols. This section +will demonstrate how Zeek reports on email traffic using ports 25, 465, and 587 +TCP. + +Remember that to see the meaning of each field in the :file:`smtp.log`, check +:zeek:see:`SMTP::Info`. + +Inspecting SMTP Traffic +======================= + +The following is a capture of an SMTP session retrieved from an online packet +capture database. I have reconstructed the session using :program:`tcpflow` and +edited it to remove material not necessary to make my point. + +.. literal-emph:: + + SMTP server: 220-xc90.websitewelcome.com ESMTP Exim 4.69 #1 Mon, 05 Oct 2009 01:05:54 -0500 + 220-We do not authorize the use of this system to transport unsolicited, + 220 and/or bulk e-mail. + + SMTP client: **EHLO GP** + + SMTP server: 250-xc90.websitewelcome.com Hello GP [122.162.143.157] + 250-SIZE 52428800 + 250-PIPELINING + 250-AUTH PLAIN LOGIN + 250-STARTTLS + 250 HELP + + SMTP client: **AUTH LOGIN** + + SMTP server: 334 VXNlcm5hbWU6 + + SMTP client: **Z3VycGFydGFwQHBhdHJpb3RzLmlu** + + SMTP server: 334 UGFzc3dvcmQ6 + + SMTP client: **cHVuamFiQDEyMw==** + + SMTP server: 235 Authentication succeeded + + SMTP client: **MAIL FROM: ** + + SMTP server: 250 OK + + SMTP client: **RCPT TO: ** + + SMTP server: 250 Accepted + + SMTP client: **DATA** + + SMTP server: 354 Enter message, ending with "." on a line by itself + + SMTP client: **From: "Gurpartap Singh" ** + **To: ** + **Subject: SMTP** + **Date: Mon, 5 Oct 2009 11:36:07 +0530** + **Message-ID: <000301ca4581$ef9e57f0$cedb07d0$@in>** + **MIME-Version: 1.0** + **Content-Type: multipart/mixed;** + **.boundary="----=_NextPart_000_0004_01CA45B0.095693F0"** + **X-Mailer: Microsoft Office Outlook 12.0** + **Thread-Index: AcpFgem9BvjjZEDeR1Kh8i+hUyVo0A==** + **Content-Language: en-us** + **x-cr-hashedpuzzle: SeA= AAR2 ADaH BpiO C4G1 D1gW FNB1 FPkR Fn+W HFCP HnYJ JO7s Kum6 KytW LFcI LjUt;1;cgBhAGoAXwBkAGUAbwBsADIAMAAwADIAaQBuAEAAeQBhAGgAbwBvAC4AYwBvAC4AaQBuAA==;Sosha1_v1;7;{CAA37F59-1850-45C7-8540-AA27696B5398};ZwB1AHIAcABhAHIAdABhAHAAQABwAGEAdAByAGkAbwB0AHMALgBpAG4A;Mon, 05 Oct 2009 06:06:01 GMT;UwBNAFQAUAA=** + **x-cr-puzzleid: {CAA37F59-1850-45C7-8540-AA27696B5398}** + + **This is a multipart message in MIME format.** + + **------=_NextPart_000_0004_01CA45B0.095693F0** + **Content-Type: multipart/alternative;** + **.boundary="----=_NextPart_001_0005_01CA45B0.095693F0"** + + + **------=_NextPart_001_0005_01CA45B0.095693F0** + **Content-Type: text/plain;** + **.charset="us-ascii"** + **Content-Transfer-Encoding: 7bit** + + **Hello** + + + + **I send u smtp pcap file** + + **Find the attachment** + + + + **GPS** + + + **------=_NextPart_001_0005_01CA45B0.095693F0** + **Content-Type: text/html;** + **.charset="us-ascii"** + **Content-Transfer-Encoding: quoted-printable** + + **** + + **** + **** + **** + **