diff --git a/CHANGES b/CHANGES index 63483835ba..5c1dc5574e 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,18 @@ +2.0-beta-47 | 2011-11-16 08:24:33 -0800 + + * Catch if logged sets do not contain only atomic types. (Bernhard + Amann) + + * Promote libz and libmagic to required dependencies. (Jon Siwek) + + * Fix parallel make from top-level to work on more platforms. (Jon + Siwek) + + * Add decode_base64_custom(). Addresses #670 (Jon Siwek) + + * A bunch of Sphinx-doc reorgs and polishing. (Jon Siwek) + 2.0-beta-28 | 2011-11-14 20:09:28 -0800 * Binary packaging script tweaks. We now require CMake 2.8.6. (Jon Siwek) diff --git a/CMakeLists.txt b/CMakeLists.txt index 78e9344a4c..241a5b29d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,6 +53,8 @@ FindRequiredPackage(BISON) FindRequiredPackage(PCAP) FindRequiredPackage(OpenSSL) FindRequiredPackage(BIND) +FindRequiredPackage(LibMagic) +FindRequiredPackage(ZLIB) if (NOT BinPAC_ROOT_DIR AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/aux/binpac/CMakeLists.txt) @@ -72,26 +74,12 @@ include_directories(BEFORE ${OpenSSL_INCLUDE_DIR} ${BIND_INCLUDE_DIR} ${BinPAC_INCLUDE_DIR} + ${LibMagic_INCLUDE_DIR} + ${ZLIB_INCLUDE_DIR} ) # Optional Dependencies -set(HAVE_LIBMAGIC false) -find_package(LibMagic) -if (LIBMAGIC_FOUND) - set(HAVE_LIBMAGIC true) - include_directories(BEFORE ${LibMagic_INCLUDE_DIR}) - list(APPEND OPTLIBS ${LibMagic_LIBRARY}) -endif () - -set(HAVE_LIBZ false) -find_package(ZLIB) -if (ZLIB_FOUND) - set(HAVE_LIBZ true) - include_directories(BEFORE ${ZLIB_INCLUDE_DIR}) - list(APPEND OPTLIBS ${ZLIB_LIBRARY}) -endif () - set(USE_GEOIP false) find_package(LibGeoIP) if (LIBGEOIP_FOUND) @@ -110,6 +98,16 @@ if (ENABLE_PERFTOOLS) endif () endif () +set(brodeps + ${BinPAC_LIBRARY} + ${PCAP_LIBRARY} + ${OpenSSL_LIBRARIES} + ${BIND_LIBRARY} + ${LibMagic_LIBRARY} + ${ZLIB_LIBRARY} + ${OPTLIBS} +) + ######################################################################## ## System Introspection @@ -184,8 +182,6 @@ message( "\nAux. Tools: ${INSTALL_AUX_TOOLS}" "\n" "\nGeoIP: ${USE_GEOIP}" - "\nlibz: ${HAVE_LIBZ}" - "\nlibmagic: ${HAVE_LIBMAGIC}" "\nGoogle perftools: ${USE_PERFTOOLS}" "\n" "\n================================================================\n" diff --git a/INSTALL b/INSTALL index d1351a502f..97b2f4b0b9 100644 --- a/INSTALL +++ b/INSTALL @@ -14,16 +14,15 @@ before you begin: * OpenSSL (headers and libraries) http://www.openssl.org -Bro can make uses of some optional libraries if they are found at -installation time: - * Libmagic For identifying file types (e.g., in FTP transfers). - * LibGeoIP For geo-locating IP addresses. - * Libz For decompressing HTTP bodies by the HTTP analyzer, and for compressed Bro-to-Bro communication. +Bro can make uses of some optional libraries if they are found at +installation time: + + * LibGeoIP For geo-locating IP addresses. Bro also needs the following tools, but on most systems they will already come preinstalled: @@ -33,15 +32,14 @@ already come preinstalled: * Flex (Fast Lexical Analyzer) * Perl (Used only during the Bro build process) - Installation ============ To build and install into ``/usr/local/bro``:: - > ./configure - > make - > make install + ./configure + make + make install This will first build Bro into a directory inside the distribution called ``build/``, using default build options. It then installs all @@ -50,7 +48,7 @@ required files into ``/usr/local/bro``, including the Bro binary in You can specify a different installation directory with:: - > ./configure --prefix= + ./configure --prefix= Note that ``/usr`` and ``/opt/bro`` are the standard prefixes for binary Bro packages to be installed, so those are typically not good @@ -71,11 +69,8 @@ Running Bro =========== Bro is a complex program and it takes a bit of time to get familiar -with it. A good place for newcomers to start is the quick start guide -available here: - - http://www.bro-ids.org/documentation/quickstart.html - +with it. A good place for newcomers to start is the +:doc:`quick start guide `. For developers that wish to run Bro directly from the ``build/`` directory (i.e., without performing ``make install``), they will have @@ -83,9 +78,9 @@ to first adjust ``BROPATH`` to look for scripts inside the build directory. Sourcing either ``build/bro-path-dev.sh`` or ``build/bro-path-dev.csh`` as appropriate for the current shell accomplishes this and also augments your ``PATH`` so you can use the -Bro binary directly: +Bro binary directly:: - > ./configure - > make - > source build/bro-path-dev.sh - > bro + ./configure + make + source build/bro-path-dev.sh + bro diff --git a/Makefile b/Makefile index c736ecdbcb..c5f4bf67ce 100644 --- a/Makefile +++ b/Makefile @@ -11,23 +11,11 @@ VERSION_FULL=$(REPO)-`cat VERSION` VERSION_MIN=$(REPO)-`cat VERSION`-minimal HAVE_MODULES=git submodule | grep -v cmake >/dev/null -all: configured - ( cd $(BUILD) && make ) +SUBDIRS = $(BUILD) +$(SUBDIRS):: configured + $(MAKE) -C $@ $(MAKECMDGOALS) -install: configured - ( cd $(BUILD) && make install ) - -install-aux: configured - ( cd $(BUILD) && make install-aux ) - -clean: configured docclean - ( cd $(BUILD) && make clean ) - -doc: configured - ( cd $(BUILD) && make doc ) - -docclean: configured - ( cd $(BUILD) && make docclean ) +all install install-aux doc docclean clean: $(SUBDIRS) dist: @rm -rf $(VERSION_FULL) $(VERSION_FULL).tgz diff --git a/VERSION b/VERSION index ebffe616f6..deb6c1aac5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0-beta-28 +2.0-beta-47 diff --git a/aux/binpac b/aux/binpac index 777b8a21c4..0f99acfbf6 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 777b8a21c4c74e1f62e8b9896b082e8c059b539f +Subproject commit 0f99acfbf6205830f0db699a75554262c26427f9 diff --git a/aux/bro-aux b/aux/bro-aux index 906f970df5..1a7a9357fb 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 906f970df5f708582c7002069b787d5af586b46f +Subproject commit 1a7a9357fba88a43c90a39d8d72b42fa53b89b75 diff --git a/aux/broccoli b/aux/broccoli index e02e3cc89a..a1a03c6868 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit e02e3cc89a3efb3d7ec376154e24835b4b828be8 +Subproject commit a1a03c686866bd30ee086ff933128055a20ebd56 diff --git a/aux/broctl b/aux/broctl index 6fb4e5689d..03d39aa5d4 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 6fb4e5689d2ae0d1c4ab7af0a8df80e6eaa98fb6 +Subproject commit 03d39aa5d4ab24cd9b8e404a9ceb583d5270444c diff --git a/cmake b/cmake index 704e255d7e..44f2985475 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 704e255d7ef2faf926836c1c64d16c5b8a02b063 +Subproject commit 44f2985475e5ff6cc9061683e21ef4b184bdfc7e diff --git a/config.h.in b/config.h.in index 3783a8390d..62aa34ef43 100644 --- a/config.h.in +++ b/config.h.in @@ -14,12 +14,6 @@ /* Define if you have the `getopt_long' function. */ #cmakedefine HAVE_GETOPT_LONG -/* Define if you have the `magic' library (-lmagic). */ -#cmakedefine HAVE_LIBMAGIC - -/* Define if you have the `z' library (-lz). */ -#cmakedefine HAVE_LIBZ - /* We are on a Linux system */ #cmakedefine HAVE_LINUX diff --git a/doc/CHANGES b/doc/CHANGES new file mode 120000 index 0000000000..3e8bc8c0c8 --- /dev/null +++ b/doc/CHANGES @@ -0,0 +1 @@ +../CHANGES \ No newline at end of file diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index acef059ce8..7685230980 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -1,4 +1,73 @@ -add_custom_target(doc) -add_custom_target(docclean) +set(BIF_SRC_DIR ${PROJECT_SOURCE_DIR}/src) +set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output) +set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out) +set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/sphinx-sources) + +set(MASTER_POLICY_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/policy_index) +set(MASTER_PACKAGE_INDEX ${CMAKE_CURRENT_BINARY_DIR}/scripts/pkg_index) + +file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*") + +# configure the Sphinx config file (expand variables CMake might know about) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in + ${CMAKE_CURRENT_BINARY_DIR}/conf.py + @ONLY) add_subdirectory(scripts) + +# The "broxygen" target generates reST documentation for any outdated bro +# scripts and then uses Sphinx to generate HTML documentation from the reST +add_custom_target(broxygen + # copy the template documentation to the build directory + # to give as input for sphinx + COMMAND "${CMAKE_COMMAND}" -E copy_directory + ${DOC_SOURCE_DIR} + ${DOC_SOURCE_WORKDIR} + # copy generated policy script documentation into the + # working copy of the template documentation + COMMAND "${CMAKE_COMMAND}" -E copy_directory + ${RST_OUTPUT_DIR} + ${DOC_SOURCE_WORKDIR}/scripts + # append to the master index of all policy scripts + COMMAND cat ${MASTER_POLICY_INDEX} >> + ${DOC_SOURCE_WORKDIR}/scripts/index.rst + # append to the master index of all policy packages + COMMAND cat ${MASTER_PACKAGE_INDEX} >> + ${DOC_SOURCE_WORKDIR}/scripts/packages.rst + # construct a reST file for each group + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bin/group_index_generator.py + ${CMAKE_CURRENT_BINARY_DIR}/scripts/group_list + ${CMAKE_CURRENT_BINARY_DIR}/scripts + ${DOC_SOURCE_WORKDIR}/scripts + # tell sphinx to generate html + COMMAND sphinx-build + -b html + -c ${CMAKE_CURRENT_BINARY_DIR} + -d ${DOC_OUTPUT_DIR}/doctrees + ${DOC_SOURCE_WORKDIR} + ${DOC_OUTPUT_DIR}/html + # create symlink to the html output directory for convenience + COMMAND "${CMAKE_COMMAND}" -E create_symlink + ${DOC_OUTPUT_DIR}/html + ${CMAKE_BINARY_DIR}/html + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "[Sphinx] Generating HTML policy script docs" + # SOURCES just adds stuff to IDE projects as a convenience + SOURCES ${DOC_SOURCES}) + +# The "sphinxclean" target removes just the Sphinx input/output directories +# from the build directory. +add_custom_target(broxygen-clean + COMMAND "${CMAKE_COMMAND}" -E remove_directory + ${DOC_SOURCE_WORKDIR} + COMMAND "${CMAKE_COMMAND}" -E remove_directory + ${DOC_OUTPUT_DIR} + VERBATIM) + +add_dependencies(broxygen broxygen-clean restdoc) + +add_custom_target(doc) +add_custom_target(docclean) +add_dependencies(doc broxygen) +add_dependencies(docclean broxygen-clean restclean) diff --git a/doc/INSTALL.rst b/doc/INSTALL.rst new file mode 120000 index 0000000000..99d491b4f8 --- /dev/null +++ b/doc/INSTALL.rst @@ -0,0 +1 @@ +../INSTALL \ No newline at end of file diff --git a/doc/README b/doc/README index 1e30d5ccc9..a9f2cdbc4e 100644 --- a/doc/README +++ b/doc/README @@ -5,29 +5,37 @@ Documentation This directory contains Bro documentation in reStructured text format (see http://docutils.sourceforge.net/rst.html). -Please note that for now these files are primarily intended for use on -http://www.bro-ids.org. While the Bro build process will render local -versions into ``build/doc/`` (if docutils is found), the resulting -HTML is very minimalistic and some features are not supported. In -particular, some links will be broken. +It is the root of a Sphinx source tree and can be modified to add more +common/general documentation, style sheets, JavaScript, etc. The Sphinx +config file is produced from ``conf.py.in``, and can be edited to change +various Sphinx options. +There is also a custom Sphinx domain implemented in ``source/ext/bro.py`` +which adds some reST directives and roles that aid in generating useful +index entries and cross-references. Other extensions can be added in +a similar fashion. + +Either the ``make doc`` or ``make broxygen`` can be used to locally +render the reST files into HTML. Those targets depend on: + +* Python interpreter >= 2.5 +* `Sphinx `_ >= 1.0.1 + +After completion, HTML documentation is symlinked in ``build/html``. + +There's also ``make docclean`` and ``make broxygenclean`` targets to +clean the resulting documentation. Notes for Writing Documentation ------------------------------- -* If you want to refer to a Bro script that's part of the - distribution, use {{'`foo.bro - <{{autodoc_bro_scripts}}/path/to/foo.html>`_'}}. For example, - ``{{'{{autodoc_bro_scripts}}/scripts/base/frameworks/notice/main.html}}'}}``. +* If you want to refer to a document that's part of the + distribution, it currently needs to be copied or otherwise symlinked + somewhere in to this Sphinx source tree. Then, it can be referenced + in a toc tree or with the :doc: role. Use the :download: role to + refer to static files that will not undergo sphinx rendering. -* If you want to refer to a page on the Bro web site, use the - ``docroot`` macro (e.g., - ``{{'href="{{docroot}}/download/index.html"'}}). Make sure to - include the ``index.html`` for the main pages, just as in the - example. - -* If you want to refer to page inside this directory, use a relative - path with HTML extension. (e.g., ``href="quickstart.html``). +* If you want to refer to a page on the Bro web site, use an HTTP URL. Guidelines ---------- diff --git a/doc/_static/broxygen.css b/doc/_static/broxygen.css new file mode 100644 index 0000000000..d9c39a8506 --- /dev/null +++ b/doc/_static/broxygen.css @@ -0,0 +1,16 @@ +a:hover +{ + text-decoration:none; + color:#c24444; +} + +div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 +{ +background-color:#ffffff; +border-bottom: 1px solid #aaa; +} + +th.field-name +{ +white-space:nowrap; +} diff --git a/doc/scripts/source/_static/showhide.js b/doc/_static/showhide.js similarity index 100% rename from doc/scripts/source/_static/showhide.js rename to doc/_static/showhide.js diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html new file mode 100644 index 0000000000..90c448037e --- /dev/null +++ b/doc/_templates/layout.html @@ -0,0 +1,10 @@ +{% extends "!layout.html" %} + +{% set css_files = css_files + ["_static/broxygen.css"] %} + +{% block extrahead %} + + +{% endblock %} + +{% block relbar2 %}{% endblock %} diff --git a/doc/scripts/group_index_generator.py b/doc/bin/group_index_generator.py similarity index 100% rename from doc/scripts/group_index_generator.py rename to doc/bin/group_index_generator.py diff --git a/doc/cluster.rst b/doc/cluster.rst index afd5f2fe35..9bffc35fec 100644 --- a/doc/cluster.rst +++ b/doc/cluster.rst @@ -11,9 +11,7 @@ Architecture The figure below illustrates the main components of a Bro cluster. -.. {{git_pull('bro:doc/deployment.png')}} - -.. image:: deployment.bro.png +.. image:: images/deployment.png Tap *** diff --git a/doc/components/broccoli-python/README.rst b/doc/components/broccoli-python/README.rst new file mode 120000 index 0000000000..4187e87202 --- /dev/null +++ b/doc/components/broccoli-python/README.rst @@ -0,0 +1 @@ +../../../aux/broccoli/bindings/broccoli-python/README \ No newline at end of file diff --git a/doc/components/broccoli/README.rst b/doc/components/broccoli/README.rst new file mode 120000 index 0000000000..d32c70ccd9 --- /dev/null +++ b/doc/components/broccoli/README.rst @@ -0,0 +1 @@ +../../../aux/broccoli/README \ No newline at end of file diff --git a/doc/components/broctl/README.rst b/doc/components/broctl/README.rst new file mode 120000 index 0000000000..cba305f48a --- /dev/null +++ b/doc/components/broctl/README.rst @@ -0,0 +1 @@ +../../../aux/broctl/doc/broctl.rst \ No newline at end of file diff --git a/doc/components/btest/README.rst b/doc/components/btest/README.rst new file mode 120000 index 0000000000..0da2935df1 --- /dev/null +++ b/doc/components/btest/README.rst @@ -0,0 +1 @@ +../../../aux/btest/README \ No newline at end of file diff --git a/doc/components/capstats/README.rst b/doc/components/capstats/README.rst new file mode 120000 index 0000000000..cb2380145d --- /dev/null +++ b/doc/components/capstats/README.rst @@ -0,0 +1 @@ +../../../aux/broctl/aux/capstats/README \ No newline at end of file diff --git a/doc/components/pysubnettree/README.rst b/doc/components/pysubnettree/README.rst new file mode 120000 index 0000000000..42ce17d303 --- /dev/null +++ b/doc/components/pysubnettree/README.rst @@ -0,0 +1 @@ +../../../aux/broctl/aux/pysubnettree/README \ No newline at end of file diff --git a/doc/components/trace-summary/README.rst b/doc/components/trace-summary/README.rst new file mode 120000 index 0000000000..78778364bd --- /dev/null +++ b/doc/components/trace-summary/README.rst @@ -0,0 +1 @@ +../../../aux/broctl/aux/trace-summary/README \ No newline at end of file diff --git a/doc/scripts/conf.py.in b/doc/conf.py.in similarity index 87% rename from doc/scripts/conf.py.in rename to doc/conf.py.in index 419209f580..e997796eda 100644 --- a/doc/scripts/conf.py.in +++ b/doc/conf.py.in @@ -15,7 +15,7 @@ import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('source/ext')) +sys.path.insert(0, os.path.abspath('sphinx-sources/ext')) # -- General configuration ----------------------------------------------------- @@ -24,10 +24,10 @@ sys.path.insert(0, os.path.abspath('source/ext')) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['bro'] +extensions = ['bro', 'rst_directive'] # Add any paths that contain templates here, relative to this directory. -templates_path = ['source/_templates'] +templates_path = ['sphinx-sources/_templates', 'sphinx-sources/_static'] # The suffix of source filenames. source_suffix = '.rst' @@ -95,7 +95,31 @@ html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +html_theme_options = { +"rightsidebar": "true", +"stickysidebar": "true", +"externalrefs": "true", +"footerbgcolor": "#333", +"footertextcolor": "#ddd", +"sidebarbgcolor": "#ffffff", +#"sidebarbtncolor": "", +"sidebartextcolor": "#333", +"sidebarlinkcolor": "#2a85a7", +"relbarbgcolor": "#ffffff", +"relbartextcolor": "#333", +"relbarlinkcolor": "#2a85a7", +"bgcolor": "#ffffff", +"textcolor": "#333", +"linkcolor": "#2a85a7", +"visitedlinkcolor": "#2a85a7", +"headbgcolor": "#f0f0f0", +"headtextcolor": "#000", +"headlinkcolor": "#2a85a7", +"codebgcolor": "#FFFAE2", +#"codetextcolor": "", +"bodyfont": "Arial, Helvetica, sans-serif", +"headfont": "Palatino,'Palatino Linotype',Georgia,serif", +} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] @@ -119,7 +143,7 @@ html_theme = 'default' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['source/_static'] +html_static_path = ['sphinx-sources/_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. @@ -130,7 +154,9 @@ html_static_path = ['source/_static'] #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +html_sidebars = { +'**': ['localtoc.html', 'sourcelink.html', 'searchbox.html'], +} # Additional templates that should be rendered to pages, maps page names to # template names. @@ -163,7 +189,7 @@ html_static_path = ['source/_static'] #html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'Brodoc' +htmlhelp_basename = 'Broxygen' # -- Options for LaTeX output -------------------------------------------------- diff --git a/doc/scripts/source/ext/bro.py b/doc/ext/bro.py similarity index 100% rename from doc/scripts/source/ext/bro.py rename to doc/ext/bro.py diff --git a/doc/ext/bro_lexer/__init__.py b/doc/ext/bro_lexer/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/doc/ext/bro_lexer/__init__.pyc b/doc/ext/bro_lexer/__init__.pyc new file mode 100644 index 0000000000..1b03eaf1c1 Binary files /dev/null and b/doc/ext/bro_lexer/__init__.pyc differ diff --git a/doc/ext/bro_lexer/bro.py b/doc/ext/bro_lexer/bro.py new file mode 100644 index 0000000000..8cb4475f3b --- /dev/null +++ b/doc/ext/bro_lexer/bro.py @@ -0,0 +1,76 @@ +from pygments.lexer import RegexLexer, bygroups, include +from pygments.token import * + +__all__ = ["BroLexer"] + +class BroLexer(RegexLexer): + name = 'Bro' + aliases = ['bro'] + filenames = ['*.bro'] + + _hex = r'[0-9a-fA-F_]+' + _float = r'((\d*\.?\d+)|(\d+\.?\d*))([eE][-+]?\d+)?' + _h = r'[A-Za-z0-9][-A-Za-z0-9]*' + + tokens = { + 'root': [ + # Whitespace + ('^@.*?\n', Comment.Preproc), + (r'#.*?\n', Comment.Single), + (r'\n', Text), + (r'\s+', Text), + (r'\\\n', Text), + # Keywords + (r'(add|alarm|break|case|const|continue|delete|do|else|enum|event' + r'|export|for|function|if|global|local|module|next' + r'|of|print|redef|return|schedule|when|while)\b', Keyword), + (r'(addr|any|bool|count|counter|double|file|int|interval|net' + r'|pattern|port|record|set|string|subnet|table|time|timer' + r'|vector)\b', Keyword.Type), + (r'(T|F)\b', Keyword.Constant), + (r'(&)((?:add|delete|expire)_func|attr|(create|read|write)_expire' + r'|default|disable_print_hook|raw_output|encrypt|group|log' + r'|mergeable|optional|persistent|priority|redef' + r'|rotate_(?:interval|size)|synchronized)\b', bygroups(Punctuation, + Keyword)), + (r'\s+module\b', Keyword.Namespace), + # Addresses, ports and networks + (r'\d+/(tcp|udp|icmp|unknown)\b', Number), + (r'(\d+\.){3}\d+', Number), + (r'(' + _hex + r'){7}' + _hex, Number), + (r'0x' + _hex + r'(' + _hex + r'|:)*::(' + _hex + r'|:)*', Number), + (r'((\d+|:)(' + _hex + r'|:)*)?::(' + _hex + r'|:)*', Number), + (r'(\d+\.\d+\.|(\d+\.){2}\d+)', Number), + # Hostnames + (_h + r'(\.' + _h + r')+', String), + # Numeric + (_float + r'\s+(day|hr|min|sec|msec|usec)s?\b', Literal.Date), + (r'0[xX]' + _hex, Number.Hex), + (_float, Number.Float), + (r'\d+', Number.Integer), + (r'/', String.Regex, 'regex'), + (r'"', String, 'string'), + # Operators + (r'[!%*/+-:<=>?~|]', Operator), + (r'([-+=&|]{2}|[+-=!><]=)', Operator), + (r'(in|match)\b', Operator.Word), + (r'[{}()\[\]$.,;]', Punctuation), + # Identfier + (r'([_a-zA-Z]\w*)(::)', bygroups(Name, Name.Namespace)), + (r'[a-zA-Z_][a-zA-Z_0-9]*', Name) + ], + 'string': [ + (r'"', String, '#pop'), + (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape), + (r'[^\\"\n]+', String), + (r'\\\n', String), + (r'\\', String) + ], + 'regex': [ + (r'/', String.Regex, '#pop'), + (r'\\[\\nt/]', String.Regex), # String.Escape is too intense. + (r'[^\\/\n]+', String.Regex), + (r'\\\n', String.Regex), + (r'\\', String.Regex) + ] + } diff --git a/doc/ext/bro_lexer/bro.pyc b/doc/ext/bro_lexer/bro.pyc new file mode 100644 index 0000000000..6471e1528d Binary files /dev/null and b/doc/ext/bro_lexer/bro.pyc differ diff --git a/doc/ext/rst_directive.py b/doc/ext/rst_directive.py new file mode 100644 index 0000000000..434eef2c61 --- /dev/null +++ b/doc/ext/rst_directive.py @@ -0,0 +1,180 @@ +def setup(app): + pass + +# -*- coding: utf-8 -*- +""" + +Modified version of the the Pygments reStructuredText directive. -Robin + +This provides two new directives: + + - .. code:: [] + + Highlights the following code block according to if + given (e.g., "c", "python", etc.). + + - .. console:: + + Highlits the following code block as a shell session. + + For compatibility with the original version, "sourcecode" is + equivalent to "code". + +Original comment: + + The Pygments reStructuredText directive + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This fragment is a Docutils_ 0.5 directive that renders source code + (to HTML only, currently) via Pygments. + + To use it, adjust the options below and copy the code into a module + that you import on initialization. The code then automatically + registers a ``sourcecode`` directive that you can use instead of + normal code blocks like this:: + + .. sourcecode:: python + + My code goes here. + + If you want to have different code styles, e.g. one with line numbers + and one without, add formatters with their names in the VARIANTS dict + below. You can invoke them instead of the DEFAULT one by using a + directive option:: + + .. sourcecode:: python + :linenos: + + My code goes here. + + Look at the `directive documentation`_ to get all the gory details. + + .. _Docutils: http://docutils.sf.net/ + .. _directive documentation: + http://docutils.sourceforge.net/docs/howto/rst-directives.html + + :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +# Options +# ~~~~~~~ + +# Set to True if you want inline CSS styles instead of classes +INLINESTYLES = False + +from pygments.formatters import HtmlFormatter + +class MyHtmlFormatter(HtmlFormatter): + def format_unencoded(self, tokensource, outfile): + + # A NOP currently. + new_tokens = [] + for (i, piece) in tokensource: + new_tokens += [(i, piece)] + + return super(MyHtmlFormatter, self).format_unencoded(new_tokens, outfile) + +# The default formatter +DEFAULT = MyHtmlFormatter(noclasses=INLINESTYLES, cssclass="pygments") + +# Add name -> formatter pairs for every variant you want to use +VARIANTS = { + # 'linenos': HtmlFormatter(noclasses=INLINESTYLES, linenos=True), +} + + +import textwrap + +from docutils import nodes +from docutils.parsers.rst import directives, Directive + +from pygments import highlight +from pygments.lexers import get_lexer_by_name, guess_lexer, TextLexer +from pygments.token import Text, Keyword, Error, Operator, Name +from pygments.filter import Filter + +# Ugly hack to register the Bro lexer. I'm sure there's a better way to do it, +# but it's not obvious ... +from bro_lexer.bro import BroLexer +from pygments.lexers._mapping import LEXERS +LEXERS['BroLexer'] = ('bro_lexer.bro', BroLexer.name, BroLexer.aliases, BroLexer.filenames, ()) + +class Pygments(Directive): + """ Source code syntax hightlighting. + """ + #max_line_length = 68 + max_line_length = 0 + + required_arguments = 0 + optional_arguments = 1 + final_argument_whitespace = True + option_spec = dict([(key, directives.flag) for key in VARIANTS]) + has_content = True + + def wrapped_content(self): + content = [] + + if Console.max_line_length: + for line in self.content: + content += textwrap.wrap(line, Console.max_line_length, subsequent_indent=" ") + else: + content = self.content + + return u'\n'.join(content) + + def run(self): + self.assert_has_content() + + content = self.wrapped_content() + + if len(self.arguments) > 0: + try: + lexer = get_lexer_by_name(self.arguments[0]) + except (ValueError, IndexError): + # lexer not found, use default. + lexer = TextLexer() + else: + lexer = guess_lexer(content) + + # import sys + # print >>sys.stderr, self.arguments, lexer.__class__ + + # take an arbitrary option if more than one is given + formatter = self.options and VARIANTS[self.options.keys()[0]] or DEFAULT + parsed = highlight(content, lexer, formatter) + return [nodes.raw('', parsed, format='html')] + +class MyFilter(Filter): + def filter(self, lexer, stream): + + bol = True + + for (ttype, value) in stream: + # Color the '>' prompt sign. + if bol and ttype is Text and value == ">": + ttype = Name.Variable.Class # This gives us a nice red. + + # Discolor builtin, that can look funny. + if ttype is Name.Builtin: + ttype = Text + + bol = value.endswith("\n") + + yield (ttype, value) + +class Console(Pygments): + required_arguments = 0 + optional_arguments = 0 + + def run(self): + self.assert_has_content() + content = self.wrapped_content() + lexer = get_lexer_by_name("sh") + lexer.add_filter(MyFilter()) + parsed = highlight(content, lexer, DEFAULT) + return [nodes.raw('', parsed, format='html')] + +directives.register_directive('sourcecode', Pygments) +directives.register_directive('code', Pygments) +directives.register_directive('console', Console) diff --git a/doc/faq.rst b/doc/faq.rst new file mode 100644 index 0000000000..bdb1f50292 --- /dev/null +++ b/doc/faq.rst @@ -0,0 +1,145 @@ + +========================== +Frequently Asked Questions +========================== + +.. raw:: html + +
+ +.. contents:: + +Installation and Configuration +============================== + +How can I tune my operating system for best capture performance? +---------------------------------------------------------------- + +Here are some pointers to more information: + +* Fabian Schneider's research on `high performance packet capture + `_ + +* `NSMWiki `_ has page on + *Collecting Data*. + +* An `IMC 2010 paper + `_ by + Lothar Braun et. al evaluates packet capture performance on + commodity hardware + +What does an error message like ``internal error: NB-DNS error`` mean? +--------------------------------------------------------------------------------------------------------------------------------- + +That often means that DNS is not set up correctly on the system +running Bro. Try verifying from the command line that DNS lookups +work, e.g., ``host www.google.com``. + + +Usage +===== + +How can I identify backscatter? +------------------------------- + +Identifying backscatter via connections labeled as ``OTH`` is not +a reliable means to detect backscatter. Use rather the following +procedure: + +* Enable connection history via ``redef record_state_history=T`` to + track all control/data packet types in connection logs. + +* Backscatter is now visible in terms of connections that never had an + initial ``SYN`` but started instead with a ``SYN-ACK`` or ``RST`` + (though this latter generally is just discarded). + +Is there help for understanding Bro's resource consumption? +----------------------------------------------------------- + +There are two scripts that collect statistics on resource usage: +``stats.bro`` and ``profiling.bro``. The former is quite lightweight, +while the latter should only be used for debugging. Furthermore, +there's also ``print-globals.bro``, which prints the size of all +global script variable at termination. + +How can I capture packets as an unprivileged user? +-------------------------------------------------- + +Normally, unprivileged users cannot capture packets from a network +interface, which means they would not be able to use Bro to read/analyze +live traffic. However, there are ways to enable packet capture +permission for non-root users, which is worth doing in the context of +using Bro to monitor live traffic + +With Linux Capabilities +^^^^^^^^^^^^^^^^^^^^^^^ + +Fully implemented since Linux kernel 2.6.24, capabilities are a way of +parceling superuser privileges into distinct units. Attach capabilities +required to capture packets to the ``bro`` executable file like this: + +.. console:: + + sudo setcap cap_net_raw,cap_net_admin=eip /path/to/bro + +Now any unprivileged user should have the capability to capture packets +using Bro provided that they have the traditional file permissions to +read/execute the ``bro`` binary. + +With BPF Devices +^^^^^^^^^^^^^^^^ + +Systems using Berkeley Packet Filter (BPF) (e.g. FreeBSD & Mac OS X) +can allow users with read access to a BPF device to capture packets from +it using libpcap. + +* Example of manually changing BPF device permissions to allow users in + the ``admin`` group to capture packets: + +.. console:: + + sudo chgrp admin /dev/bpf* + sudo chmod g+r /dev/bpf* + +* Example of configuring devfs to set permissions of BPF devices, adding + entries to ``/etc/devfs.conf`` to grant ``admin`` group permission to + capture packets: + +.. console:: + + sudo sh -c 'echo "own bpf root:admin" >> /etc/devfs.conf' + sudo sh -c 'echo "perm bpf 0640" >> /etc/devfs.conf' + sudo service devfs restart + +.. note:: As of Mac OS X 10.6, the BPF device is on devfs, but the used version + of devfs isn't capable of setting the device permissions. The permissions + can be changed manually, but they will not survive a reboot. + +Why isn't Bro producing the logs I expect? (A Note About Checksums) +------------------------------------------------------------------- + +Normally, Bro's event engine will discard packets which don't have valid +checksums. This can be a problem if one wants to analyze locally +generated/captured traffic on a system that offloads checksumming to the +network adapter. In that case, all transmitted/captured packets will have +bad checksums because they haven't yet been calculated by the NIC, thus +such packets will not undergo analysis defined in Bro policy scripts as they +normally would. Bad checksums in traces may also be a result of some packet +alteration tools. + +Bro has two options to workaround such situations and ignore bad checksums: + +1) The ``-C`` command line option to ``bro``. +2) An option called ``ignore_checksums`` that can be redefined at the policy + policy script layer (e.g. in your ``$PREFIX/share/bro/site/local/bro``): + + .. code:: bro + + redef ignore_checksums = T; + +The other alternative is to disable checksum offloading for your +network adapter, but this is not always possible or desirable. + +.. raw:: html + +
diff --git a/doc/geoip.rst b/doc/geoip.rst index 53413b5bee..bd9ae0c08d 100644 --- a/doc/geoip.rst +++ b/doc/geoip.rst @@ -3,7 +3,7 @@ GeoLocation =========== -.. class:: opening +.. rst-class:: opening During the process of creating policy scripts the need may arise to find the geographic location for an IP address. Bro has support diff --git a/doc/deployment.png b/doc/images/deployment.png similarity index 100% rename from doc/deployment.png rename to doc/images/deployment.png diff --git a/doc/index.rst b/doc/index.rst index c7c85bd21f..022552f3d4 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,50 +1,72 @@ - +.. Bro documentation master file Bro Documentation ================= -`Getting Started <{{git('bro:doc/quickstart.rst')}}>`_ - A quick introduction into using Bro 2.x. +.. toctree:: + :maxdepth: 1 -`Bro 1.5 to 2.0 Upgrade Guide <{{git('bro:doc/upgrade.rst')}}>`_ - Guidelines and notes about upgrading from Bro 1.5 to 2.x. Lots of - things have changed, so make sure to read this when upgrading. - -`BroControl <{{git('broctl:doc/broctl.rst')}}>`_ - An interactive console for managing Bro installations. - -`Script Reference <{{autodoc_bro_scripts}}/index.html>`_ - A complete reference of all policy scripts shipped with Bro. - -`FAQ <{{docroot}}/documentation/faq.html>`_ - A list with frequently asked questions. - -`How to Report a Problem <{{docroot}}/documentation/reporting-problems.html>`_ - Some advice for when you see Bro doing something you believe it - shouldn't. + INSTALL + quickstart + upgrade + faq + reporting-problems Frameworks ---------- -Bro comes with a number of frameworks, some of which are described in -more detail here: +.. toctree:: + :maxdepth: 1 -`Notice <{{git('bro:doc/notice.rst')}}>`_ - The notice framework. - -`Logging <{{git('bro:doc/logging.rst')}}>`_ - Customizing and extensing Bro's logging. - -`Cluster <{{git('bro:doc/cluster.rst')}}>`_ - Setting up a Bro Cluster when a single box can't handle the traffic anymore. - -`Signatures <{{git('bro:doc/signatures.rst')}}>`_ - Bro has support for traditional NIDS signatures as well. + notice + logging + cluster + signatures How-Tos ------- -We also collect more specific How-Tos on specific topics: +.. toctree:: + :maxdepth: 1 -`Using GeoIP in Bro scripts <{{git('bro:doc/geoip.rst')}}>`_ - Installation and usage of the the GeoIP library. + geoip + +Script Reference +---------------- + +.. toctree:: + :maxdepth: 1 + + scripts/common + scripts/builtins + scripts/bifs + scripts/packages + scripts/index + +Other Bro Components +-------------------- + +.. toctree:: + :maxdepth: 1 + + components/btest/README + components/broccoli/README + components/broccoli-python/README + components/broctl/README + components/capstats/README + components/pysubnettree/README + components/trace-summary/README + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`search` + +Internal References +------------------- + +.. toctree:: + :maxdepth: 1 + + scripts/internal diff --git a/doc/logging.rst b/doc/logging.rst index e7151b7f47..2817cadd45 100644 --- a/doc/logging.rst +++ b/doc/logging.rst @@ -2,7 +2,7 @@ Customizing Bro's Logging ========================== -.. class:: opening +.. rst-class:: opening Bro comes with a flexible key-value based logging interface that allows fine-grained control of what gets logged and how it is diff --git a/doc/notice.rst b/doc/notice.rst index 1322d44585..b1ffdacb75 100644 --- a/doc/notice.rst +++ b/doc/notice.rst @@ -2,7 +2,7 @@ Notice Framework ================ -.. class:: opening +.. rst-class:: opening One of the easiest ways to customize Bro is writing a local notice policy. Bro can detect a large number of potentially interesting diff --git a/doc/quickstart.rst b/doc/quickstart.rst index 22523e1618..0534186cde 100644 --- a/doc/quickstart.rst +++ b/doc/quickstart.rst @@ -3,12 +3,13 @@ .. _MacPorts: http://www.macports.org .. _Fink: http://www.finkproject.org .. _Homebrew: http://mxcl.github.com/homebrew +.. _bro downloads page: http://bro-ids.org/download/index.html ================= Quick Start Guide ================= -.. class:: opening +.. rst-class:: opening The short story for getting Bro up and running in a simple configuration for analysis of either live traffic from a network interface or a packet @@ -26,20 +27,19 @@ source code forms. Pre-Built Binary Release Packages --------------------------------- -See the `downloads page <{{docroot}}/download/index.html>`_ for currently -supported/targeted platforms. +See the `bro downloads page`_ for currently supported/targeted platforms. * RPM .. console:: - > sudo yum localinstall Bro-all*.rpm + sudo yum localinstall Bro-all*.rpm * DEB .. console:: - > sudo gdebi Bro-all-*.deb + sudo gdebi Bro-all-*.deb * MacOS Disk Image with Installer @@ -60,13 +60,13 @@ Required Dependencies .. console:: - > sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig + sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel file-devel * DEB/Debian-based Linux: .. console:: - > sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig + sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev libmagic-dev * FreeBSD @@ -75,7 +75,7 @@ Required Dependencies .. console:: - > sudo pkg_add -r cmake swig bison python + sudo pkg_add -r cmake swig bison python * Mac OS X @@ -94,29 +94,28 @@ Required Dependencies Optional Dependencies ~~~~~~~~~~~~~~~~~~~~~ -Bro can use libmagic for identifying file types, libGeoIP for geo-locating -IP addresses, libz for (de)compression during analysis and communication, -and sendmail for sending emails. +Bro can use libGeoIP for geo-locating IP addresses and sendmail for +sending emails. * RPM/RedHat-based Linux: .. console:: - > sudo yum install zlib-devel file-devel GeoIP-devel sendmail + sudo yum install GeoIP-devel sendmail * DEB/Debian-based Linux: .. console:: - > sudo apt-get install zlib1g-dev libmagic-dev libgeoip-dev sendmail + sudo apt-get install libgeoip-dev sendmail * Ports-based FreeBSD .. console:: - > sudo pkg_add -r GeoIP + sudo pkg_add -r GeoIP - libz, libmagic, and sendmail are typically already available. + sendmail is typically already available. * Mac OS X @@ -125,20 +124,20 @@ and sendmail for sending emails. Fink Homebrew), they should be automatically detected and Bro will compile against them. -Additional steps may be needed to `get the right GeoIP database -<{{git('bro:doc/geoip.rst')}}>`_. +Additional steps may be needed to :doc:`get the right GeoIP database ` Compiling Bro Source Code ~~~~~~~~~~~~~~~~~~~~~~~~~ Bro releases are bundled into source packages for convenience and -available from the `downloads page <{{docroot}}/download/index.html>`_. +available from the `bro downloads page`_. -The latest Bro development versions are obtainable through git repositories -hosted at `git.bro-ids.org `_. See our `git development -documentation <{{docroot}}/development/process.html>`_ for comprehensive -information on Bro's use of git revision control, but the short story for -downloading the full source code experience for Bro via git is: +The latest Bro development versions are obtainable through git +repositories hosted at `git.bro-ids.org `_. See +our `git development documentation +`_ for comprehensive +information on Bro's use of git revision control, but the short story +for downloading the full source code experience for Bro via git is: .. console:: @@ -157,9 +156,9 @@ desired root install path): .. console:: - > ./configure --prefix=/desired/install/path - > make - > make install + ./configure --prefix=/desired/install/path + make + make install The default installation prefix is ``/usr/local/bro``, which would typically require root privileges when doing the ``make install``. @@ -174,13 +173,13 @@ Bourne-Shell Syntax: .. console:: - > export PATH=/usr/local/bro/bin:$PATH + export PATH=/usr/local/bro/bin:$PATH C-Shell Syntax: .. console:: - > setenv PATH /usr/local/bro/bin:$PATH + setenv PATH /usr/local/bro/bin:$PATH Or substitute ``/opt/bro/bin`` instead if you installed from a binary package. @@ -211,7 +210,7 @@ Now start the BroControl shell like: .. console:: - > broctl + broctl Since this is the first-time use of the shell, perform an initial installation of the BroControl configuration: @@ -233,10 +232,9 @@ policy and output the results in ``$PREFIX/logs``. .. note:: The user starting BroControl needs permission to capture network traffic. If you are not root, you may need to grant further - privileges to the account you're using; see the `FAQ - <{{docroot}}/documentation/faq.html>`_. Also, if it - looks like Bro is not seeing any traffic, check out the FAQ entry - checksum offloading. + privileges to the account you're using; see the :doc:`FAQ `. + Also, if it looks like Bro is not seeing any traffic, check out + the FAQ entry on checksum offloading. You can leave it running for now, but to stop this Bro instance you would do: @@ -244,9 +242,7 @@ You can leave it running for now, but to stop this Bro instance you would do: [BroControl] > stop -We also recommend to insert the following entry into `crontab`: - -.. console:: +We also recommend to insert the following entry into `crontab`:: 0-59/5 * * * * $PREFIX/bin/broctl cron @@ -373,9 +369,7 @@ the variable's value may not change at run-time, but whose initial value can be modified via the ``redef`` operator at parse-time. So let's continue on our path to modify the behavior for the two SSL -and SSH notices. Looking at -`$PREFIX/share/bro/base/frameworks/notice/main.bro -<{{autodoc_bro_scripts}}/scripts/base/frameworks/notice/main.html>`_, +and SSH notices. Looking at :doc:`scripts/base/frameworks/notice/main`, we see that it advertises: .. code:: bro @@ -477,8 +471,7 @@ tweak the most basic options. Here's some suggestions on what to explore next: * Reading the code of scripts that ship with Bro is also a great way to gain understanding of the language and how you can start writing your own custom analysis. -* Review the `FAQ <{{docroot}}/documentation/faq.html>`_. -* Check out more `documentation <{{docroot}}/documentation/index.html>`_. +* Review the :doc:`FAQ `. * Continue reading below for another mini-tutorial on using Bro as a standalone command-line utility. @@ -496,7 +489,7 @@ Analyzing live traffic from an interface is simple: .. console:: - > bro -i en0 + bro -i en0 ``en0`` can be replaced by the interface of your choice and for the list of scripts, you can just use "all" for now to perform all the default analysis @@ -504,7 +497,7 @@ that's available. Bro will output log files into the working directory. -.. note:: The `FAQ <{{docroot}}/documentation/faq.html>`_ entries about +.. note:: The :doc:`FAQ ` entries about capturing as an unprivileged user and checksum offloading are particularly relevant at this point. @@ -513,7 +506,7 @@ command-line: .. console:: - > bro -i en0 local + bro -i en0 local This will cause Bro to print a warning about lacking the ``Site::local_nets`` variable being configured. You can supply this @@ -522,7 +515,7 @@ in place of the example subnets): .. console:: - > bro -r mypackets.trace local "Site::local_nets += { 1.2.3.0/24, 5.6.7.0/24 }" + bro -r mypackets.trace local "Site::local_nets += { 1.2.3.0/24, 5.6.7.0/24 }" Reading Packet Capture (pcap) Files @@ -533,7 +526,7 @@ like this: .. console:: - > sudo tcpdump -i en0 -s 0 -w mypackets.trace + sudo tcpdump -i en0 -s 0 -w mypackets.trace Where ``en0`` can be replaced by the correct interface for your system as shown by e.g. ``ifconfig``. (The ``-s 0`` argument tells it to capture @@ -544,7 +537,7 @@ and tell Bro to perform all the default analysis on the capture which primarily .. console:: - > bro -r mypackets.trace + bro -r mypackets.trace Bro will output log files into the working directory. @@ -553,7 +546,7 @@ script that we include as a suggested configuration: .. console:: - > bro -r mypackets.trace local + bro -r mypackets.trace local Telling Bro Which Scripts to Load @@ -563,7 +556,7 @@ A command-line invocation of Bro typically looks like: .. console:: - > bro + bro Where the last arguments are the specific policy scripts that this Bro instance will load. These arguments don't have to include the ``.bro`` @@ -578,7 +571,7 @@ logging) and adds SSL certificate validation. .. console:: - > bro -r mypackets.trace protocols/ssl/validate-certs + bro -r mypackets.trace protocols/ssl/validate-certs You might notice that a script you load from the command line uses the ``@load`` directive in the Bro language to declare dependence on other scripts. diff --git a/doc/reporting-problems.rst b/doc/reporting-problems.rst new file mode 100644 index 0000000000..a1105708eb --- /dev/null +++ b/doc/reporting-problems.rst @@ -0,0 +1,194 @@ + +Reporting Problems +================== + +.. rst-class:: opening + + Here we summarizes some steps to follow when you see Bro doing + something it shouldn't. To provide help, it is often crucial for + us to have a way of reliably reproducing the effect you're seeing. + Unfortunately, reproducing problems can be rather tricky with Bro + because more often than not, they occur only in either very rare + situations or only after Bro has been running for some time. In + particular, getting a small trace showing a specific effect can be + a real problem. In the following, we'll summarize some strategies + to this end. + +Reporting Problems +------------------ + +Generally, when you encounter a problem with Bro, the best thing to do +is opening a new ticket in `Bro's issue tracker +`__ and include information on how to +reproduce the issue. Ideallt, your ticket should come with the +following: + +* The Bro version you're using (if working directly from the git + repository, the branch and revision number.) + +* The output you're seeing along with a description what you'd expect + Bro to do instead. + +* A *small* trace in `libpcap format `__ + demonstrating the effect (assuming the problem doesn't happen right + at startup already). + +* The exact command-line you're using to run Bro with that trace. If + you can, please try to run the Bro binary directly from the command + line rather than using BroControl. + +* Any non-standard scripts you're using (but please only those really + necessary; just a small code snippet triggering the problem would + perfect). + +* If you encounter a crash, information from the core dump, such as + the stack backtrace, can be very helpful. See below for more on + this. + + +How Do I Get a Trace File? +-------------------------- + +As Bro is usually running live, coming up with a small trace file that +reproduces a problem can turn out to be quite a challenge. Often it +works to best to start with a large trace that triggers the problem, +and then successively thin it out as much a possible. + +To get to the initial large trace, here are few things you can try: + +* Capture a trace with `tcpdump `__, either + on the same interface Bro is running on, or on another host where + you can generate traffic of the kind likely triggering the problem + (e.g., if you're seeing problems with the HTTP analyzer, record some + of your Web browsing on your desktop.) When using tcpdump, don't + forget to record *complete* packets (``tcpdump -s 0 ...``). You can + reduce the amount of traffic captured by using a suitable BPF filter + (e.g., for HTTP only, try ``port 80``). + +* Bro's command-line option ``-w `` records all packets it + processes into the given the file. You can then later run Bro + offline on this trace and it will process the packets in the same + way as it did live. This is particularly helpful with problems that + only occur after Bro has already been running for some time. For + example, sometimes a crash may be triggered by a particular kind of + traffic only occurring rarely. Running Bro live with ``-w`` and + then, after the crash, offline on the recorded trace might, with a + little bit of luck, reproduce the the problem reliably. However, be + careful with ``-w``: it can result in huge trace files, quickly + filling up your disk. (One way to mitigate the space issues is to + periodically delete the trace file by configuring + ``rotate-logs.bro`` accordingly. BroControl does that for you if you + set its ``SaveTraces`` option.) + +* Finally, you can try running Bro on a publically available trace + file, such as `anonymized FTP traffic `__, `headers-only enterprise traffic + `__, or + `Defcon traffic `__. Some of these + particularly stress certain components of Bro (e.g., the Defcon + traces contain tons of scans). + +Once you have a trace that demonstrates the effect, you will often +notice that it's pretty big, in particular if recorded from the link +you're monitoring. Therefore, the next step is to shrink its size as +much as possible. Here are a few things you can try to this end: + +* Very often, a single connection is able to demonstrate the problem. + If you can identify which one it is (e.g., from one of Bro's + ``*.log`` files) you can extract the connection's packets from the + trace usong tcpdump by filtering for the corresponding 4-tuple of + addresses and ports: + + .. console:: + + > tcpdump -r large.trace -w small.trace host and port and host and port + +* If you can't reduce the problem to a connection, try to identify + either a host pair or a single host triggering it, and filter down + the trace accordingly. + +* You can try to extract a smaller time slice from the trace using + `TCPslice `__. For example, to + extract the first 100 seconds from the trace: + + .. console:: + + # Test comment + > tcpslice +100 out + +Alternatively, tcpdump extracts the first ``n`` packets with its +option ``-c ``. + + +Getting More Information After a Crash +-------------------------------------- + +If Bro crashes, a *core dump* can be very helpful to nail down the +problem. Examining a core is not for the faint of heart but can reveal +extremely useful information. + +First, you should configure Bro with the option ``--enable-debug`` and +recompile; this will disable all compiler optimizations and thus make +the core dump more useful (don't expect great performance with this +version though; compiling Bro without optimization has a noticeable +impact on its CPU usage.). Then enable core dumps if you don't have +already (e.g., ``ulimit -c unlimited`` if you're using a bash). + +Once Bro has crashed, start gdb with the Bro binary and the file +containing the core dump. (Alternatively, you can also run Bro +directly inside gdb instead of working from a core file.) The first +helpful information to include with your tracker ticket is a stack +backtrace, which you get with gdb's ``bt`` command: + +.. console:: + + > gdb bro core + [...] + > bt + + +If the crash occurs inside Bro's script interpreter, the next thing to +do is identifying the line of script code processed just before the +abnormal termination. Look for methods in the stack backtrace which +belong to any of the script interpreter's classes. Roughly speaking, +these are all classes with names ending in ``Expr``, ``Stmt``, or +``Val``. Then climb up the stack with ``up`` until you reach the first +of these methods. The object to which ``this`` is pointing will have a +``Location`` object, which in turn contains the file name and line +number of the corresponding piece of script code. Continuing the +example from above, here's how to get that information: + +.. console:: + + [in gdb] + > up + > ... + > up + > print this->location->filename + > print this->location->first_line + + +If the crash occurs while processing input packets but you cannot +directly tell which connection is responsible (and thus not extract +its packets from the trace as suggested above), try getting the +4-tuple of the connection currently being processed from the core dump +by again examining the stack backtrace, this time looking for methods +belonging to the ``Connection`` class. That class has members +``orig_addr``/``resp_addr`` and ``orig_port``/``resp_port`` storing +(pointers to) the IP addresses and ports respectively: + +.. console:: + + [in gdb] + > up + > ... + > up + > printf "%08x:%04x %08x:%04x\n", *this->orig_addr, this->orig_port, *this->resp_addr, this->resp_port + + +Note that these values are stored in `network byte order +`__ +so you will need flip the bytes around if you are on a low-endian +machine (which is why the above example prints them in hex). For +example, if an IP address prints as ``0100007f`` , that's 127.0.0.1 . + diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt index 1cf2f768ec..c14eab6d25 100644 --- a/doc/scripts/CMakeLists.txt +++ b/doc/scripts/CMakeLists.txt @@ -1,16 +1,3 @@ -set(BIF_SRC_DIR ${PROJECT_SOURCE_DIR}/src) -set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output) -set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out) -set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source) -set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/source) - -file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*") - -# configure the Sphinx config file (expand variables CMake might know about) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in - ${CMAKE_CURRENT_BINARY_DIR}/conf.py - @ONLY) - # find out what BROPATH to use when executing bro execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev OUTPUT_VARIABLE BROPATH @@ -48,7 +35,7 @@ endif () # which summary text can be extracted at build time # ${group}_doc_names: a running list of reST style document names that can be # given to a :doc: role, shared indices with ${group}_files -# + macro(REST_TARGET srcDir broInput) set(absSrcPath ${srcDir}/${broInput}) get_filename_component(basename ${broInput} NAME) @@ -86,7 +73,7 @@ macro(REST_TARGET srcDir broInput) elseif (${extension} STREQUAL ".bif.bro") set(group bifs) elseif (relDstDir) - set(pkgIndex scripts/${relDstDir}/index) + set(pkgIndex ${relDstDir}/index) set(group ${pkgIndex}) # add package index to master package list if not already in it list(FIND MASTER_PKG_LIST ${pkgIndex} _found) @@ -134,6 +121,7 @@ macro(REST_TARGET srcDir broInput) ARGS -rf .state *.log *.rst DEPENDS bro DEPENDS ${absSrcPath} + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMENT "[Bro] Generating reST docs for ${broInput}" ) @@ -143,12 +131,10 @@ endmacro(REST_TARGET) include(DocSourcesList.cmake) # create temporary list of all docs to include in the master policy/index file -set(MASTER_POLICY_INDEX ${CMAKE_CURRENT_BINARY_DIR}/policy_index) file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}") # create the temporary list of all packages to include in the master # policy/packages.rst file -set(MASTER_PACKAGE_INDEX ${CMAKE_CURRENT_BINARY_DIR}/pkg_index) set(MASTER_PKG_INDEX_TEXT "") foreach (pkg ${MASTER_PKG_LIST}) # strip of the trailing /index for the link name @@ -186,11 +172,11 @@ if (EXISTS ${RST_OUTPUT_DIR}) list(FIND ALL_REST_OUTPUTS ${_doc} _found) if (_found EQUAL -1) file(REMOVE ${_doc}) - message(STATUS "AutoDoc: remove stale reST doc: ${_doc}") + message(STATUS "Broxygen: remove stale reST doc: ${_doc}") string(REPLACE .rst .bro _brofile ${_doc}) if (EXISTS ${_brofile}) file(REMOVE ${_brofile}) - message(STATUS "AutoDoc: remove stale bro source: ${_brofile}") + message(STATUS "Broxygen: remove stale bro source: ${_brofile}") endif () endif () endforeach () @@ -211,57 +197,3 @@ add_custom_target(restclean COMMAND "${CMAKE_COMMAND}" -E remove_directory ${RST_OUTPUT_DIR} VERBATIM) - -# The "sphinxdoc" target generates reST documentation for any outdated bro -# scripts and then uses Sphinx to generate HTML documentation from the reST -add_custom_target(sphinxdoc - # copy the template documentation to the build directory - # to give as input for sphinx - COMMAND "${CMAKE_COMMAND}" -E copy_directory - ${DOC_SOURCE_DIR} - ${DOC_SOURCE_WORKDIR} - # copy generated policy script documentation into the - # working copy of the template documentation - COMMAND "${CMAKE_COMMAND}" -E copy_directory - ${RST_OUTPUT_DIR} - ${DOC_SOURCE_WORKDIR}/scripts - # append to the master index of all policy scripts - COMMAND cat ${MASTER_POLICY_INDEX} >> - ${DOC_SOURCE_WORKDIR}/scripts/index.rst - # append to the master index of all policy packages - COMMAND cat ${MASTER_PACKAGE_INDEX} >> - ${DOC_SOURCE_WORKDIR}/packages.rst - # construct a reST file for each group - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/group_index_generator.py - ${CMAKE_CURRENT_BINARY_DIR}/group_list - ${CMAKE_CURRENT_BINARY_DIR} - ${DOC_SOURCE_WORKDIR} - # tell sphinx to generate html - COMMAND sphinx-build - -b html - -c ${CMAKE_CURRENT_BINARY_DIR} - -d ${DOC_OUTPUT_DIR}/doctrees - ${DOC_SOURCE_WORKDIR} - ${DOC_OUTPUT_DIR}/html - # create symlink to the html output directory for convenience - COMMAND "${CMAKE_COMMAND}" -E create_symlink - ${DOC_OUTPUT_DIR}/html - ${CMAKE_BINARY_DIR}/html - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "[Sphinx] Generating HTML policy script docs" - # SOURCES just adds stuff to IDE projects as a convenience - SOURCES ${DOC_SOURCES}) - -# The "sphinxclean" target removes just the Sphinx input/output directories -# from the build directory. -add_custom_target(sphinxclean - COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${DOC_SOURCE_WORKDIR} - COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${DOC_OUTPUT_DIR} - VERBATIM) - -add_dependencies(sphinxdoc sphinxclean restdoc) - -add_dependencies(doc sphinxdoc) -add_dependencies(docclean sphinxclean restclean) diff --git a/doc/scripts/README b/doc/scripts/README index ca7f28492b..b3e44914f4 100644 --- a/doc/scripts/README +++ b/doc/scripts/README @@ -15,29 +15,10 @@ by CMake: ``build/`` directory inside ``reST`` (a symlink to ``doc/scripts/rest_output``). -``doc`` - - This target depends on a Python interpreter (>=2.5) and - `Sphinx `_ being installed. Sphinx can be - installed like:: - - > sudo easy_install sphinx - - This target will first build ``restdoc`` target and then copy the - resulting reST files as an input directory to Sphinx. - - After completion, HTML documentation can be located in the CMake - ``build/`` directory inside ``html`` (a symlink to - ``doc/scripts/out/html``) - ``restclean`` This target removes any reST documentation that has been generated so far. -``docclean`` - - This target removes Sphinx inputs and outputs from the CMake ``build/`` dir. - The ``genDocSourcesList.sh`` script can be run to automatically generate ``DocSourcesList.cmake``, which is the file CMake uses to define the list of documentation targets. This script should be run after adding new @@ -54,18 +35,10 @@ script's name to the blacklist, then append a ``rest_target()`` to the ``statictext`` variable where the first argument is the source directory containing the policy script to document, the second argument is the file name of the policy script, and the third argument is the path/name of a -pre-created reST document in the ``source/`` directory to which the +pre-created reST document in the ``../`` source directory to which the ``make doc`` process can append script documentation references. This pre-created reST document should also then be linked to from the TOC tree -in ``source/index.rst``. - -The Sphinx source tree template in ``source/`` can be modified to add more -common/general documentation, style sheets, JavaScript, etc. The Sphinx -config file is produced from ``conf.py.in``, so that can be edited to change -various Sphinx options, like setting the default HTML rendering theme. -There is also a custom Sphinx domain implemented in ``source/ext/bro.py`` -which adds some reST directives and roles that aid in generating useful -index entries and cross-references. +in ``../index.rst``. See ``example.bro`` for an example of how to document a Bro script such that ``make doc`` will be able to produce reST/HTML documentation for it. diff --git a/doc/scripts/bifs.rst b/doc/scripts/bifs.rst new file mode 100644 index 0000000000..eaae0e13b8 --- /dev/null +++ b/doc/scripts/bifs.rst @@ -0,0 +1,5 @@ +.. This is a stub doc to which broxygen appends during the build process + +Built-In Functions (BIFs) +========================= + diff --git a/doc/scripts/source/builtins.rst b/doc/scripts/builtins.rst similarity index 100% rename from doc/scripts/source/builtins.rst rename to doc/scripts/builtins.rst diff --git a/doc/scripts/source/common.rst b/doc/scripts/common.rst similarity index 100% rename from doc/scripts/source/common.rst rename to doc/scripts/common.rst diff --git a/doc/scripts/index.rst b/doc/scripts/index.rst new file mode 100644 index 0000000000..5de203afe7 --- /dev/null +++ b/doc/scripts/index.rst @@ -0,0 +1,8 @@ +.. This is a stub doc to which broxygen appends during the build process + +Index of All Bro Scripts +======================== + +.. toctree:: + :maxdepth: 1 + diff --git a/doc/scripts/internal.rst b/doc/scripts/internal.rst new file mode 100644 index 0000000000..a6c10f1cfb --- /dev/null +++ b/doc/scripts/internal.rst @@ -0,0 +1,5 @@ +.. This is a stub doc to which broxygen appends during the build process + +Internal Scripts +================ + diff --git a/doc/scripts/source/packages.rst b/doc/scripts/packages.rst similarity index 78% rename from doc/scripts/source/packages.rst rename to doc/scripts/packages.rst index 35f6633fd4..47e974b0c8 100644 --- a/doc/scripts/source/packages.rst +++ b/doc/scripts/packages.rst @@ -1,7 +1,7 @@ -.. This is a stub doc to which the build process can append. +.. This is a stub doc to which broxygen appends during the build process -Bro Script Packages -=================== +Index of All Bro Script Packages +================================ Bro has the following script packages (e.g. collections of related scripts in a common directory). If the package directory contains a ``__load__.bro`` diff --git a/doc/scripts/source/_templates/layout.html b/doc/scripts/source/_templates/layout.html deleted file mode 100644 index 5685e1dc37..0000000000 --- a/doc/scripts/source/_templates/layout.html +++ /dev/null @@ -1,5 +0,0 @@ -{% extends "!layout.html" %} -{% block extrahead %} - - {{ super() }} -{% endblock %} diff --git a/doc/scripts/source/bifs.rst b/doc/scripts/source/bifs.rst deleted file mode 100644 index 6a42cafafc..0000000000 --- a/doc/scripts/source/bifs.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. This is a stub doc to which the build process can append. - -Built-In Functions (BIFs) -========================= - diff --git a/doc/scripts/source/index.rst b/doc/scripts/source/index.rst deleted file mode 100644 index a144644208..0000000000 --- a/doc/scripts/source/index.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. Bro documentation master file - -Welcome to Bro's documentation! -=============================== - -Contents: - -.. toctree:: - :maxdepth: 1 - :glob: - - common - builtins - internal - bifs - packages - scripts/index - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`search` diff --git a/doc/scripts/source/internal.rst b/doc/scripts/source/internal.rst deleted file mode 100644 index 817c76e725..0000000000 --- a/doc/scripts/source/internal.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. This is a stub doc to which the build process can append. - -Internal Scripts -================ - diff --git a/doc/scripts/source/scripts/index.rst b/doc/scripts/source/scripts/index.rst deleted file mode 100644 index fd11a3924e..0000000000 --- a/doc/scripts/source/scripts/index.rst +++ /dev/null @@ -1,6 +0,0 @@ -Index of All Bro Script Documentation -===================================== - -.. toctree:: - :maxdepth: 1 - diff --git a/doc/signatures.rst b/doc/signatures.rst index 25dfc31f5e..e2f9a8d702 100644 --- a/doc/signatures.rst +++ b/doc/signatures.rst @@ -3,7 +3,7 @@ Signatures ========== -.. class:: opening +.. rst-class:: opening Bro relies primarily on its extensive scripting language for defining and analyzing detection policies. In addition, however, @@ -47,8 +47,8 @@ piece of payload which triggered the pattern match. To turn such ``signature_match`` events into actual alarms, you can load Bro's ``signature.bro`` script. This script contains a default -event handler that raises ``SensitiveSignature`` `Notices -`_ (as well as others; see the beginning of the script). +event handler that raises ``SensitiveSignature`` :doc:`Notices ` +(as well as others; see the beginning of the script). As signatures are independent of Bro's policy scripts, they are put into their own file(s). There are two ways to specify which files diff --git a/doc/upgrade.rst b/doc/upgrade.rst index 696c64ef7f..71cc5e401d 100644 --- a/doc/upgrade.rst +++ b/doc/upgrade.rst @@ -3,7 +3,7 @@ Upgrading From Bro 1.5 to 2.0 ============================= -.. class:: opening +.. rst-class:: opening This guide details differences between Bro versions 1.5 and 2.0 that may be important for users to know as they work on updating @@ -39,7 +39,7 @@ version. The two rules of thumb are: if you need help. Below we summarize changes from 1.x to 2.x in more detail. This list -isn't complete, see the `CHANGES <{{git('bro:CHANGES', 'txt')}}>`_ file in the +isn't complete, see the :download:`CHANGES ` file in the distribution for the full story. Default Scripts @@ -131,8 +131,8 @@ Logging Framework endpoint. - The new logging framework makes it possible to extend, customize, - and filter logs very easily. See `the logging framework - <{{git('bro:doc/logging.rst')}}>`_ more information on usage. + and filter logs very easily. See the :doc:`logging framework ` + more information on usage. - A common pattern found in the new scripts is to store logging stream records for protocols inside the ``connection`` records so that @@ -155,8 +155,7 @@ Notice Framework The way users interact with "notices" has changed significantly in order to make it easier to define a site policy and more extensible -for adding customized actions. See the `the notice framework -<{{git('bro:doc/notice.rst')}}>`_. +for adding customized actions. See the :doc:`notice framework `. New Default Settings @@ -198,7 +197,7 @@ Variable Naming - Identifiers may have been renamed to conform to new `scripting conventions - <{{docroot}}/development/script-conventions.html>`_ + `_ BroControl @@ -240,7 +239,7 @@ Development Infrastructure Bro development has moved from using SVN to Git for revision control. Users that like to use the latest Bro developments by checking it out from the source repositories should see the `development process -<{{docroot}}/development/process.html>`_. Note that all the various +`_. Note that all the various sub-components now reside on their own repositories. However, the top-level Bro repository includes them as git submodules so it's easu to check them all out simultaneously. diff --git a/src/Base64.cc b/src/Base64.cc index 9008837f35..b0da8ea74c 100644 --- a/src/Base64.cc +++ b/src/Base64.cc @@ -1,14 +1,27 @@ #include "config.h" #include "Base64.h" -static int base64_table[256]; +int Base64Decoder::default_base64_table[256]; +const string Base64Decoder::default_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -static void init_base64_table() +int* Base64Decoder::InitBase64Table(const string& alphabet) { - static int table_initialized = 0; + assert(alphabet.size() == 64); - if ( ++table_initialized > 1 ) - return; + static bool default_table_initialized = false; + + if ( alphabet == default_alphabet && default_table_initialized ) + return default_base64_table; + + int* base64_table = 0; + + if ( alphabet == default_alphabet ) + { + base64_table = default_base64_table; + default_table_initialized = true; + } + else + base64_table = new int[256]; int i; for ( i = 0; i < 256; ++i ) @@ -16,28 +29,36 @@ static void init_base64_table() for ( i = 0; i < 26; ++i ) { - base64_table['A' + i] = i; - base64_table['a' + i] = i + 26; + base64_table[int(alphabet[0 + i])] = i; + base64_table[int(alphabet[26 + i])] = i + 26; } for ( i = 0; i < 10; ++i ) - base64_table['0' + i] = i + 52; + base64_table[int(alphabet[52 + i])] = i + 52; // Casts to avoid compiler warnings. - base64_table[int('+')] = 62; - base64_table[int('/')] = 63; + base64_table[int(alphabet[62])] = 62; + base64_table[int(alphabet[63])] = 63; base64_table[int('=')] = 0; + + return base64_table; } -Base64Decoder::Base64Decoder(Analyzer* arg_analyzer) +Base64Decoder::Base64Decoder(Analyzer* arg_analyzer, const string& alphabet) { - init_base64_table(); + base64_table = InitBase64Table(alphabet.size() ? alphabet : default_alphabet); base64_group_next = 0; base64_padding = base64_after_padding = 0; errored = 0; analyzer = arg_analyzer; } +Base64Decoder::~Base64Decoder() + { + if ( base64_table != default_base64_table ) + delete base64_table; + } + int Base64Decoder::Decode(int len, const char* data, int* pblen, char** pbuf) { int blen; @@ -142,13 +163,21 @@ int Base64Decoder::Done(int* pblen, char** pbuf) return 0; } -BroString* decode_base64(const BroString* s) + +BroString* decode_base64(const BroString* s, const BroString* a) { + if ( a && a->Len() != 64 ) + { + reporter->Error("base64 decoding alphabet is not 64 characters: %s", + a->CheckString()); + return 0; + } + int buf_len = int((s->Len() + 3) / 4) * 3 + 1; int rlen2, rlen = buf_len; char* rbuf2, *rbuf = new char[rlen]; - Base64Decoder dec(0); + Base64Decoder dec(0, a ? a->CheckString() : ""); if ( dec.Decode(s->Len(), (const char*) s->Bytes(), &rlen, &rbuf) == -1 ) goto err; diff --git a/src/Base64.h b/src/Base64.h index 7f351a2c2b..0e02c94cf0 100644 --- a/src/Base64.h +++ b/src/Base64.h @@ -13,11 +13,11 @@ class Base64Decoder { public: - // is used for error reporting, and it should be zero - // when the decoder is called by the built-in function - // decode_base64(). - Base64Decoder(Analyzer* analyzer); - ~Base64Decoder() { } + // is used for error reporting, and it should be zero when + // the decoder is called by the built-in function decode_base64(). + // Empty alphabet indicates the default base64 alphabet. + Base64Decoder(Analyzer* analyzer, const string& alphabet = ""); + ~Base64Decoder(); // A note on Decode(): // @@ -57,8 +57,13 @@ protected: int base64_after_padding; int errored; // if true, we encountered an error - skip further processing Analyzer* analyzer; + int* base64_table; + + static int* InitBase64Table(const string& alphabet); + static int default_base64_table[256]; + static const string default_alphabet; }; -BroString* decode_base64(const BroString* s); +BroString* decode_base64(const BroString* s, const BroString* a = 0); #endif /* base64_h */ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1693bad4eb..f215fe5d9c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -423,15 +423,7 @@ add_definitions(-DBRO_BUILD_PATH="${CMAKE_CURRENT_BINARY_DIR}") add_executable(bro ${bro_SRCS} ${bro_HEADERS}) -set(brolibs - ${BinPAC_LIBRARY} - ${PCAP_LIBRARY} - ${OpenSSL_LIBRARIES} - ${BIND_LIBRARY} - ${OPTLIBS} -) - -target_link_libraries(bro ${brolibs}) +target_link_libraries(bro ${brodeps}) install(TARGETS bro DESTINATION bin) install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${BRO_SCRIPT_INSTALL_PATH}/base) diff --git a/src/ChunkedIO.cc b/src/ChunkedIO.cc index ff84a343c7..f5bcb4b7c1 100644 --- a/src/ChunkedIO.cc +++ b/src/ChunkedIO.cc @@ -1170,8 +1170,6 @@ void ChunkedIOSSL::Stats(char* buffer, int length) ChunkedIO::Stats(buffer + i, length - i); } -#ifdef HAVE_LIBZ - bool CompressedChunkedIO::Init() { zin.zalloc = 0; @@ -1348,5 +1346,3 @@ void CompressedChunkedIO::Stats(char* buffer, int length) io->Stats(buffer + i, length - i); buffer[length-1] = '\0'; } - -#endif /* HAVE_LIBZ */ diff --git a/src/ChunkedIO.h b/src/ChunkedIO.h index ca95f4b40b..56b5656945 100644 --- a/src/ChunkedIO.h +++ b/src/ChunkedIO.h @@ -287,8 +287,6 @@ private: static SSL_CTX* ctx; }; -#ifdef HAVE_LIBZ - #include // Wrapper class around a another ChunkedIO which the (un-)compresses data. @@ -335,6 +333,4 @@ protected: unsigned long uncompressed_bytes_written; }; -#endif /* HAVE_LIBZ */ - #endif diff --git a/src/FileAnalyzer.cc b/src/FileAnalyzer.cc index 672d1e1e09..d4064e8144 100644 --- a/src/FileAnalyzer.cc +++ b/src/FileAnalyzer.cc @@ -3,23 +3,19 @@ #include "FileAnalyzer.h" #include "Reporter.h" -#ifdef HAVE_LIBMAGIC magic_t File_Analyzer::magic = 0; magic_t File_Analyzer::magic_mime = 0; -#endif File_Analyzer::File_Analyzer(Connection* conn) : TCP_ApplicationAnalyzer(AnalyzerTag::File, conn) { buffer_len = 0; -#ifdef HAVE_LIBMAGIC if ( ! magic ) { InitMagic(&magic, MAGIC_NONE); InitMagic(&magic_mime, MAGIC_MIME); } -#endif } void File_Analyzer::DeliverStream(int len, const u_char* data, bool orig) @@ -52,13 +48,11 @@ void File_Analyzer::Identify() const char* descr = 0; const char* mime = 0; -#ifdef HAVE_LIBMAGIC if ( magic ) descr = magic_buffer(magic, buffer, buffer_len); if ( magic_mime ) mime = magic_buffer(magic_mime, buffer, buffer_len); -#endif val_list* vl = new val_list; vl->append(BuildConnVal()); @@ -68,7 +62,6 @@ void File_Analyzer::Identify() ConnectionEvent(file_transferred, vl); } -#ifdef HAVE_LIBMAGIC void File_Analyzer::InitMagic(magic_t* magic, int flags) { *magic = magic_open(flags); @@ -83,4 +76,3 @@ void File_Analyzer::InitMagic(magic_t* magic, int flags) *magic = 0; } } -#endif diff --git a/src/FileAnalyzer.h b/src/FileAnalyzer.h index 8c1890bb85..dcf9d22e8e 100644 --- a/src/FileAnalyzer.h +++ b/src/FileAnalyzer.h @@ -5,9 +5,7 @@ #include "TCP.h" -#ifdef HAVE_LIBMAGIC #include -#endif class File_Analyzer : public TCP_ApplicationAnalyzer { public: @@ -31,12 +29,10 @@ protected: char buffer[BUFFER_SIZE]; int buffer_len; -#ifdef HAVE_LIBMAGIC static void InitMagic(magic_t* magic, int flags); static magic_t magic; static magic_t magic_mime; -#endif }; #endif diff --git a/src/HTTP.cc b/src/HTTP.cc index 71fa1a3dd0..b41933156d 100644 --- a/src/HTTP.cc +++ b/src/HTTP.cc @@ -43,9 +43,7 @@ HTTP_Entity::HTTP_Entity(HTTP_Message *arg_message, MIME_Entity* parent_entity, header_length = 0; deliver_body = (http_entity_data != 0); encoding = IDENTITY; -#ifdef HAVE_LIBZ zip = 0; -#endif } void HTTP_Entity::EndOfData() @@ -53,7 +51,6 @@ void HTTP_Entity::EndOfData() if ( DEBUG_http ) DEBUG_MSG("%.6f: end of data\n", network_time); -#ifdef HAVE_LIBZ if ( zip ) { zip->Done(); @@ -61,7 +58,6 @@ void HTTP_Entity::EndOfData() zip = 0; encoding = IDENTITY; } -#endif if ( body_length ) http_message->MyHTTP_Analyzer()-> @@ -179,7 +175,6 @@ private: void HTTP_Entity::DeliverBody(int len, const char* data, int trailing_CRLF) { -#ifdef HAVE_LIBZ if ( encoding == GZIP || encoding == DEFLATE ) { ZIP_Analyzer::Method method = @@ -198,7 +193,6 @@ void HTTP_Entity::DeliverBody(int len, const char* data, int trailing_CRLF) zip->NextStream(len, (const u_char*) data, false); } else -#endif DeliverBodyClear(len, data, trailing_CRLF); } @@ -450,9 +444,7 @@ void HTTP_Entity::SubmitAllHeaders() // content-length headers or if connection is to be closed afterwards // anyway. else if ( http_message->MyHTTP_Analyzer()->IsConnectionClose () -#ifdef HAVE_LIBZ || encoding == GZIP || encoding == DEFLATE -#endif ) { // FIXME: Using INT_MAX is kind of a hack here. Better diff --git a/src/HTTP.h b/src/HTTP.h index 13b87d219f..00524da20d 100644 --- a/src/HTTP.h +++ b/src/HTTP.h @@ -29,10 +29,8 @@ public: int expect_body); ~HTTP_Entity() { -#ifdef HAVE_LIBZ if ( zip ) { zip->Done(); delete zip; } -#endif } void EndOfData(); @@ -55,9 +53,7 @@ protected: int64_t header_length; int deliver_body; enum { IDENTITY, GZIP, COMPRESS, DEFLATE } encoding; -#ifdef HAVE_LIBZ ZIP_Analyzer* zip; -#endif MIME_Entity* NewChildEntity() { return new HTTP_Entity(http_message, this, 1); } diff --git a/src/IRC.cc b/src/IRC.cc index caf7c492b6..1918300ba2 100644 --- a/src/IRC.cc +++ b/src/IRC.cc @@ -1188,15 +1188,10 @@ void IRC_Analyzer::DeliverStream(int length, const u_char* line, bool orig) if ( orig_status == REGISTERED && resp_status == REGISTERED && orig_zip_status == ACCEPT_ZIP && resp_zip_status == ACCEPT_ZIP ) { -#ifdef HAVE_LIBZ orig_zip_status = ZIP_LOADED; resp_zip_status = ZIP_LOADED; AddSupportAnalyzer(new ZIP_Analyzer(Conn(), true)); AddSupportAnalyzer(new ZIP_Analyzer(Conn(), false)); -#else - reporter->Error("IRC analyzer lacking libz support"); - Remove(); -#endif } return; diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 6eaace3893..80f6c26ac0 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -151,7 +151,7 @@ bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) if ( ! t->IsSet() ) return false; - return IsCompatibleType(t->AsSetType()->Indices()->PureType()); + return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true); } case TYPE_VECTOR: diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 324da2c92b..a21a7abc60 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -1222,10 +1222,7 @@ bool RemoteSerializer::SendCapabilities(Peer* peer) uint32 caps = 0; -#ifdef HAVE_LIBZ caps |= Peer::COMPRESSION; -#endif - caps |= Peer::PID_64BIT; caps |= Peer::NEW_CACHE_STRATEGY; @@ -2106,11 +2103,9 @@ bool RemoteSerializer::ProcessPhaseDone() bool RemoteSerializer::HandshakeDone(Peer* peer) { -#ifdef HAVE_LIBZ if ( peer->caps & Peer::COMPRESSION && peer->comp_level > 0 ) if ( ! SendToChild(MSG_COMPRESS, peer, 1, peer->comp_level) ) return false; -#endif if ( ! (peer->caps & Peer::PID_64BIT) ) Log(LogInfo, "peer does not support 64bit PIDs; using compatibility mode", peer); @@ -3699,11 +3694,6 @@ bool SocketComm::ProcessListen() bool SocketComm::ProcessParentCompress() { -#ifndef HAVE_LIBZ - InternalError("supposed to enable compression but don't have zlib"); - return false; -#else - assert(parent_args); uint32* args = (uint32*) parent_args->data; @@ -3727,7 +3717,6 @@ bool SocketComm::ProcessParentCompress() Log(fmt("enabling compression (level %d)", level), parent_peer); return true; -#endif } bool SocketComm::ProcessRemoteMessage(SocketComm::Peer* peer) @@ -3847,10 +3836,6 @@ bool SocketComm::ProcessPeerCompress(Peer* peer) { peer->state = MSG_NONE; -#ifndef HAVE_LIBZ - Error("peer compresses although we do not support it", peer); - return false; -#else if ( ! parent_peer->compressor ) { parent_peer->io = new CompressedChunkedIO(parent_peer->io); @@ -3862,7 +3847,6 @@ bool SocketComm::ProcessPeerCompress(Peer* peer) ((CompressedChunkedIO*) peer->io)->EnableDecompression(); Log("enabling decompression", peer); return true; -#endif } bool SocketComm::Connect(Peer* peer) diff --git a/src/ZIP.cc b/src/ZIP.cc index 26095d1f11..0ebe93abe6 100644 --- a/src/ZIP.cc +++ b/src/ZIP.cc @@ -2,8 +2,6 @@ #include "ZIP.h" -#ifdef HAVE_LIBZ - ZIP_Analyzer::ZIP_Analyzer(Connection* conn, bool orig, Method arg_method) : TCP_SupportAnalyzer(AnalyzerTag::Zip, conn, orig) { @@ -89,4 +87,3 @@ void ZIP_Analyzer::DeliverStream(int len, const u_char* data, bool orig) } while ( zip->avail_out == 0 ); } -#endif diff --git a/src/ZIP.h b/src/ZIP.h index ab5d2ce68b..6a8a180d1a 100644 --- a/src/ZIP.h +++ b/src/ZIP.h @@ -5,8 +5,6 @@ #include "config.h" -#ifdef HAVE_LIBZ - #include "zlib.h" #include "TCP.h" @@ -29,4 +27,3 @@ protected: }; #endif -#endif diff --git a/src/bro.bif b/src/bro.bif index 3350de2867..a2f97356a7 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1860,6 +1860,18 @@ function decode_base64%(s: string%): string } %} +function decode_base64_custom%(s: string, a: string%): string + %{ + BroString* t = decode_base64(s->AsString(), a->AsString()); + if ( t ) + return new StringVal(t); + else + { + reporter->Error("error in decoding string %s", s->CheckString()); + return new StringVal(""); + } + %} + %%{ #include "DCE_RPC.h" @@ -3272,18 +3284,15 @@ function enable_raw_output%(f: file%): any %} %%{ -#ifdef HAVE_LIBMAGIC extern "C" { #include } -#endif %%} function identify_data%(data: string, return_mime: bool%): string %{ const char* descr = ""; -#ifdef HAVE_LIBMAGIC static magic_t magic_mime = 0; static magic_t magic_descr = 0; @@ -3309,7 +3318,6 @@ function identify_data%(data: string, return_mime: bool%): string } descr = magic_buffer(*magic, data->Bytes(), data->Len()); -#endif return new StringVal(descr); %} diff --git a/testing/btest/Baseline/bifs.decode_base64/out b/testing/btest/Baseline/bifs.decode_base64/out new file mode 100644 index 0000000000..af0d32fbb8 --- /dev/null +++ b/testing/btest/Baseline/bifs.decode_base64/out @@ -0,0 +1,6 @@ +bro +bro +bro +bro +bro +bro diff --git a/testing/btest/bifs/decode_base64.bro b/testing/btest/bifs/decode_base64.bro new file mode 100644 index 0000000000..d4cbd2f37d --- /dev/null +++ b/testing/btest/bifs/decode_base64.bro @@ -0,0 +1,14 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +global default_alphabet: string = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +global my_alphabet: string = "!#$%&/(),-.:;<>@[]^ `_{|}~abcdefghijklmnopqrstuvwxyz0123456789+?"; + +print decode_base64("YnJv"); +print decode_base64_custom("YnJv", default_alphabet); +print decode_base64_custom("}n-v", my_alphabet); + +print decode_base64("YnJv"); +print decode_base64_custom("YnJv", default_alphabet); +print decode_base64_custom("}n-v", my_alphabet); diff --git a/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro b/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro index 53b340d174..dd01e62413 100644 --- a/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro +++ b/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro @@ -2,7 +2,6 @@ # will normalize mime types other than the target type to prevent sensitivity # to varying versions of libmagic. -# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h # @TEST-EXEC: bro -r $TRACES/http-pipelined-requests.trace %INPUT > output # @TEST-EXEC: btest-diff http.log diff --git a/testing/btest/scripts/base/protocols/irc/dcc-extract.test b/testing/btest/scripts/base/protocols/irc/dcc-extract.test index 0324a3f28f..b6bf43ac50 100644 --- a/testing/btest/scripts/base/protocols/irc/dcc-extract.test +++ b/testing/btest/scripts/base/protocols/irc/dcc-extract.test @@ -2,7 +2,6 @@ # correctly extracted. The mime type of the file transferred is normalized # to prevent sensitivity to libmagic version being used. -# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h # @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT # @TEST-EXEC: btest-diff irc.log # @TEST-EXEC: btest-diff irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat diff --git a/testing/btest/scripts/base/protocols/smtp/mime-extract.test b/testing/btest/scripts/base/protocols/smtp/mime-extract.test index ba0869f6d9..8dc8bc9e31 100644 --- a/testing/btest/scripts/base/protocols/smtp/mime-extract.test +++ b/testing/btest/scripts/base/protocols/smtp/mime-extract.test @@ -1,4 +1,3 @@ -# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h # @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT # @TEST-EXEC: btest-diff smtp_entities.log # @TEST-EXEC: btest-diff smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat diff --git a/testing/btest/scripts/base/protocols/smtp/mime.test b/testing/btest/scripts/base/protocols/smtp/mime.test index 6d50d5919f..2b80e148ff 100644 --- a/testing/btest/scripts/base/protocols/smtp/mime.test +++ b/testing/btest/scripts/base/protocols/smtp/mime.test @@ -1,7 +1,6 @@ # Checks logging of mime types and md5 calculation. Mime type in the log # is normalized to prevent sensitivity to libmagic version. -# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h # @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT # @TEST-EXEC: btest-diff smtp_entities.log