From 881071cc99207af49e4512ca59c6ed1fc57e5e95 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 15 Mar 2011 17:50:30 -0700 Subject: [PATCH 01/56] Extending conn_id with a globally unique identifiers. --- policy/bro.init | 1 + src/Conn.cc | 43 +++++++++++++++++++++++++++++++++++++++++++ src/Conn.h | 7 +++++++ src/Hash.h | 2 +- src/util.cc | 20 ++++++++++++++++++++ src/util.h | 4 +--- 6 files changed, 73 insertions(+), 4 deletions(-) diff --git a/policy/bro.init b/policy/bro.init index b3f78f0689..03bdbc56ce 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -21,6 +21,7 @@ type conn_id: record { orig_p: port; resp_h: addr; resp_p: port; + uid: string; }; type icmp_conn: record { diff --git a/src/Conn.cc b/src/Conn.cc index 50afe41d0c..6cb24c3446 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -179,6 +179,8 @@ Connection::Connection(NetSessions* s, HashKey* k, double t, const ConnID* id) ++current_connections; ++total_connections; + uid = CalculateUID(); + TimerMgr::Tag* tag = current_iosrc->GetCurrentTag(); conn_timer_mgr = tag ? new TimerMgr::Tag(*tag) : 0; @@ -215,6 +217,43 @@ Connection::~Connection() --external_connections; } +uint64 Connection::uid_counter = 0; +uint64 Connection::uid_instance = 0; + +uint64 Connection::CalculateUID() + { + if ( uid_instance == 0 ) + { + // This is the first time we need a UID. Calculate the instance ID by + // hashing something likely to be unique. + struct { + char hostname[128]; + struct timeval time; + pid_t pid; + } unique; + + gethostname(unique.hostname, 128); + unique.hostname[sizeof(unique.hostname)-1] = '\0'; + gettimeofday(&unique.time, 0); + unique.pid = getpid(); + + uid_instance = HashKey::HashBytes(&unique, sizeof(unique)); + ++uid_instance; // Now it's larger than zero. + } + + // Now calculate the unique ID for this connection. + struct { + uint64 counter; + hash_t instance; + } key; + + key.counter = ++uid_counter; + key.instance = uid_instance; + + uint64_t h = HashKey::HashBytes(&key, sizeof(key)); + return h; + } + void Connection::Done() { finished = 1; @@ -346,6 +385,10 @@ RecordVal* Connection::BuildConnVal() id_val->Assign(1, new PortVal(ntohs(orig_port), prot_type)); id_val->Assign(2, new AddrVal(resp_addr)); id_val->Assign(3, new PortVal(ntohs(resp_port), prot_type)); + + char tmp[16]; + id_val->Assign(4, new StringVal(uitoa_n(uid, tmp, sizeof(tmp), 62))); + conn_val->Assign(0, id_val); orig_endp = new RecordVal(endpoint); diff --git a/src/Conn.h b/src/Conn.h index bbe3c2b3f6..1da1f97b57 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -279,6 +279,8 @@ public: void AddHistory(char code) { history += code; } + uint64 CalculateUID(); + void DeleteTimer(double t); // Sets the root of the analyzer tree as well as the primary PIA. @@ -368,6 +370,11 @@ protected: string history; uint32 hist_seen; + uint64 uid; // Globally unique connection ID. + + static uint64 uid_counter; // Counter for uids. + static uint64 uid_instance; // Once computed instance ID. + TransportLayerAnalyzer* root_analyzer; PIA* primary_PIA; }; diff --git a/src/Hash.h b/src/Hash.h index fa1f00f91f..9d6809edc7 100644 --- a/src/Hash.h +++ b/src/Hash.h @@ -11,7 +11,7 @@ #define UHASH_KEY_SIZE 32 -typedef unsigned int hash_t; +typedef uint64 hash_t; typedef enum { HASH_KEY_INT, diff --git a/src/util.cc b/src/util.cc index d8390a866c..ee5552899c 100644 --- a/src/util.cc +++ b/src/util.cc @@ -340,6 +340,26 @@ int atoi_n(int len, const char* s, const char** end, int base, int& result) return 1; } +char* uitoa_n(uint64 value, char* str, int n, int base) + { + static char dig[] = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + int i = 0; + uint64 v; + char* p, *q; + char c; + + v = value; + + do { + str[i++] = dig[v % base]; + v /= base; + } while ( v && i < n ); + + str[i] = '\0'; + + return str; + } + int strstr_n(const int big_len, const u_char* big, const int little_len, const u_char* little) { diff --git a/src/util.h b/src/util.h index ab7cb3c6fe..df01e92af6 100644 --- a/src/util.h +++ b/src/util.h @@ -112,6 +112,7 @@ extern char* strcasestr(const char* s, const char* find); extern const char* strpbrk_n(size_t len, const char* s, const char* charset); extern int atoi_n(int len, const char* s, const char** end, int base, int& result); +extern char* uitoa_n(uint64 value, char* str, int n, int base); int strstr_n(const int big_len, const unsigned char* big, const int little_len, const unsigned char* little); extern int fputs(int len, const char* s, FILE* fp); @@ -151,9 +152,6 @@ extern void init_random_seed(uint32 seed, const char* load_file, extern uint64 rand64bit(); -#define UHASH_KEY_SIZE 32 -extern uint8 uhash_key[UHASH_KEY_SIZE]; - // Each event source that may generate events gets an internally unique ID. // This is always LOCAL for a local Bro. For remote event sources, it gets // assigned by the RemoteSerializer. From d84d4b8a5707483af6b15f91029f531bfb58d9fc Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 16 Mar 2011 14:25:49 -0700 Subject: [PATCH 02/56] Moving uid from conn_id to connection, and making output determistic if a hash seed is given. --- policy/bro.init | 2 +- src/Conn.cc | 45 ++++++++++++++++++++++++++++----------------- src/Net.cc | 1 + src/Net.h | 4 ++++ src/main.cc | 2 ++ src/util.cc | 4 ++-- 6 files changed, 38 insertions(+), 20 deletions(-) diff --git a/policy/bro.init b/policy/bro.init index 03bdbc56ce..371e6ec1f8 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -21,7 +21,6 @@ type conn_id: record { orig_p: port; resp_h: addr; resp_p: port; - uid: string; }; type icmp_conn: record { @@ -93,6 +92,7 @@ type connection: record { addl: string; hot: count; # how hot; 0 = don't know or not hot history: string; + uid: string; }; type SYN_packet: record { diff --git a/src/Conn.cc b/src/Conn.cc index 6cb24c3446..09a65f15be 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -222,23 +222,34 @@ uint64 Connection::uid_instance = 0; uint64 Connection::CalculateUID() { - if ( uid_instance == 0 ) + if ( uid_instance == 0 ) { - // This is the first time we need a UID. Calculate the instance ID by - // hashing something likely to be unique. - struct { - char hostname[128]; - struct timeval time; - pid_t pid; - } unique; + // This is the first time we need a UID. + if ( ! bro_deterministic_output ) + { + // In live mode, with determistic output not explicitly + // requested, calculate the instance ID by hashing something + // likely to be unique. + struct { + char hostname[128]; + struct timeval time; + pid_t pid; + int rnd; + } unique; - gethostname(unique.hostname, 128); - unique.hostname[sizeof(unique.hostname)-1] = '\0'; - gettimeofday(&unique.time, 0); - unique.pid = getpid(); + gethostname(unique.hostname, 128); + unique.hostname[sizeof(unique.hostname)-1] = '\0'; + gettimeofday(&unique.time, 0); + unique.pid = getpid(); + unique.rnd = random(); - uid_instance = HashKey::HashBytes(&unique, sizeof(unique)); - ++uid_instance; // Now it's larger than zero. + uid_instance = HashKey::HashBytes(&unique, sizeof(unique)); + ++uid_instance; // Now it's larger than zero. + } + + else + // Generate determistic UIDs. + uid_instance = 1; } // Now calculate the unique ID for this connection. @@ -386,9 +397,6 @@ RecordVal* Connection::BuildConnVal() id_val->Assign(2, new AddrVal(resp_addr)); id_val->Assign(3, new PortVal(ntohs(resp_port), prot_type)); - char tmp[16]; - id_val->Assign(4, new StringVal(uitoa_n(uid, tmp, sizeof(tmp), 62))); - conn_val->Assign(0, id_val); orig_endp = new RecordVal(endpoint); @@ -406,6 +414,9 @@ RecordVal* Connection::BuildConnVal() conn_val->Assign(6, new StringVal("")); // addl conn_val->Assign(7, new Val(0, TYPE_COUNT)); // hot conn_val->Assign(8, new StringVal("")); // history + + char tmp[16]; + conn_val->Assign(9, new StringVal(uitoa_n(uid, tmp, sizeof(tmp), 62))); } if ( root_analyzer ) diff --git a/src/Net.cc b/src/Net.cc index bc56556ee5..06bb581c0f 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -68,6 +68,7 @@ double processing_start_time = 0.0; // time started working on current pkt double bro_start_time = 0.0; // time Bro started. double bro_start_network_time; // timestamp of first packet double last_watchdog_proc_time = 0.0; // value of above during last watchdog +bool bro_deterministic_output = 0; // whether determistic output is desired bool terminating = false; // whether we're done reading and finishing up PacketSortGlobalPQ* packet_sorter = 0; diff --git a/src/Net.h b/src/Net.h index 87c0ce2499..88e2ce0dcf 100644 --- a/src/Net.h +++ b/src/Net.h @@ -73,6 +73,10 @@ extern double bro_start_time; // i.e. the timestamp of the first packet. extern double bro_start_network_time; +// True if determistic output is requested. This is set if the user specifies +// a seed for the random number generator. +extern bool bro_deterministic_output; + // True if we're a in the process of cleaning-up just before termination. extern bool terminating; diff --git a/src/main.cc b/src/main.cc index 22f8ee6976..11dfc34a07 100644 --- a/src/main.cc +++ b/src/main.cc @@ -667,6 +667,8 @@ int main(int argc, char** argv) bro_start_time = current_time(true); + bro_deterministic_output = (seed || seed_load_file); + init_random_seed(seed, seed_load_file, seed_save_file); // DEBUG_MSG("HMAC key: %s\n", md5_digest_print(shared_hmac_md5_key)); init_hash_function(); diff --git a/src/util.cc b/src/util.cc index ee5552899c..8df9188466 100644 --- a/src/util.cc +++ b/src/util.cc @@ -352,7 +352,7 @@ char* uitoa_n(uint64 value, char* str, int n, int base) do { str[i++] = dig[v % base]; - v /= base; + v /= base; } while ( v && i < n ); str[i] = '\0'; @@ -798,7 +798,7 @@ const char* bro_path() if ( ! path ) path = ".:" POLICYDEST ":" - POLICYDEST "/sigs:" + POLICYDEST "/sigs:" POLICYDEST "/time-machine:" POLICYDEST "/site"; From c472931eb9a201141c4aa28ba14628d28233a077 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 20 Apr 2011 20:09:33 -0500 Subject: [PATCH 03/56] Fixing example.bro's auto-reST generation baseline test. Adds a diff canonifier that skips diffing the places where example.bro may use MutableVal derivatives (e.g. sets/tables), which don't always generate the same ordering in the reST docs across runs. --- .../Baseline/doc.autogen-reST-example/example.rst | 2 +- .../btest/Scripts/doc/example-diff-canonifier.py | 15 +++++++++++++++ testing/btest/btest.cfg | 1 + testing/btest/doc/autogen-reST-example | 2 +- 4 files changed, 18 insertions(+), 2 deletions(-) create mode 100755 testing/btest/Scripts/doc/example-diff-canonifier.py diff --git a/testing/btest/Baseline/doc.autogen-reST-example/example.rst b/testing/btest/Baseline/doc.autogen-reST-example/example.rst index eb125eda23..8e735b787e 100644 --- a/testing/btest/Baseline/doc.autogen-reST-example/example.rst +++ b/testing/btest/Baseline/doc.autogen-reST-example/example.rst @@ -13,7 +13,7 @@ these comments are transferred directly into the auto-generated `reStructuredText `_ (reST) document's summary section. -.. tip:: You can embed directives and roles within ``##``-stylized comments +.. tip:: You can embed directives and roles within ``##``-stylized comments. :Author: Jon Siwek diff --git a/testing/btest/Scripts/doc/example-diff-canonifier.py b/testing/btest/Scripts/doc/example-diff-canonifier.py new file mode 100755 index 0000000000..e0b8c110cc --- /dev/null +++ b/testing/btest/Scripts/doc/example-diff-canonifier.py @@ -0,0 +1,15 @@ +#!/usr/bin/python + +import sys +import re + +# MutableVal derivatives (e.g. sets/tables) don't always generate the same +# ordering in the reST documentation, so just don't bother diffing +# the places where example.bro uses them. + +RE1 = "\d*/tcp" +RE2 = "tcp port \d*" + +for line in sys.stdin.readlines(): + if re.search(RE1, line) is None and re.search(RE2, line) is None: + print line diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 52f7e6280a..0eee6883ef 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -12,5 +12,6 @@ BRO_SEED_FILE=%(testbase)s/random.seed PATH=%(testbase)s/../../build/src:%(testbase)s/../../aux/btest:%(default_path)s TEST_DIFF_CANONIFIER=%(testbase)s/Scripts/diff-canonifier TRACES=%(testbase)s/Traces +SCRIPTS=%(testbase)s/Scripts DIST=%(testbase)s/../.. BUILD=%(testbase)s/../../build diff --git a/testing/btest/doc/autogen-reST-example b/testing/btest/doc/autogen-reST-example index 7870259cad..c47a604373 100644 --- a/testing/btest/doc/autogen-reST-example +++ b/testing/btest/doc/autogen-reST-example @@ -1,2 +1,2 @@ @TEST-EXEC: bro --doc-scripts $DIST/doc/example.bro -@TEST-EXEC: btest-diff example.rst +@TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/doc/example-diff-canonifier.py btest-diff example.rst From 4634d92394ee1e887da151d8dcb35f7c28f56eb8 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 20 Apr 2011 21:11:32 -0500 Subject: [PATCH 04/56] Move stuff related to policy script documentation from doc/ to doc/scripts/ --- doc/CMakeLists.txt | 43 +----------------- doc/README | 45 +------------------ doc/scripts/CMakeLists.txt | 42 +++++++++++++++++ doc/scripts/README | 44 ++++++++++++++++++ doc/{ => scripts}/conf.py.in | 0 doc/{ => scripts}/example.bro | 0 doc/scripts/{ => scripts}/BroToReST.py.in | 0 .../{ => scripts}/generate_reST_docs.py.in | 2 +- doc/{ => scripts}/source/_static/showhide.js | 0 .../source/_templates/layout.html | 0 doc/{ => scripts}/source/builtins.rst | 0 doc/{ => scripts}/source/common.rst | 0 doc/{ => scripts}/source/ext/bro.py | 0 doc/{ => scripts}/source/index.rst | 0 doc/{ => scripts}/source/policy/bifs.rst | 0 doc/{ => scripts}/source/policy/default.rst | 0 doc/{ => scripts}/source/policy/index.rst | 0 doc/{ => scripts}/source/policy/internal.rst | 0 doc/{ => scripts}/source/policy/user.rst | 0 testing/btest/doc/autogen-reST-example | 2 +- 20 files changed, 90 insertions(+), 88 deletions(-) create mode 100644 doc/scripts/CMakeLists.txt create mode 100644 doc/scripts/README rename doc/{ => scripts}/conf.py.in (100%) rename doc/{ => scripts}/example.bro (100%) rename doc/scripts/{ => scripts}/BroToReST.py.in (100%) rename doc/scripts/{ => scripts}/generate_reST_docs.py.in (95%) rename doc/{ => scripts}/source/_static/showhide.js (100%) rename doc/{ => scripts}/source/_templates/layout.html (100%) rename doc/{ => scripts}/source/builtins.rst (100%) rename doc/{ => scripts}/source/common.rst (100%) rename doc/{ => scripts}/source/ext/bro.py (100%) rename doc/{ => scripts}/source/index.rst (100%) rename doc/{ => scripts}/source/policy/bifs.rst (100%) rename doc/{ => scripts}/source/policy/default.rst (100%) rename doc/{ => scripts}/source/policy/index.rst (100%) rename doc/{ => scripts}/source/policy/internal.rst (100%) rename doc/{ => scripts}/source/policy/user.rst (100%) diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 00e0974919..e2c3f25f4e 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -1,42 +1 @@ -set(POLICY_SRC_DIR ${PROJECT_SOURCE_DIR}/policy) -set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out) -set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source) -set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/source) - -file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*") - -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in - ${CMAKE_CURRENT_BINARY_DIR}/conf.py - @ONLY) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_reST_docs.py.in - ${CMAKE_CURRENT_BINARY_DIR}/generate_reST_docs.py - @ONLY) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/BroToReST.py.in - ${CMAKE_CURRENT_BINARY_DIR}/BroToReST.py - @ONLY) - -add_custom_target(doc - COMMAND "${CMAKE_COMMAND}" -E copy_directory - ${DOC_SOURCE_DIR} - ${DOC_SOURCE_WORKDIR} - COMMAND python generate_reST_docs.py - COMMAND sphinx-build - -b html - -c ${CMAKE_CURRENT_BINARY_DIR} - -d ${DOC_OUTPUT_DIR}/doctrees - ${DOC_SOURCE_WORKDIR} - ${DOC_OUTPUT_DIR}/html - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "[Sphinx] Generating Script Documentation" - VERBATIM - # SOURCES just adds stuff to IDE projects as a convienience - SOURCES ${DOC_SOURCES}) - -add_dependencies(doc bro doc-clean) - -add_custom_target(doc-clean - COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${CMAKE_CURRENT_BINARY_DIR}/source - COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${DOC_OUTPUT_DIR} - VERBATIM) +add_subdirectory(scripts) diff --git a/doc/README b/doc/README index 150260ed09..1333ed77b7 100644 --- a/doc/README +++ b/doc/README @@ -1,44 +1 @@ -This directory contains scripts and templates that can be used to automate -the generation of Bro script documentation. Two build targets are defined -by CMake: - -``make doc`` - - This target depends on a Python interpreter (>=2.5) and - `Sphinx `_ being installed. Sphinx can be - installed like:: - - > sudo easy_install sphinx - - This target will also first build the bro binary if it is not already - since the generation of reStructuredText (reST) documentation from - Bro scripts is integrated within the parsing process. - - After completion, HTML documentation can be located inside the CMake - ``build/`` directory as ``build/doc/out/html``. The generated reST - documentation will be located in ``build/doc/source/policy``. - -``make doc-clean`` - - This target removes Sphinx inputs and outputs from the CMake ``build/`` dir. - -To schedule a script to be documented, edit ``scripts/generate_reST_docs.py.in`` -and try adding the name of the script along with an optional script group to -the ``docs`` dictionary. That python script also shows other, more specialized -methods for generating documentation for some types of corner-cases. - -When adding a new logical grouping for generated scripts, create a new -reST document in ``source/policy/.rst`` and add some default -documentation. References to (and summaries of) documents associated with -the group get appended to this file during the ``make doc`` process. - -The Sphinx source tree template in ``source/`` can be modified to add more -common/general documentation, style sheets, JavaScript, etc. The Sphinx -config file is produced from ``conf.py.in``, so that can be edited to change -various Sphinx options, like setting the default HTML rendering theme. -There is also a custom Sphinx domain implemented in ``source/ext/bro.py`` -which adds some reST directives and roles that aid in generating useful -index entries and cross-references. - -See ``example.bro`` for an example of how to document a Bro script such that -``make doc`` will be able to produce reST/HTML documentation for it. +TODO diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt new file mode 100644 index 0000000000..00e0974919 --- /dev/null +++ b/doc/scripts/CMakeLists.txt @@ -0,0 +1,42 @@ +set(POLICY_SRC_DIR ${PROJECT_SOURCE_DIR}/policy) +set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out) +set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source) +set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/source) + +file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*") + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in + ${CMAKE_CURRENT_BINARY_DIR}/conf.py + @ONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_reST_docs.py.in + ${CMAKE_CURRENT_BINARY_DIR}/generate_reST_docs.py + @ONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/BroToReST.py.in + ${CMAKE_CURRENT_BINARY_DIR}/BroToReST.py + @ONLY) + +add_custom_target(doc + COMMAND "${CMAKE_COMMAND}" -E copy_directory + ${DOC_SOURCE_DIR} + ${DOC_SOURCE_WORKDIR} + COMMAND python generate_reST_docs.py + COMMAND sphinx-build + -b html + -c ${CMAKE_CURRENT_BINARY_DIR} + -d ${DOC_OUTPUT_DIR}/doctrees + ${DOC_SOURCE_WORKDIR} + ${DOC_OUTPUT_DIR}/html + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "[Sphinx] Generating Script Documentation" + VERBATIM + # SOURCES just adds stuff to IDE projects as a convienience + SOURCES ${DOC_SOURCES}) + +add_dependencies(doc bro doc-clean) + +add_custom_target(doc-clean + COMMAND "${CMAKE_COMMAND}" -E remove_directory + ${CMAKE_CURRENT_BINARY_DIR}/source + COMMAND "${CMAKE_COMMAND}" -E remove_directory + ${DOC_OUTPUT_DIR} + VERBATIM) diff --git a/doc/scripts/README b/doc/scripts/README new file mode 100644 index 0000000000..b7114ec262 --- /dev/null +++ b/doc/scripts/README @@ -0,0 +1,44 @@ +This directory contains scripts and templates that can be used to automate +the generation of Bro script documentation. Two build targets are defined +by CMake: + +``make doc`` + + This target depends on a Python interpreter (>=2.5) and + `Sphinx `_ being installed. Sphinx can be + installed like:: + + > sudo easy_install sphinx + + This target will also first build the bro binary if it is not already + since the generation of reStructuredText (reST) documentation from + Bro scripts is integrated within the parsing process. + + After completion, HTML documentation can be located inside the CMake + ``build/`` directory as ``doc/scripts/out/html``. The generated + reST documentation will be located in ``doc/scripts/source/policy``. + +``make doc-clean`` + + This target removes Sphinx inputs and outputs from the CMake ``build/`` dir. + +To schedule a script to be documented, edit ``scripts/generate_reST_docs.py.in`` +and try adding the name of the script along with an optional script group to +the ``docs`` dictionary. That python script also shows other, more specialized +methods for generating documentation for some types of corner-cases. + +When adding a new logical grouping for generated scripts, create a new +reST document in ``source/policy/.rst`` and add some default +documentation. References to (and summaries of) documents associated with +the group get appended to this file during the ``make doc`` process. + +The Sphinx source tree template in ``source/`` can be modified to add more +common/general documentation, style sheets, JavaScript, etc. The Sphinx +config file is produced from ``conf.py.in``, so that can be edited to change +various Sphinx options, like setting the default HTML rendering theme. +There is also a custom Sphinx domain implemented in ``source/ext/bro.py`` +which adds some reST directives and roles that aid in generating useful +index entries and cross-references. + +See ``example.bro`` for an example of how to document a Bro script such that +``make doc`` will be able to produce reST/HTML documentation for it. diff --git a/doc/conf.py.in b/doc/scripts/conf.py.in similarity index 100% rename from doc/conf.py.in rename to doc/scripts/conf.py.in diff --git a/doc/example.bro b/doc/scripts/example.bro similarity index 100% rename from doc/example.bro rename to doc/scripts/example.bro diff --git a/doc/scripts/BroToReST.py.in b/doc/scripts/scripts/BroToReST.py.in similarity index 100% rename from doc/scripts/BroToReST.py.in rename to doc/scripts/scripts/BroToReST.py.in diff --git a/doc/scripts/generate_reST_docs.py.in b/doc/scripts/scripts/generate_reST_docs.py.in similarity index 95% rename from doc/scripts/generate_reST_docs.py.in rename to doc/scripts/scripts/generate_reST_docs.py.in index 3278c42367..c54146314f 100755 --- a/doc/scripts/generate_reST_docs.py.in +++ b/doc/scripts/scripts/generate_reST_docs.py.in @@ -44,7 +44,7 @@ GenDocs(stdin_docs, True) # The example documentation script doesn't live on the BROPATH, so # explicitly generate the docs for it like this: -BroToReST("example.bro", False, ["@PROJECT_SOURCE_DIR@/doc"], group="internal").GenDoc() +BroToReST("example.bro", False, ["@PROJECT_SOURCE_DIR@/doc/scripts"], group="internal").GenDoc() # Generate documentation for stuff that's always loaded into bro by default: cmd = "echo '' | %s %s" % (BRO, BRO_ARGS) diff --git a/doc/source/_static/showhide.js b/doc/scripts/source/_static/showhide.js similarity index 100% rename from doc/source/_static/showhide.js rename to doc/scripts/source/_static/showhide.js diff --git a/doc/source/_templates/layout.html b/doc/scripts/source/_templates/layout.html similarity index 100% rename from doc/source/_templates/layout.html rename to doc/scripts/source/_templates/layout.html diff --git a/doc/source/builtins.rst b/doc/scripts/source/builtins.rst similarity index 100% rename from doc/source/builtins.rst rename to doc/scripts/source/builtins.rst diff --git a/doc/source/common.rst b/doc/scripts/source/common.rst similarity index 100% rename from doc/source/common.rst rename to doc/scripts/source/common.rst diff --git a/doc/source/ext/bro.py b/doc/scripts/source/ext/bro.py similarity index 100% rename from doc/source/ext/bro.py rename to doc/scripts/source/ext/bro.py diff --git a/doc/source/index.rst b/doc/scripts/source/index.rst similarity index 100% rename from doc/source/index.rst rename to doc/scripts/source/index.rst diff --git a/doc/source/policy/bifs.rst b/doc/scripts/source/policy/bifs.rst similarity index 100% rename from doc/source/policy/bifs.rst rename to doc/scripts/source/policy/bifs.rst diff --git a/doc/source/policy/default.rst b/doc/scripts/source/policy/default.rst similarity index 100% rename from doc/source/policy/default.rst rename to doc/scripts/source/policy/default.rst diff --git a/doc/source/policy/index.rst b/doc/scripts/source/policy/index.rst similarity index 100% rename from doc/source/policy/index.rst rename to doc/scripts/source/policy/index.rst diff --git a/doc/source/policy/internal.rst b/doc/scripts/source/policy/internal.rst similarity index 100% rename from doc/source/policy/internal.rst rename to doc/scripts/source/policy/internal.rst diff --git a/doc/source/policy/user.rst b/doc/scripts/source/policy/user.rst similarity index 100% rename from doc/source/policy/user.rst rename to doc/scripts/source/policy/user.rst diff --git a/testing/btest/doc/autogen-reST-example b/testing/btest/doc/autogen-reST-example index c47a604373..7f55384ce6 100644 --- a/testing/btest/doc/autogen-reST-example +++ b/testing/btest/doc/autogen-reST-example @@ -1,2 +1,2 @@ -@TEST-EXEC: bro --doc-scripts $DIST/doc/example.bro +@TEST-EXEC: bro --doc-scripts $DIST/doc/scripts/example.bro @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/doc/example-diff-canonifier.py btest-diff example.rst From 17314fa144c11a0012c2af35e17dc49c9994edb2 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 21 Apr 2011 13:34:37 -0500 Subject: [PATCH 05/56] Add parser error hint when in doc mode about checking ## comment syntax. --- src/parse.y | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/parse.y b/src/parse.y index 8e8a75cad7..debdf02177 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1622,7 +1622,7 @@ opt_doc_list: int yyerror(const char msg[]) { - char* msgbuf = new char[strlen(msg) + strlen(last_tok) + 64]; + char* msgbuf = new char[strlen(msg) + strlen(last_tok) + 128]; if ( last_tok[0] == '\n' ) sprintf(msgbuf, "%s, on previous line", msg); @@ -1631,6 +1631,10 @@ int yyerror(const char msg[]) else sprintf(msgbuf, "%s, at or near \"%s\"", msg, last_tok); + if ( generate_documentation ) + strcat(msgbuf, "\nDocumentation mode is enabled: " + "remember to check syntax of ## style comments\n"); + error(msgbuf); return 0; From c5a19f7cdca94fd90c89ef078efe9a960d52cfb2 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Fri, 22 Apr 2011 08:00:28 -0400 Subject: [PATCH 06/56] Fixed another do_split bug and included a test for the fix. --- src/strings.bif | 5 ++++- testing/btest/Baseline/bifs.string_splitting/out | 9 +++++++++ testing/btest/bifs/string_splitting.bro | 3 +++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/strings.bif b/src/strings.bif index 191d059b84..485fcdb108 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -244,8 +244,11 @@ Val* do_split(StringVal* str_val, RE_Matcher* re, TableVal* other_sep, --n; } - if ( num_sep >= max_num_sep ) + if ( max_num_sep && num_sep >= max_num_sep ) + { offset = end_of_s - s; + n=0; + } Val* ind = new Val(++num, TYPE_COUNT); a->Assign(ind, new StringVal(offset, (const char*) s)); diff --git a/testing/btest/Baseline/bifs.string_splitting/out b/testing/btest/Baseline/bifs.string_splitting/out index 7e0c0dfcfa..31bd2c963a 100644 --- a/testing/btest/Baseline/bifs.string_splitting/out +++ b/testing/btest/Baseline/bifs.string_splitting/out @@ -2,3 +2,12 @@ [1] = X-Mailer, [2] = Testing Test (http://www.example.com) } +{ +[1] = A , +[6] = =, +[4] = =, +[7] = D, +[5] = C , +[2] = =, +[3] = B +} diff --git a/testing/btest/bifs/string_splitting.bro b/testing/btest/bifs/string_splitting.bro index 1920db75a2..44068fe510 100644 --- a/testing/btest/bifs/string_splitting.bro +++ b/testing/btest/bifs/string_splitting.bro @@ -6,4 +6,7 @@ event bro_init() { local a = "X-Mailer: Testing Test (http://www.example.com)"; print split1(a, /:[[:blank:]]*/); + + a = "A = B = C = D"; + print split_all(a, /=/); } From 964060c32f5dc809d75ddca77cd8af7ae3646d2b Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 22 Apr 2011 18:07:29 -0700 Subject: [PATCH 07/56] Fixing bug with nested record coercions. --- src/Val.cc | 1 + .../language.record-recursive-coercion/output | 1 + .../language/record-recursive-coercion.bro | 24 +++++++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 testing/btest/Baseline/language.record-recursive-coercion/output create mode 100644 testing/btest/language/record-recursive-coercion.bro diff --git a/src/Val.cc b/src/Val.cc index a313e810cc..8d0fbdf499 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -2976,6 +2976,7 @@ RecordVal* RecordVal::CoerceTo(const RecordType* t, Val* aggr) const Expr* rhs = new ConstExpr(Lookup(i)->Ref()); Expr* e = new RecordCoerceExpr(rhs, ar_t->FieldType(t_i)->AsRecordType()); ar->Assign(t_i, e->Eval(0)); + continue; } ar->Assign(t_i, Lookup(i)->Ref()); diff --git a/testing/btest/Baseline/language.record-recursive-coercion/output b/testing/btest/Baseline/language.record-recursive-coercion/output new file mode 100644 index 0000000000..37c916713f --- /dev/null +++ b/testing/btest/Baseline/language.record-recursive-coercion/output @@ -0,0 +1 @@ +[major=4, minor=4, minor2=, addl=] diff --git a/testing/btest/language/record-recursive-coercion.bro b/testing/btest/language/record-recursive-coercion.bro new file mode 100644 index 0000000000..eda80e3d11 --- /dev/null +++ b/testing/btest/language/record-recursive-coercion.bro @@ -0,0 +1,24 @@ +# @TEST-EXEC: bro %INPUT >output 2>&1 +# @TEST-EXEC: btest-diff output + +type Version: record { + major: count &optional; + minor: count &optional; + minor2: count &optional; + addl: string &optional; +}; + +type Info: record { + name: string; + version: Version; +}; + +global matched_software: table[string] of Info = { + ["OpenSSH_4.4"] = [$name="OpenSSH", $version=[$major=4,$minor=4]], +}; + +event bro_init() + { + for ( sw in matched_software ) + print matched_software[sw]$version; + } From 46b1fd985013f1b599ed4d7ee4887da8b09ca5d9 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 22 Apr 2011 18:40:14 -0700 Subject: [PATCH 08/56] Delete operator for record fields. "delete x$y" now resets record field "x" back to its original state if it is either &optional or has a &default. "delete" may not be used with non-optional/default fields. --- src/Expr.cc | 12 ++++++++++- src/Expr.h | 3 +++ .../Baseline/istate.sync/receiver.vars.log | 16 +++++++------- .../Baseline/istate.sync/sender.vars.log | 16 +++++++------- .../Baseline/language.delete-field/output | 4 ++++ .../language.wrong-delete-field/output | 1 + testing/btest/istate/sync.bro | 4 +++- testing/btest/language/delete-field.bro | 21 +++++++++++++++++++ testing/btest/language/wrong-delete-field.bro | 11 ++++++++++ 9 files changed, 70 insertions(+), 18 deletions(-) create mode 100644 testing/btest/Baseline/language.delete-field/output create mode 100644 testing/btest/Baseline/language.wrong-delete-field/output create mode 100644 testing/btest/language/delete-field.bro create mode 100644 testing/btest/language/wrong-delete-field.bro diff --git a/src/Expr.cc b/src/Expr.cc index 71b94c7aa7..d70e7573a5 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -3113,9 +3113,14 @@ Expr* FieldExpr::Simplify(SimplifyType simp_type) return this; } +int FieldExpr::CanDel() const + { + return td->FindAttr(ATTR_DEFAULT) || td->FindAttr(ATTR_OPTIONAL); + } + void FieldExpr::Assign(Frame* f, Val* v, Opcode opcode) { - if ( IsError() || ! v ) + if ( IsError() ) return; if ( field < 0 ) @@ -3130,6 +3135,11 @@ void FieldExpr::Assign(Frame* f, Val* v, Opcode opcode) } } +void FieldExpr::Delete(Frame* f) + { + Assign(f, 0, OP_ASSIGN_IDX); + } + Val* FieldExpr::Fold(Val* v) const { Val* result = v->AsRecordVal()->Lookup(field); diff --git a/src/Expr.h b/src/Expr.h index 3d07f68df1..0f6ee67106 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -688,8 +688,11 @@ public: int Field() const { return field; } + int CanDel() const; + Expr* Simplify(SimplifyType simp_type); void Assign(Frame* f, Val* v, Opcode op = OP_ASSIGN); + void Delete(Frame* f); Expr* MakeLvalue(); diff --git a/testing/btest/Baseline/istate.sync/receiver.vars.log b/testing/btest/Baseline/istate.sync/receiver.vars.log index 2d7f88c7da..10ea000b52 100644 --- a/testing/btest/Baseline/istate.sync/receiver.vars.log +++ b/testing/btest/Baseline/istate.sync/receiver.vars.log @@ -13,22 +13,22 @@ Jodel file "test2" of string /^?(abbcdefgh)$?/ { +6, 4, 5, -3, -6, -2 +2, +3 } { +[4, JKL] = 104, [3, GHI] = 103, -[2, DEF] = 103, -[4, JKL] = 104 +[2, DEF] = 103 } { -[6767] = /^?(QWERTZ)$?/, [12346] = /^?(12345)$?/, -[12345] = /^?(12345)$?/ +[12345] = /^?(12345)$?/, +[6767] = /^?(QWERTZ)$?/ } 6667/tcp [2, 20, 3, 4] -[a=zxzxzx, b=[a=pop, b=43, c=9.999], c=[a=IOIOI, b=201, c=612.2], d=6.6666] +[a=zxzxzx, b=[a=pop, b=43, c=9.999], c=[a=IOIOI, b=201, c=612.2], d=6.6666, e=] diff --git a/testing/btest/Baseline/istate.sync/sender.vars.log b/testing/btest/Baseline/istate.sync/sender.vars.log index 2d7f88c7da..10ea000b52 100644 --- a/testing/btest/Baseline/istate.sync/sender.vars.log +++ b/testing/btest/Baseline/istate.sync/sender.vars.log @@ -13,22 +13,22 @@ Jodel file "test2" of string /^?(abbcdefgh)$?/ { +6, 4, 5, -3, -6, -2 +2, +3 } { +[4, JKL] = 104, [3, GHI] = 103, -[2, DEF] = 103, -[4, JKL] = 104 +[2, DEF] = 103 } { -[6767] = /^?(QWERTZ)$?/, [12346] = /^?(12345)$?/, -[12345] = /^?(12345)$?/ +[12345] = /^?(12345)$?/, +[6767] = /^?(QWERTZ)$?/ } 6667/tcp [2, 20, 3, 4] -[a=zxzxzx, b=[a=pop, b=43, c=9.999], c=[a=IOIOI, b=201, c=612.2], d=6.6666] +[a=zxzxzx, b=[a=pop, b=43, c=9.999], c=[a=IOIOI, b=201, c=612.2], d=6.6666, e=] diff --git a/testing/btest/Baseline/language.delete-field/output b/testing/btest/Baseline/language.delete-field/output new file mode 100644 index 0000000000..12d04d02d3 --- /dev/null +++ b/testing/btest/Baseline/language.delete-field/output @@ -0,0 +1,4 @@ +a: 20 +20 +a: not set +5 diff --git a/testing/btest/Baseline/language.wrong-delete-field/output b/testing/btest/Baseline/language.wrong-delete-field/output new file mode 100644 index 0000000000..c51fb6a37e --- /dev/null +++ b/testing/btest/Baseline/language.wrong-delete-field/output @@ -0,0 +1 @@ +/da/home/robin/bro/master/testing/btest/.tmp/language.wrong-delete-field/wrong-delete-field.bro, line 11 (delete x$a): error, illegal delete statement diff --git a/testing/btest/istate/sync.bro b/testing/btest/istate/sync.bro index 04f3f69dcf..d537ca10bc 100644 --- a/testing/btest/istate/sync.bro +++ b/testing/btest/istate/sync.bro @@ -39,13 +39,14 @@ type type2: record { b: type1; c: type1; d: double; + e: double &optional; }; global foo17: type2 = [ $a = "yuyuyu", $b = [$a="rec1", $b=100, $c=1.24], $c = [$a="rec2", $b=200, $c=2.24], - $d = 7.77 + $d = 7.77, $e=100.0 ] &persistent &synchronized; # Print variables. @@ -127,6 +128,7 @@ function modify() ++foo17$c$b; foo17$c$c = 612.2; foo17$d = 6.6666; + delete foo17$e; foo2 = 1234567; } diff --git a/testing/btest/language/delete-field.bro b/testing/btest/language/delete-field.bro new file mode 100644 index 0000000000..0aad10d55f --- /dev/null +++ b/testing/btest/language/delete-field.bro @@ -0,0 +1,21 @@ + +# @TEST-EXEC: bro %INPUT >output 2>&1 +# @TEST-EXEC: btest-diff output + +type X: record { + a: count &optional; + b: count &default=5; +}; + +function p(x: X) + { + print x?$a ? fmt("a: %d", x$a) : "a: not set"; + print x$b; + } + + +global x: X = [$a=20, $b=20]; +p(x); +delete x$a; +delete x$b; +p(x); diff --git a/testing/btest/language/wrong-delete-field.bro b/testing/btest/language/wrong-delete-field.bro new file mode 100644 index 0000000000..deffe379f4 --- /dev/null +++ b/testing/btest/language/wrong-delete-field.bro @@ -0,0 +1,11 @@ + +# @TEST-EXEC-FAIL: bro %INPUT >output 2>&1 +# @TEST-EXEC: btest-diff output + +type X: record { + a: count; +}; + +global x: X = [$a=20]; + +delete x$a; From c41da9ca99dbefde13068b6d5d07220d19042878 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sat, 23 Apr 2011 10:38:16 -0700 Subject: [PATCH 09/56] Fixing bug with deleting still unset record fields of table type. --- src/Val.cc | 2 +- .../Baseline/language.delete-field-set/output | 1 + testing/btest/language/delete-field-set.bro | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/language.delete-field-set/output create mode 100644 testing/btest/language/delete-field-set.bro diff --git a/src/Val.cc b/src/Val.cc index 8d0fbdf499..965ea3d026 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -2883,7 +2883,7 @@ RecordVal::~RecordVal() void RecordVal::Assign(int field, Val* new_val, Opcode op) { - if ( Lookup(field) && + if ( new_val && Lookup(field) && record_type->FieldType(field)->Tag() == TYPE_TABLE && new_val->AsTableVal()->FindAttr(ATTR_MERGEABLE) ) { diff --git a/testing/btest/Baseline/language.delete-field-set/output b/testing/btest/Baseline/language.delete-field-set/output new file mode 100644 index 0000000000..ccf6dbc793 --- /dev/null +++ b/testing/btest/Baseline/language.delete-field-set/output @@ -0,0 +1 @@ +[a=, b=, c=] diff --git a/testing/btest/language/delete-field-set.bro b/testing/btest/language/delete-field-set.bro new file mode 100644 index 0000000000..9469dbb2f0 --- /dev/null +++ b/testing/btest/language/delete-field-set.bro @@ -0,0 +1,17 @@ + +# @TEST-EXEC: bro %INPUT >output 2>&1 +# @TEST-EXEC: btest-diff output + +type FooBar: record { + a: set[string] &default=set(); + b: table[string] of count &default=table(); + c: vector of string &default=vector(); +}; + +global test: FooBar; + +delete test$a; +delete test$b; +delete test$c; + +print test; From 5662fe7358701382082a3dfbb539ec2af9c6d356 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sat, 23 Apr 2011 10:47:14 -0700 Subject: [PATCH 10/56] Updating baselines. --- testing/btest/Baseline/bifs.netbios-functions/out | 4 ++-- testing/btest/Baseline/bifs.string_splitting/out | 14 +++++++------- .../Baseline/language.rec-table-default/output | 4 ++-- .../Baseline/logging.remote-types/receiver.ssh.log | 2 +- .../Baseline/logging.remote/sender.ssh.failure.log | 6 +++--- .../btest/Baseline/logging.remote/sender.ssh.log | 10 +++++----- .../Baseline/logging.remote/sender.ssh.success.log | 4 ++-- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/testing/btest/Baseline/bifs.netbios-functions/out b/testing/btest/Baseline/bifs.netbios-functions/out index dacf5b9ab5..d849dbff71 100644 --- a/testing/btest/Baseline/bifs.netbios-functions/out +++ b/testing/btest/Baseline/bifs.netbios-functions/out @@ -2,7 +2,7 @@ MARTIN 3 WORKGROUP 27 -^A^B__MSBROWSE__^B -1 ISATAP 0 +^A^B__MSBROWSE__^B +1 diff --git a/testing/btest/Baseline/bifs.string_splitting/out b/testing/btest/Baseline/bifs.string_splitting/out index 31bd2c963a..8514916834 100644 --- a/testing/btest/Baseline/bifs.string_splitting/out +++ b/testing/btest/Baseline/bifs.string_splitting/out @@ -1,13 +1,13 @@ { -[1] = X-Mailer, -[2] = Testing Test (http://www.example.com) +[2] = Testing Test (http://www.example.com), +[1] = X-Mailer } { -[1] = A , -[6] = =, -[4] = =, -[7] = D, -[5] = C , [2] = =, +[4] = =, +[6] = =, +[7] = D, +[1] = A , +[5] = C , [3] = B } diff --git a/testing/btest/Baseline/language.rec-table-default/output b/testing/btest/Baseline/language.rec-table-default/output index 845fd8354d..a1531d06ee 100644 --- a/testing/btest/Baseline/language.rec-table-default/output +++ b/testing/btest/Baseline/language.rec-table-default/output @@ -5,9 +5,9 @@ } { -C, A, -B +B, +C } { diff --git a/testing/btest/Baseline/logging.remote-types/receiver.ssh.log b/testing/btest/Baseline/logging.remote-types/receiver.ssh.log index d703c68b96..9a203f1bac 100644 --- a/testing/btest/Baseline/logging.remote-types/receiver.ssh.log +++ b/testing/btest/Baseline/logging.remote-types/receiver.ssh.log @@ -1,2 +1,2 @@ # b i e c p sn n a d t iv s sc ss se vc ve -T -42 SSH::SSH 21 123 10.0.0.0/24 10.0.0.0 1.2.3.4 3.14 1303439439.02908 100.0 hurz 1,4,2,3 CC,AA,BB EMPTY 10,20,30 EMPTY +T -42 SSH::SSH 21 123 10.0.0.0/24 10.0.0.0 1.2.3.4 3.14 1303580757.69082 100.0 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY diff --git a/testing/btest/Baseline/logging.remote/sender.ssh.failure.log b/testing/btest/Baseline/logging.remote/sender.ssh.failure.log index 815b0366b3..ce48f547f6 100644 --- a/testing/btest/Baseline/logging.remote/sender.ssh.failure.log +++ b/testing/btest/Baseline/logging.remote/sender.ssh.failure.log @@ -1,4 +1,4 @@ # t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299718503.72819 1.2.3.4 1234 2.3.4.5 80 failure US -1299718503.72819 1.2.3.4 1234 2.3.4.5 80 failure UK -1299718503.72819 1.2.3.4 1234 2.3.4.5 80 failure MX +1303580768.27075 1.2.3.4 1234 2.3.4.5 80 failure US +1303580768.27075 1.2.3.4 1234 2.3.4.5 80 failure UK +1303580768.27075 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/logging.remote/sender.ssh.log b/testing/btest/Baseline/logging.remote/sender.ssh.log index 2f159ad6a7..eb392df781 100644 --- a/testing/btest/Baseline/logging.remote/sender.ssh.log +++ b/testing/btest/Baseline/logging.remote/sender.ssh.log @@ -1,6 +1,6 @@ # t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299718503.72819 1.2.3.4 1234 2.3.4.5 80 success - -1299718503.72819 1.2.3.4 1234 2.3.4.5 80 failure US -1299718503.72819 1.2.3.4 1234 2.3.4.5 80 failure UK -1299718503.72819 1.2.3.4 1234 2.3.4.5 80 success BR -1299718503.72819 1.2.3.4 1234 2.3.4.5 80 failure MX +1303580768.27075 1.2.3.4 1234 2.3.4.5 80 success - +1303580768.27075 1.2.3.4 1234 2.3.4.5 80 failure US +1303580768.27075 1.2.3.4 1234 2.3.4.5 80 failure UK +1303580768.27075 1.2.3.4 1234 2.3.4.5 80 success BR +1303580768.27075 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/logging.remote/sender.ssh.success.log b/testing/btest/Baseline/logging.remote/sender.ssh.success.log index 2838cc68b8..3861250477 100644 --- a/testing/btest/Baseline/logging.remote/sender.ssh.success.log +++ b/testing/btest/Baseline/logging.remote/sender.ssh.success.log @@ -1,3 +1,3 @@ # t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299718503.72819 1.2.3.4 1234 2.3.4.5 80 success - -1299718503.72819 1.2.3.4 1234 2.3.4.5 80 success BR +1303580768.27075 1.2.3.4 1234 2.3.4.5 80 success - +1303580768.27075 1.2.3.4 1234 2.3.4.5 80 success BR From f10d2e10eaee43fc760684980d95f6e8b94ff068 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 26 Apr 2011 22:13:04 -0500 Subject: [PATCH 11/56] Overhaul of "doc" build target for generating policy script documentation. It's now all implemented in CMake scripting. The generation of reST docs is now a distinct target, "restdoc", while the target to generate HTML docs, "doc", depends on "restdoc". reST doc generation supports incremental builds (documentation for a given policy script is only regenerated when it is out of date), but HTML doc generation via ``make doc`` is not incremental (Sphinx always starts with fresh input). Building the "restdoc" target is now covered by a btest to ensure all policy scripts are parse-able when Bro is in "doc mode". Generated reST docs should now support "@load"ing from subdirectories. e.g. "@load foo/baz" and "@load bar/baz" will now generate the right xref links. --- doc/scripts/CMakeLists.txt | 236 ++++++++++++++++-- doc/scripts/README | 49 ++-- doc/scripts/scripts/BroToReST.py.in | 152 ----------- doc/scripts/scripts/generate_reST_docs.py.in | 71 ------ doc/scripts/source/{policy => }/bifs.rst | 0 doc/scripts/source/{policy => }/default.rst | 0 doc/scripts/source/index.rst | 8 +- doc/scripts/source/{policy => }/internal.rst | 0 doc/scripts/source/policy/index.rst | 4 - doc/scripts/source/{policy => }/user.rst | 0 src/BroDoc.cc | 17 +- .../doc.autogen-reST-example/example.rst | 14 +- testing/btest/doc/autogen-reST-all | 1 + testing/btest/doc/autogen-reST-example | 2 +- 14 files changed, 276 insertions(+), 278 deletions(-) delete mode 100755 doc/scripts/scripts/BroToReST.py.in delete mode 100755 doc/scripts/scripts/generate_reST_docs.py.in rename doc/scripts/source/{policy => }/bifs.rst (100%) rename doc/scripts/source/{policy => }/default.rst (100%) rename doc/scripts/source/{policy => }/internal.rst (100%) rename doc/scripts/source/{policy => }/user.rst (100%) create mode 100644 testing/btest/doc/autogen-reST-all diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt index 00e0974919..acf9541fae 100644 --- a/doc/scripts/CMakeLists.txt +++ b/doc/scripts/CMakeLists.txt @@ -1,42 +1,250 @@ set(POLICY_SRC_DIR ${PROJECT_SOURCE_DIR}/policy) +set(RST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/rest_output) set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/out) set(DOC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source) set(DOC_SOURCE_WORKDIR ${CMAKE_CURRENT_BINARY_DIR}/source) file(GLOB_RECURSE DOC_SOURCES FOLLOW_SYMLINKS "*") +# configure the Sphinx config file (expand variables CMake might know about) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in ${CMAKE_CURRENT_BINARY_DIR}/conf.py @ONLY) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_reST_docs.py.in - ${CMAKE_CURRENT_BINARY_DIR}/generate_reST_docs.py - @ONLY) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/BroToReST.py.in - ${CMAKE_CURRENT_BINARY_DIR}/BroToReST.py - @ONLY) +# find out what BROPATH to use when executing bro +execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev + OUTPUT_VARIABLE BROPATH + RESULT_VARIABLE retval + OUTPUT_STRIP_TRAILING_WHITESPACE) +if (NOT ${retval} EQUAL 0) + message(FATAL_ERROR "Problem setting BROPATH") +endif () + +# This macro is used to add a new makefile target for reST policy script +# documentation that can be generated using Bro itself to parse policy scripts. +# It's called like: +# +# rest_target(srcDir broInput [group]) +# +# srcDir: the directory which contains broInput +# broInput: the file name of a bro policy script +# group: optional name of group that the script documentation will belong to +# +# In addition to adding the makefile target, several CMake variables are set: +# +# MASTER_POLICY_INDEX_TEXT: a running list of policy scripts docs that have +# been generated so far, formatted such that it can be appended to a file +# that ends in a Sphinx toctree directive +# ALL_REST_OUTPUTS: a running list (the CMake list type) of all reST docs +# that are to be generated +# MASTER_GROUP_LIST: a running list (the CMake list type) of all script groups +# ${group}_TEXT: a running list of policy script :doc: references and summary +# text for a given group +# +macro(REST_TARGET srcDir broInput) + get_filename_component(basename ${broInput} NAME_WE) + get_filename_component(extension ${broInput} EXT) + get_filename_component(relDstDir ${broInput} PATH) + + if (${extension} STREQUAL ".bif.bro") + set(basename "${basename}.bif") + elseif (${extension} STREQUAL ".init") + set(basename "${basename}.init") + endif () + + set (restFile "${basename}.rst") + + if (NOT relDstDir) + set(docName "${basename}") + set(dstDir "${RST_OUTPUT_DIR}") + else () + set(docName "${relDstDir}/${basename}") + set(dstDir "${RST_OUTPUT_DIR}/${relDstDir}") + endif () + + set(restOutput "${dstDir}/${restFile}") + + set(indexEntry " ${docName} <${docName}>") + set(MASTER_POLICY_INDEX_TEXT "${MASTER_POLICY_INDEX_TEXT}\n${indexEntry}") + list(APPEND ALL_REST_OUTPUTS ${restOutput}) + + if (NOT "${ARGN}" STREQUAL "") + set(group ${ARGN}) + # add group to master group list if not already in it + list(FIND MASTER_GROUP_LIST ${group} _found) + if (_found EQUAL -1) + list(APPEND MASTER_GROUP_LIST ${group}) + if (MASTER_GROUP_LIST_TEXT) + set(MASTER_GROUP_LIST_TEXT "${MASTER_GROUP_LIST_TEXT}\n${group}") + else () + set(MASTER_GROUP_LIST_TEXT "${group}") + endif () + endif () + + # add script's summary documentation to text associated with the group + set(${group}_TEXT "${${group}_TEXT}\n\n:doc:`/policy/${docName}`\n") + file(STRINGS ${srcDir}/${broInput} summary_text REGEX "^\#\#!") + foreach (line ${summary_text}) + string(REGEX REPLACE "^\#\#!" " " line ${line}) + set(${group}_TEXT "${${group}_TEXT}\n${line}") + endforeach () + else () + set(group "") + endif () + + if (${group} STREQUAL "default" OR ${group} STREQUAL "bifs") + set(BRO_ARGS --doc-scripts --exec '') + else () + set(BRO_ARGS --doc-scripts ${srcDir}/${broInput}) + endif () + + add_custom_command(OUTPUT ${restOutput} + # delete any leftover state from previous bro runs + COMMAND "${CMAKE_COMMAND}" + ARGS -E remove_directory .state + # generate the reST documentation using bro + COMMAND BROPATH=${BROPATH} ${CMAKE_BINARY_DIR}/src/bro + ARGS ${BRO_ARGS} || (rm -rf .state *.log *.rst && exit 1) + # move generated doc into a new directory tree that + # defines the final structure of documents + COMMAND "${CMAKE_COMMAND}" + ARGS -E make_directory ${dstDir} + COMMAND "${CMAKE_COMMAND}" + ARGS -E copy ${restFile} ${restOutput} + # copy the bro policy script, too + COMMAND "${CMAKE_COMMAND}" + ARGS -E copy ${srcDir}/${broInput} ${dstDir} + # clean up the build directory + COMMAND rm + ARGS -rf .state *.log *.rst + DEPENDS bro + DEPENDS ${srcDir}/${broInput} + COMMENT "[Bro] Generating reST docs for ${broInput}" + ) + +endmacro(REST_TARGET) + +# Schedule Bro scripts for which to generate documentation. +# Note: the script may be located in a subdirectory off of one of the main +# directories in BROPATH. In that case, just list the script as 'foo/bar.bro' +rest_target(${POLICY_SRC_DIR} alarm.bro user) +rest_target(${POLICY_SRC_DIR} arp.bro user) +rest_target(${POLICY_SRC_DIR} conn.bro user) +rest_target(${POLICY_SRC_DIR} dhcp.bro user) +rest_target(${POLICY_SRC_DIR} dns.bro user) +rest_target(${POLICY_SRC_DIR} ftp.bro user) +rest_target(${POLICY_SRC_DIR} http.bro user) +rest_target(${POLICY_SRC_DIR} http-reply.bro user) +rest_target(${POLICY_SRC_DIR} http-request.bro user) +rest_target(${POLICY_SRC_DIR} irc.bro user) +rest_target(${POLICY_SRC_DIR} smtp.bro user) +rest_target(${POLICY_SRC_DIR} ssl.bro user) +rest_target(${POLICY_SRC_DIR} ssl-ciphers.bro user) +rest_target(${POLICY_SRC_DIR} ssl-errors.bro user) +rest_target(${POLICY_SRC_DIR} synflood.bro user) +rest_target(${POLICY_SRC_DIR} tcp.bro user) +rest_target(${POLICY_SRC_DIR} udp.bro user) +rest_target(${POLICY_SRC_DIR} weird.bro user) +rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal) + +# Finding out what scripts bro will generate documentation for by default +# can be done like: `bro --doc-scripts --exec ""` +rest_target(${POLICY_SRC_DIR} bro.init default) +rest_target(${POLICY_SRC_DIR} logging-ascii.bro default) +rest_target(${POLICY_SRC_DIR} logging.bro default) +rest_target(${POLICY_SRC_DIR} pcap.bro default) +rest_target(${POLICY_SRC_DIR} server-ports.bro default) +rest_target(${CMAKE_BINARY_DIR}/src bro.bif.bro bifs) +rest_target(${CMAKE_BINARY_DIR}/src const.bif.bro bifs) +rest_target(${CMAKE_BINARY_DIR}/src event.bif.bro bifs) +rest_target(${CMAKE_BINARY_DIR}/src logging.bif.bro bifs) +rest_target(${CMAKE_BINARY_DIR}/src strings.bif.bro bifs) +rest_target(${CMAKE_BINARY_DIR}/src types.bif.bro bifs) + +# create temporary list of all docs to include in the master policy/index file +file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/tmp_policy_index + "${MASTER_POLICY_INDEX_TEXT}") + +# create temporary file containing list of all groups +file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/group_list + "${MASTER_GROUP_LIST_TEXT}") + +# create temporary file containing the summary text for all scripts in a group +foreach (group ${MASTER_GROUP_LIST}) + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${group}_text + "${${group}_TEXT}") +endforeach () + +# remove previously generated docs no longer scheduled for generation +if (EXISTS ${RST_OUTPUT_DIR}) + file(GLOB_RECURSE EXISTING_REST_DOCS "${RST_OUTPUT_DIR}/*.rst") + foreach (_doc ${EXISTING_REST_DOCS}) + list(FIND ALL_REST_OUTPUTS ${_doc} _found) + if (_found EQUAL -1) + file(REMOVE ${_doc}) + message(STATUS "Removing stale reST doc: ${_doc}") + endif () + endforeach () +endif () + +# The "restdoc" target uses Bro to parse policy scripts in order to +# generate reST documentation from them. +add_custom_target(restdoc + # create symlink to the reST output directory for convenience + COMMAND "${CMAKE_COMMAND}" -E create_symlink + ${RST_OUTPUT_DIR} + ${CMAKE_BINARY_DIR}/reST + DEPENDS ${ALL_REST_OUTPUTS}) + +# The "restclean" target removes all generated reST documentation from the +# build directory. +add_custom_target(restclean + COMMAND "${CMAKE_COMMAND}" -E remove_directory + ${RST_OUTPUT_DIR} + VERBATIM) + +# The "doc" target generates reST documentation for any outdated bro scripts +# and then uses Sphinx to generate HTML documentation from the reST add_custom_target(doc + # copy the template documentation to the build directory + # to give as input for sphinx COMMAND "${CMAKE_COMMAND}" -E copy_directory ${DOC_SOURCE_DIR} ${DOC_SOURCE_WORKDIR} - COMMAND python generate_reST_docs.py + # copy generated policy script documentation into the + # working copy of the template documentation + COMMAND "${CMAKE_COMMAND}" -E copy_directory + ${RST_OUTPUT_DIR} + ${DOC_SOURCE_WORKDIR}/policy + # append to the master index of all policy scripts + COMMAND cat ${CMAKE_CURRENT_BINARY_DIR}/tmp_policy_index >> + ${DOC_SOURCE_WORKDIR}/policy/index.rst + # construct the reST file for all groups + COMMAND xargs -I{} sh -c 'cat "$$1_text" >> + "${DOC_SOURCE_WORKDIR}/$$1.rst"' -- {} < + ${CMAKE_CURRENT_BINARY_DIR}/group_list + # tell sphinx to generate html COMMAND sphinx-build -b html -c ${CMAKE_CURRENT_BINARY_DIR} -d ${DOC_OUTPUT_DIR}/doctrees ${DOC_SOURCE_WORKDIR} ${DOC_OUTPUT_DIR}/html + # create symlink to the html output directory for convenience + COMMAND "${CMAKE_COMMAND}" -E create_symlink + ${DOC_OUTPUT_DIR}/html + ${CMAKE_BINARY_DIR}/html WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "[Sphinx] Generating Script Documentation" - VERBATIM + COMMENT "[Sphinx] Generating HTML policy script docs" # SOURCES just adds stuff to IDE projects as a convienience - SOURCES ${DOC_SOURCES}) + SOURCES ${DOC_SOURCES} + DEPENDS restdoc docclean) -add_dependencies(doc bro doc-clean) - -add_custom_target(doc-clean +# The "docclean" target removes just the Sphinx input/output directories +# from the build directory. +add_custom_target(docclean COMMAND "${CMAKE_COMMAND}" -E remove_directory - ${CMAKE_CURRENT_BINARY_DIR}/source + ${DOC_SOURCE_WORKDIR} COMMAND "${CMAKE_COMMAND}" -E remove_directory ${DOC_OUTPUT_DIR} VERBATIM) diff --git a/doc/scripts/README b/doc/scripts/README index b7114ec262..85496322d9 100644 --- a/doc/scripts/README +++ b/doc/scripts/README @@ -1,8 +1,21 @@ This directory contains scripts and templates that can be used to automate -the generation of Bro script documentation. Two build targets are defined +the generation of Bro script documentation. Several build targets are defined by CMake: -``make doc`` +``restdoc`` + + This target uses Bro to parse policy scripts in order to generate + reStructuredText (reST) documentation from them. The list of scripts + for which to generate reST documentation is defined in the + ``CMakeLists.txt`` file in this directory. Script documentation is + rebuild automatically if the policy script from which it is derived + or the Bro binary becomes out of date + + The resulting output from this target can be found in the CMake + ``build/`` directory inside ``reST`` (a symlink to + ``doc/scripts/rest_output``). + +``doc`` This target depends on a Python interpreter (>=2.5) and `Sphinx `_ being installed. Sphinx can be @@ -10,27 +23,31 @@ by CMake: > sudo easy_install sphinx - This target will also first build the bro binary if it is not already - since the generation of reStructuredText (reST) documentation from - Bro scripts is integrated within the parsing process. + This target will first build ``restdoc`` target and then copy the + resulting reST files as an input directory to Sphinx. - After completion, HTML documentation can be located inside the CMake - ``build/`` directory as ``doc/scripts/out/html``. The generated - reST documentation will be located in ``doc/scripts/source/policy``. + After completion, HTML documentation can be located in the CMake + ``build/`` directory inside ``html`` (a symlink to + ``doc/scripts/out/html``) -``make doc-clean`` +``restclean`` + + This target removes any reST documentation that has been generated so far. + +``docclean`` This target removes Sphinx inputs and outputs from the CMake ``build/`` dir. -To schedule a script to be documented, edit ``scripts/generate_reST_docs.py.in`` -and try adding the name of the script along with an optional script group to -the ``docs`` dictionary. That python script also shows other, more specialized -methods for generating documentation for some types of corner-cases. +To schedule a script to be documented, edit ``CMakeLists.txt`` inside this +directory add a call to the ``rest_target()`` macro. Calling that macro +with a group name for the script is optional, but if not given, the only +link to the script will be in the master TOC tree for all policy scripts. When adding a new logical grouping for generated scripts, create a new -reST document in ``source/policy/.rst`` and add some default -documentation. References to (and summaries of) documents associated with -the group get appended to this file during the ``make doc`` process. +reST document in ``source/.rst`` and add some default +documentation for the group. References to (and summaries of) documents +associated with the group get appended to this file during the +``make doc`` process. The Sphinx source tree template in ``source/`` can be modified to add more common/general documentation, style sheets, JavaScript, etc. The Sphinx diff --git a/doc/scripts/scripts/BroToReST.py.in b/doc/scripts/scripts/BroToReST.py.in deleted file mode 100755 index 89a8e1cc57..0000000000 --- a/doc/scripts/scripts/BroToReST.py.in +++ /dev/null @@ -1,152 +0,0 @@ -#! /usr/bin/env python - -import os -import subprocess -import shutil -import glob -import string -import sys - -BRO = "@CMAKE_BINARY_DIR@/src/bro" -BROPATHDEV = "`@CMAKE_BINARY_DIR@/bro-path-dev`" -BRO_ARGS = "--doc-scripts" -DOC_DST_DIR = "@DOC_SOURCE_WORKDIR@/policy" -BROPATH = subprocess.Popen("@CMAKE_BINARY_DIR@/bro-path-dev", shell=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.readline() - -class BroToReST: - """A class to encapsulate the the generation of reST documentation from - a given Bro script. - """ - - bro_src_file = None - doc_src_file = None - load_via_stdin = False - group = None - - def __init__(self, src_file, load_method=False, search_dir=None, group=None): - """ - :param src_file: the file name of a Bro script (not a path) - :param load_method: T if script must be loaded by Bro via a stdin - redirection of "@load