Merge remote-tracking branch 'origin/master' into topic/seth/sumstats-updates

doc/file-analysis.rst

@@ -82,9 +82,9 @@ attached, they start receiving the contents of the file as Bro extracts
 it from an ongoing network connection.  What they do with the file
 contents is up to the particular file analyzer implementation, but
 they'll typically either report further information about the file via
-events (e.g. :bro:see:`FileAnalysis::ANALYZER_MD5` will report the
+events (e.g. :bro:see:`Files::ANALYZER_MD5` will report the
 file's MD5 checksum via :bro:see:`file_hash` once calculated) or they'll
-have some side effect (e.g. :bro:see:`FileAnalysis::ANALYZER_EXTRACT`
+have some side effect (e.g. :bro:see:`Files::ANALYZER_EXTRACT`
 will write the contents of the file out to the local file system).
 
 In the future there may be file analyzers that automatically attach to
@@ -98,7 +98,7 @@ explicit attachment decision:
         {
         print "new file", f$id;
         if ( f?$mime_type && f$mime_type == "text/plain" )
-            FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]);
+            Files::add_analyzer(f, Files::ANALYZER_MD5);
         }
 
     event file_hash(f: fa_file, kind: string, hash: string)
@@ -113,26 +113,27 @@ output::
     file_hash, Cx92a0ym5R8, md5, 397168fd09991a0e712254df7bc639ac
 
 Some file analyzers might have tunable parameters that need to be
-specified in the call to :bro:see:`FileAnalysis::add_analyzer`:
+specified in the call to :bro:see:`Files::add_analyzer`:
 
 .. code:: bro
 
     event file_new(f: fa_file)
         {
-        FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT,
-                                       $extract_filename="./myfile"]);
+        Files::add_analyzer(f, Files::ANALYZER_EXTRACT,
+                            [$extract_filename="myfile"]);
         }
 
 In this case, the file extraction analyzer doesn't generate any further
-events, but does have the side effect of writing out the file contents
-to the local file system at the specified location of ``./myfile``.  Of
-course, for a network with more than a single file being transferred,
-it's probably preferable to specify a different extraction path for each
-file, unlike this example.
+events, but does have the effect of writing out the file contents to the
+local file system at the location resulting from the concatenation of
+the path specified by :bro:see:`FileExtract::prefix` and the string,
+``myfile``.  Of course, for a network with more than a single file being
+transferred, it's probably preferable to specify a different extraction
+path for each file, unlike this example.
 
 Regardless of which file analyzers end up acting on a file, general
 information about the file (e.g. size, time of last data transferred,
-MIME type, etc.) are logged in ``file_analysis.log``.
+MIME type, etc.) are logged in ``files.log``.
 
 Input Framework Integration
 ===========================
@@ -150,7 +151,7 @@ a network interface it's monitoring.  It only requires a call to
     event file_new(f: fa_file)
         {
         print "new file", f$id;
-        FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]);
+        Files::add_analyzer(f, Files::ANALYZER_MD5);
         }
 
     event file_state_remove(f: fa_file)

doc/index.rst

@@ -47,6 +47,7 @@ Script Reference
    scripts/index
    scripts/builtins
    scripts/proto-analyzers
+   scripts/file-analyzers
 
 Other Bro Components
 --------------------

doc/scripts/CMakeLists.txt

@@ -124,28 +124,34 @@ endmacro(REST_TARGET)
 # Schedule Bro scripts for which to generate documentation.
 include(DocSourcesList.cmake)
 
-# This reST target is independent of a particular Bro script...
-add_custom_command(OUTPUT proto-analyzers.rst
-    # delete any leftover state from previous bro runs
-    COMMAND "${CMAKE_COMMAND}"
-    ARGS -E remove_directory .state
-    # generate the reST documentation using bro
-    COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro
-    ARGS -b -Z base/init-bare.bro || (rm -rf .state *.log *.rst && exit 1)
-    # move generated doc into a new directory tree that
-    # defines the final structure of documents
-    COMMAND "${CMAKE_COMMAND}"
-    ARGS -E make_directory ${dstDir}
-    COMMAND "${CMAKE_COMMAND}"
-    ARGS -E copy proto-analyzers.rst ${dstDir}
-    # clean up the build directory
-    COMMAND rm
-    ARGS -rf .state *.log *.rst
-    DEPENDS bro
-    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
-    COMMENT "[Bro] Generating reST docs for proto-analyzers.rst"
-)
-list(APPEND ALL_REST_OUTPUTS proto-analyzers.rst)
+# Macro for generating reST docs that are independent of any particular Bro
+# script.
+macro(INDEPENDENT_REST_TARGET reST_file)
+    add_custom_command(OUTPUT ${reST_file}
+        # delete any leftover state from previous bro runs
+        COMMAND "${CMAKE_COMMAND}"
+        ARGS -E remove_directory .state
+        # generate the reST documentation using bro
+        COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro
+        ARGS -b -Z base/init-bare.bro || (rm -rf .state *.log *.rst && exit 1)
+        # move generated doc into a new directory tree that
+        # defines the final structure of documents
+        COMMAND "${CMAKE_COMMAND}"
+        ARGS -E make_directory ${dstDir}
+        COMMAND "${CMAKE_COMMAND}"
+        ARGS -E copy ${reST_file} ${dstDir}
+        # clean up the build directory
+        COMMAND rm
+        ARGS -rf .state *.log *.rst
+        DEPENDS bro
+        WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+        COMMENT "[Bro] Generating reST docs for ${reST_file}"
+    )
+    list(APPEND ALL_REST_OUTPUTS ${reST_file})
+endmacro(INDEPENDENT_REST_TARGET)
+
+independent_rest_target(proto-analyzers.rst)
+independent_rest_target(file-analyzers.rst)
 
 # create temporary list of all docs to include in the master policy/index file
 file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}")

doc/scripts/DocSourcesList.cmake

@@ -73,6 +73,7 @@ rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_UDP.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ZIP.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/reporter.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/strings.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/top-k.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/types.bif.bro)
 rest_target(${psd} base/files/extract/main.bro)
 rest_target(${psd} base/files/hash/main.bro)
@@ -129,6 +130,7 @@ rest_target(${psd} base/frameworks/sumstats/plugins/min.bro)
 rest_target(${psd} base/frameworks/sumstats/plugins/sample.bro)
 rest_target(${psd} base/frameworks/sumstats/plugins/std-dev.bro)
 rest_target(${psd} base/frameworks/sumstats/plugins/sum.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/topk.bro)
 rest_target(${psd} base/frameworks/sumstats/plugins/unique.bro)
 rest_target(${psd} base/frameworks/sumstats/plugins/variance.bro)
 rest_target(${psd} base/frameworks/tunnels/main.bro)
@@ -141,6 +143,7 @@ rest_target(${psd} base/protocols/dns/consts.bro)
 rest_target(${psd} base/protocols/dns/main.bro)
 rest_target(${psd} base/protocols/ftp/files.bro)
 rest_target(${psd} base/protocols/ftp/gridftp.bro)
+rest_target(${psd} base/protocols/ftp/info.bro)
 rest_target(${psd} base/protocols/ftp/main.bro)
 rest_target(${psd} base/protocols/ftp/utils-commands.bro)
 rest_target(${psd} base/protocols/ftp/utils.bro)