Merge remote branch 'origin/topic/jsiwek/misc-doc-fixes'

* origin/topic/jsiwek/misc-doc-fixes:
  More tweaks to generated script docs.
  Various changes to documentation framework.

Closes #598.
This commit is contained in:
Robin Sommer 2011-09-08 08:41:12 -07:00
commit db8ab89c3a
21 changed files with 169 additions and 119 deletions

12
CHANGES
View file

@ -1,4 +1,16 @@
1.6-dev-1221 | 2011-09-08 08:41:17 -0700
* Updates for documentation framework and script docs. (Jon Siwek)
* The script level PF_RING support isn't working so removing it.
(Seth Hall)
* Delete SSL certificates from memory after ssl_established event.
(Seth Hall)
* Small fixes for SSL analysis. (Seth Hall)
1.6-dev-1212 | 2011-09-07 16:15:28 -0700
* Internally, the UID generation can now return values from

View file

@ -13,15 +13,14 @@ all: configured
install: configured
( cd $(BUILD) && make install )
clean: configured
clean: configured docclean
( cd $(BUILD) && make clean )
( cd $(BUILD) && make docclean && make restclean )
doc: configured
( cd $(BUILD) && make doc )
docclean: configured
( cd $(BUILD) && make docclean && make restclean )
( cd $(BUILD) && make docclean )
dist:
@./pkg/make-src-packages

View file

@ -1 +1 @@
1.6-dev-1212
1.6-dev-1221

@ -1 +1 @@
Subproject commit 3679dbce2e1a7fc31065b174670fa54e35c3ae75
Subproject commit 2b9053d40d7ef497c8cef6357b59f43129976d65

View file

@ -1 +1,4 @@
add_custom_target(doc)
add_custom_target(docclean)
add_subdirectory(scripts)

View file

@ -212,9 +212,9 @@ add_custom_target(restclean
${RST_OUTPUT_DIR}
VERBATIM)
# The "doc" target generates reST documentation for any outdated bro scripts
# and then uses Sphinx to generate HTML documentation from the reST
add_custom_target(doc
# The "sphinxdoc" target generates reST documentation for any outdated bro
# scripts and then uses Sphinx to generate HTML documentation from the reST
add_custom_target(sphinxdoc
# copy the template documentation to the build directory
# to give as input for sphinx
COMMAND "${CMAKE_COMMAND}" -E copy_directory
@ -252,13 +252,16 @@ add_custom_target(doc
# SOURCES just adds stuff to IDE projects as a convenience
SOURCES ${DOC_SOURCES})
# The "docclean" target removes just the Sphinx input/output directories
# The "sphinxclean" target removes just the Sphinx input/output directories
# from the build directory.
add_custom_target(docclean
add_custom_target(sphinxclean
COMMAND "${CMAKE_COMMAND}" -E remove_directory
${DOC_SOURCE_WORKDIR}
COMMAND "${CMAKE_COMMAND}" -E remove_directory
${DOC_OUTPUT_DIR}
VERBATIM)
add_dependencies(doc docclean restdoc)
add_dependencies(sphinxdoc sphinxclean restdoc)
add_dependencies(doc sphinxdoc)
add_dependencies(docclean sphinxclean restclean)

View file

@ -16,13 +16,13 @@ rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
rest_target(${psd} base/init-default.bro internal)
rest_target(${psd} base/init-bare.bro internal)
rest_target(${CMAKE_BINARY_DIR}/src/base bro.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src/base const.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src/base event.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src/base logging.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src/base reporter.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src/base strings.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src/base types.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro)
rest_target(${CMAKE_BINARY_DIR}/src base/types.bif.bro)
rest_target(${psd} base/frameworks/cluster/main.bro)
rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro)
@ -33,6 +33,7 @@ rest_target(${psd} base/frameworks/control/main.bro)
rest_target(${psd} base/frameworks/dpd/main.bro)
rest_target(${psd} base/frameworks/intel/main.bro)
rest_target(${psd} base/frameworks/logging/main.bro)
rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
rest_target(${psd} base/frameworks/metrics/cluster.bro)
rest_target(${psd} base/frameworks/metrics/main.bro)
@ -101,6 +102,7 @@ rest_target(${psd} policy/integration/barnyard2/main.bro)
rest_target(${psd} policy/integration/barnyard2/types.bro)
rest_target(${psd} policy/misc/analysis-groups.bro)
rest_target(${psd} policy/misc/loaded-scripts.bro)
rest_target(${psd} policy/misc/pf-ring-load-balancing.bro)
rest_target(${psd} policy/misc/profiling.bro)
rest_target(${psd} policy/misc/trim-trace-file.bro)
rest_target(${psd} policy/protocols/conn/known-hosts.bro)

View file

@ -5,20 +5,6 @@
##! (reST) document's summary section.
##!
##! .. tip:: You can embed directives and roles within ``##``-stylized comments.
##!
##! A script's logging information has to be documented manually as minimally
##! shown below. Note that references may not always be possible (e.g.
##! anonymous filter functions) and a script may not need to document
##! each of "columns", "event", "filter" depending on exactly what it's doing.
##!
##! **Logging Stream ID:** :bro:enum:`Example::EXAMPLE`
##! :Columns: :bro:type:`Example::Info`
##! :Event: :bro:id:`Example::log_example`
##! :Filter: ``example-filter``
##! uses :bro:id:`Example::filter_func` to determine whether to
##! exclude the ``ts`` field
##!
##! :Author: Jon Siwek <jsiwek@ncsa.illinois.edu>
# Comments that use a single pound sign (#) are not significant to
# a script's auto-generated documentation, but ones that use a

View file

@ -73,7 +73,7 @@ bifs=`( cd ${sourcedir}/src && find . -name \*\.bif | sort )`
for file in $bifs
do
f=${file:2}.bro
echo "rest_target(\${CMAKE_BINARY_DIR}/src/base $f)" >> $outfile
echo "rest_target(\${CMAKE_BINARY_DIR}/src base/$f)" >> $outfile
done
scriptfiles=`( cd ${sourcedir}/scripts && find . -name \*\.bro | sort )`

View file

@ -52,7 +52,7 @@ export {
## If not given, all entries are recorded.
##
## rec: An instance of the streams's ``columns`` type with its
## fields set to the values to logged.
## fields set to the values to logged.
##
## Returns: True if the entry is to be recorded.
pred: function(rec: any): bool &optional;

View file

@ -1382,8 +1382,9 @@ const enable_syslog = F &redef;
const peer_description = "bro" &redef;
## If true, broadcast events/state received from one peer to other peers.
## NOTE: These options are only temporary. They will disappear when we get a
## more sophisticated script-level communication framework.
##
## .. note:: These options are only temporary. They will disappear when we get
## a more sophisticated script-level communication framework.
const forward_remote_events = F &redef;
## See :bro:id:`forward_remote_events`
const forward_remote_state_changes = F &redef;
@ -1513,6 +1514,6 @@ const skip_http_data = F &redef;
## UDP tunnels. See also: udp_tunnel_port, policy/udp-tunnel.bro.
const parse_udp_tunnels = F &redef;
## Load the logging framework here because it uses fairly deep integration with
## BiFs and script-land defined types.
# Load the logging framework here because it uses fairly deep integration with
# BiFs and script-land defined types.
@load base/frameworks/logging

View file

@ -74,11 +74,11 @@ export {
## This history is not meant to encode how much data that happened to be.
history: string &log &optional;
## Number of packets the originator sent.
## Only set if :bro:id:`use_conn_size_analyzer`=T
## Only set if :bro:id:`use_conn_size_analyzer` = T
orig_pkts: count &log &optional;
## Number IP level bytes the originator sent (as seen on the wire,
## taken from IP total_length header field).
## Only set if :bro:id:`use_conn_size_analyzer`=T
## Only set if :bro:id:`use_conn_size_analyzer` = T
orig_ip_bytes: count &log &optional;
## Number of packets the responder sent. See ``orig_pkts``.
resp_pkts: count &log &optional;

View file

@ -227,6 +227,7 @@ void BroDoc::WriteDocFile() const
WriteToDoc("%s\n", packet_filter.c_str());
}
#if 0 // Disabled for now.
BroDocObjList::const_iterator it;
bool hasPrivateIdentifiers = false;
@ -241,6 +242,7 @@ void BroDoc::WriteDocFile() const
if ( hasPrivateIdentifiers )
WriteInterface("Private Interface", '-', '~', false, false);
#endif
}
void BroDoc::WriteInterface(const char* heading, char underline,

View file

@ -167,6 +167,18 @@ public:
all.push_back(o);
}
/**
* Schedules documentation of an event handler declared by the script.
* @param o A pointer to a BroDocObj which contains the internal
* Bro language representation of the script event handler and
* also any associated comments about it.
*/
void AddEventHandler(const BroDocObj* o)
{
event_handlers.push_back(o);
all.push_back(o);
}
/**
* Schedules documentation of a function declared by the script.
* @param o A pointer to a BroDocObj which contains the internal
@ -228,6 +240,7 @@ protected:
BroDocObjList types;
BroDocObjList notices;
BroDocObjList events;
BroDocObjList event_handlers;
BroDocObjMap functions;
BroDocObjList redefs;

View file

@ -418,6 +418,7 @@ collect_headers(bro_HEADERS ${bro_SRCS})
add_definitions(-DBRO_SCRIPT_INSTALL_PATH="${BRO_SCRIPT_INSTALL_PATH}")
add_definitions(-DBRO_SCRIPT_SOURCE_PATH="${BRO_SCRIPT_SOURCE_PATH}")
add_definitions(-DBRO_BUILD_PATH="${CMAKE_CURRENT_BINARY_DIR}")
add_executable(bro ${bro_SRCS} ${bro_HEADERS})

View file

@ -1179,7 +1179,7 @@ func_hdr:
FUNC_FLAVOR_EVENT, 0, $3);
$$ = $3;
if ( generate_documentation )
current_reST_doc->AddEvent(
current_reST_doc->AddEventHandler(
new BroDocObj($2, reST_doc_comments));
}
| TOK_REDEF TOK_EVENT event_id func_params

View file

@ -80,6 +80,19 @@ static const char* canon_doc_comment(const char* comment)
return ( comment[0] == ' ' ) ? comment + 1 : comment;
}
static std::string canon_doc_func_param(const char* id_start)
{
std::string id_name(id_start, strcspn(id_start, ":"));
const char* comment = id_start + id_name.size() + 1;
std::string doc;
if ( id_name == "Returns" )
doc.append(":returns:").append(comment);
else
doc.append(":param ").append(id_name).append(":").append(comment);
return doc;
}
static ino_t get_inode_num(FILE* f, const char* filename)
{
struct stat b;
@ -155,6 +168,12 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+))
return TOK_POST_DOC;
}
<DOC>##{OWS}{ID}:.* {
const char* id_start = skip_whitespace(yytext + 2);
yylval.str = copy_string(canon_doc_func_param(id_start).c_str());
return TOK_DOC;
}
<DOC>##.* {
if ( yytext[2] != '#' )
{
@ -169,20 +188,6 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+))
// Comment is documenting either a function parameter or return type,
// so appropriate reST markup substitutions are automatically made
// in order to distinguish them from other comments.
const char* id_start = skip_whitespace(yytext + 2);
size_t id_len = strcspn(id_start, ":");
char* id_name = new char[id_len + 1];
strncpy(id_name, id_start, id_len);
id_name[id_len] = '\0';
const char* comment = id_start + id_len + 1;
std::string doc;
if ( streq(id_name, "Returns") )
doc.append(":returns:").append(comment);
else
doc.append(":param ").append(id_name).append(":").append(comment);
if ( ! reST_doc_comments )
reST_doc_comments = new std::list<std::string>();
@ -192,9 +197,8 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+))
// 2) has a blank line between it and non-field-list reST markup,
// which is required for correct HTML rendering by Sphinx
reST_doc_comments->push_back("");
reST_doc_comments->push_back(doc);
delete [] id_name;
const char* id_start = skip_whitespace(yytext + 2);
reST_doc_comments->push_back(canon_doc_func_param(id_start));
}
}

View file

@ -891,7 +891,7 @@ const char* normalize_path(const char* path)
return copy_string(new_path.c_str());
}
// Returns the subpath of the root Bro script install/source directory in
// Returns the subpath of the root Bro script install/source/build directory in
// which the loaded file is located. If it's not under a subpath of that
// directory (e.g. cwd or custom path) then the full path is returned.
void get_script_subpath(const std::string& full_filename, const char** subpath)
@ -909,11 +909,15 @@ void get_script_subpath(const std::string& full_filename, const char** subpath)
// first check if this is some subpath of the installed scripts root path,
// if not check if it's a subpath of the script source root path,
// if neither, will just use the given directory
if ( (p=my_subpath.find(BRO_SCRIPT_INSTALL_PATH)) != std::string::npos )
// then check if it's a subpath of the build directory (where BIF scripts
// will get generated).
// If none of those, will just use the given directory.
if ( (p = my_subpath.find(BRO_SCRIPT_INSTALL_PATH)) != std::string::npos )
my_subpath.erase(0, strlen(BRO_SCRIPT_INSTALL_PATH));
else if ( (p=my_subpath.find(BRO_SCRIPT_SOURCE_PATH)) != std::string::npos )
else if ( (p = my_subpath.find(BRO_SCRIPT_SOURCE_PATH)) != std::string::npos )
my_subpath.erase(0, strlen(BRO_SCRIPT_SOURCE_PATH));
else if ( (p = my_subpath.find(BRO_BUILD_PATH)) != std::string::npos )
my_subpath.erase(0, strlen(BRO_BUILD_PATH));
// if root path found, remove path separators until next path component
if ( p != std::string::npos )

View file

@ -15,20 +15,6 @@ these comments are transferred directly into the auto-generated
.. tip:: You can embed directives and roles within ``##``-stylized comments.
A script's logging information has to be documented manually as minimally
shown below. Note that references may not always be possible (e.g.
anonymous filter functions) and a script may not need to document
each of "columns", "event", "filter" depending on exactly what it's doing.
**Logging Stream ID:** :bro:enum:`Example::EXAMPLE`
:Columns: :bro:type:`Example::Info`
:Event: :bro:id:`Example::log_example`
:Filter: ``example-filter``
uses :bro:id:`Example::filter_func` to determine whether to
exclude the ``ts`` field
:Author: Jon Siwek <jsiwek@ncsa.illinois.edu>
:Imports: :doc:`policy/frameworks/software/vulnerable </scripts/policy/frameworks/software/vulnerable>`
Summary
@ -72,8 +58,6 @@ Events
:bro:id:`Example::log_example`: :bro:type:`event` This is a declaration of an example event that can be used in
logging streams and is raised once for each log entry.
:bro:id:`bro_init`: :bro:type:`event`
================================================= =============================================================
Functions
@ -233,10 +217,6 @@ Events
This is a declaration of an example event that can be used in
logging streams and is raised once for each log entry.
.. bro:id:: bro_init
:Type: :bro:type:`event` ()
Functions
~~~~~~~~~
.. bro:id:: Example::a_function
@ -309,40 +289,3 @@ Filters added::
[ssl] = tcp port 443,
[nntps] = tcp port 562
Private Interface
-----------------
State Variables
~~~~~~~~~~~~~~~
.. bro:id:: Example::example_ports
:Type: :bro:type:`set` [:bro:type:`port`]
:Attributes: :bro:attr:`&redef`
:Default:
::
{
443/tcp,
562/tcp
}
Types
~~~~~
.. bro:type:: Example::PrivateRecord
:Type: :bro:type:`record`
field1: :bro:type:`bool`
field2: :bro:type:`count`
Functions
~~~~~~~~~
.. bro:id:: Example::filter_func
:Type: :bro:type:`function` (rec: :bro:type:`Example::Info`) : :bro:type:`bool`
.. bro:id:: Example::function_without_proto
:Type: :bro:type:`function` (tag: :bro:type:`string`) : :bro:type:`string`

View file

@ -0,0 +1,57 @@
.. Automatically generated. Do not edit.
autogen-reST-func-params.bro
============================
:download:`Original Source File <autogen-reST-func-params.bro>`
Overview
--------
Summary
~~~~~~~
Types
#####
======================================== =
:bro:type:`test_rec`: :bro:type:`record`
======================================== =
Functions
#########
===================================== ======================================
:bro:id:`test_func`: :bro:type:`func` This is a global function declaration.
===================================== ======================================
Public Interface
----------------
Types
~~~~~
.. bro:type:: test_rec
:Type: :bro:type:`record`
field_func: :bro:type:`function` (i: :bro:type:`int`, j: :bro:type:`int`) : :bro:type:`string`
This is a record field function.
:param i: First param.
:param j: Second param.
:returns: A string.
Functions
~~~~~~~~~
.. bro:id:: test_func
:Type: :bro:type:`function` (i: :bro:type:`int`, j: :bro:type:`int`) : :bro:type:`string`
This is a global function declaration.
:param i: First param.
:param j: Second param.
:returns: A string.

View file

@ -0,0 +1,20 @@
# @TEST-EXEC: bro --doc-scripts %INPUT
# @TEST-EXEC: btest-diff autogen-reST-func-params.rst
## This is a global function declaration.
##
## i: First param.
## j: Second param.
##
## Returns: A string.
global test_func: function(i: int, j: int): string;
type test_rec: record {
## This is a record field function.
##
## i: First param.
## j: Second param.
##
## Returns: A string.
field_func: function(i: int, j: int): string;
};