mirror of
https://github.com/zeek/zeek.git
synced 2025-10-14 20:48:21 +00:00
Added Exec, Dir, and ActiveHTTP modules.
This commit is contained in:
Seth Hall
parent
b4824f4207
commit
0f99956417
4 changed files with 381 additions and 0 deletions
|
|
@ -5,9 +5,12 @@
|
|||
##! you actually want.
|
||||
|
||||
@load base/utils/site
|
||||
@load base/utils/active-http
|
||||
@load base/utils/addrs
|
||||
@load base/utils/conn-ids
|
||||
@load base/utils/dir
|
||||
@load base/utils/directions-and-hosts
|
||||
@load base/utils/exec
|
||||
@load base/utils/files
|
||||
@load base/utils/numbers
|
||||
@load base/utils/paths
|
||||
|
|
|
|||
120
scripts/base/utils/active-http.bro
Normal file
120
scripts/base/utils/active-http.bro
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
##! A module for performing active HTTP requests and
|
||||
##! getting the reply at runtime.
|
||||
|
||||
@load ./exec
|
||||
|
||||
module ActiveHTTP;
|
||||
|
||||
export {
|
||||
## The default timeout for HTTP requests.
|
||||
const default_max_time = 1min &redef;
|
||||
|
||||
## The default HTTP method/verb to use for requests.
|
||||
const default_method = "GET" &redef;
|
||||
|
||||
type Response: record {
|
||||
## Numeric response code from the server.
|
||||
code: count;
|
||||
## String response messgae from the server.
|
||||
msg: string;
|
||||
## Full body of the response.
|
||||
body: string &optional;
|
||||
## All headers returned by the server.
|
||||
headers: table[string] of string &optional;
|
||||
};
|
||||
|
||||
type Request: record {
|
||||
## The URL being requested.
|
||||
url: string;
|
||||
## The HTTP method/verb to use for the request.
|
||||
method: string &default=default_method;
|
||||
## Data to send to the server in the client body. Keep in
|
||||
## mind that you will probably need to set the $method field
|
||||
## to "POST" or "PUT".
|
||||
client_data: string &optional;
|
||||
## Arbitrary headers to pass to the server. Some headers
|
||||
## will be included by libCurl.
|
||||
#custom_headers: table[string] of string &optional;
|
||||
## Timeout for the request.
|
||||
max_time: interval &default=default_max_time;
|
||||
## Additional curl command line arguments. Be very careful
|
||||
## with this option since shell injection could take place
|
||||
## if careful handling of untrusted data is not applied.
|
||||
addl_curl_args: string &optional;
|
||||
};
|
||||
|
||||
## Perform an HTTP request according to the :bro:type:`Request` record.
|
||||
## This is an asynchronous function and must be called within a "when"
|
||||
## statement.
|
||||
##
|
||||
## req: A record instance representing all options for an HTTP request.
|
||||
##
|
||||
## Returns: A record with the full response message.
|
||||
global request: function(req: ActiveHTTP::Request): ActiveHTTP::Response;
|
||||
}
|
||||
|
||||
function request2curl(r: Request, bodyfile: string, headersfile: string): string
|
||||
{
|
||||
local cmd = fmt("curl -s -g -o \"%s\" -D \"%s\" -X \"%s\"",
|
||||
str_shell_escape(bodyfile),
|
||||
str_shell_escape(headersfile),
|
||||
str_shell_escape(r$method));
|
||||
|
||||
cmd = fmt("%s -m %.0f", cmd, r$max_time);
|
||||
|
||||
if ( r?$client_data )
|
||||
cmd = fmt("%s -d -", cmd);
|
||||
|
||||
if ( r?$addl_curl_args )
|
||||
cmd = fmt("%s %s", cmd, r$addl_curl_args);
|
||||
|
||||
cmd = fmt("%s \"%s\"", cmd, str_shell_escape(r$url));
|
||||
return cmd;
|
||||
}
|
||||
|
||||
function request(req: Request): ActiveHTTP::Response
|
||||
{
|
||||
local tmpfile = "/tmp/bro-activehttp-" + unique_id("");
|
||||
local bodyfile = fmt("%s_body", tmpfile);
|
||||
local headersfile = fmt("%s_headers", tmpfile);
|
||||
|
||||
local cmd = request2curl(req, bodyfile, headersfile);
|
||||
local stdin_data = req?$client_data ? req$client_data : "";
|
||||
|
||||
local resp: Response;
|
||||
resp$code = 0;
|
||||
resp$msg = "";
|
||||
resp$body = "";
|
||||
resp$headers = table();
|
||||
return when ( local result = Exec::run([$cmd=cmd, $stdin=stdin_data, $read_files=set(bodyfile, headersfile)]) )
|
||||
{
|
||||
# If there is no response line then nothing else will work either.
|
||||
if ( ! (result?$files && headersfile in result$files) )
|
||||
Reporter::error(fmt("There was a failure when requesting \"%s\" with ActiveHTTP.", req$url));
|
||||
|
||||
local headers = result$files[headersfile];
|
||||
for ( i in headers )
|
||||
{
|
||||
# The reply is the first line.
|
||||
if ( i == 0 )
|
||||
{
|
||||
local response_line = split_n(headers[0], /[[:blank:]]+/, F, 2);
|
||||
if ( |response_line| != 3 )
|
||||
return resp;
|
||||
|
||||
resp$code = to_count(response_line[2]);
|
||||
resp$msg = response_line[3];
|
||||
resp$body = join_string_vec(result$files[bodyfile], "");
|
||||
}
|
||||
else
|
||||
{
|
||||
local line = headers[i];
|
||||
local h = split1(line, /:/);
|
||||
if ( |h| != 2 )
|
||||
next;
|
||||
resp$headers[h[1]] = sub_bytes(h[2], 0, |h[2]|-1);
|
||||
}
|
||||
}
|
||||
return resp;
|
||||
}
|
||||
}
|
||||
51
scripts/base/utils/dir.bro
Normal file
51
scripts/base/utils/dir.bro
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
@load base/utils/exec
|
||||
@load base/frameworks/reporter
|
||||
@load base/utils/paths
|
||||
|
||||
module Dir;
|
||||
|
||||
export {
|
||||
## Register a directory to monitor with a callback that is called
|
||||
## every time a previously unseen file is seen. If a file is deleted
|
||||
## and seen to be gone, the file is available for being seen again in
|
||||
## the future.
|
||||
##
|
||||
## dir: The directory to monitor for files.
|
||||
##
|
||||
## callback: Callback that gets executed with each file name
|
||||
## that is found. Filenames are provided with the full path.
|
||||
global monitor: function(dir: string, callback: function(fname: string));
|
||||
|
||||
## The interval this module checks for files in directories when using
|
||||
## the :bro:see:`Dir::monitor` function.
|
||||
const polling_interval = 30sec &redef;
|
||||
}
|
||||
|
||||
event Dir::monitor_ev(dir: string, last_files: set[string], callback: function(fname: string))
|
||||
{
|
||||
when ( local result = Exec::run([$cmd=fmt("ls \"%s\"", str_shell_escape(dir))]) )
|
||||
{
|
||||
if ( result$exit_code != 0 )
|
||||
{
|
||||
Reporter::warning("Requested monitoring of non-existent directory.");
|
||||
return;
|
||||
}
|
||||
|
||||
local current_files: set[string] = set();
|
||||
local files = result$stdout;
|
||||
for ( i in files )
|
||||
{
|
||||
if ( files[i] !in last_files )
|
||||
callback(build_path_compressed(dir, files[i]));
|
||||
add current_files[files[i]];
|
||||
}
|
||||
schedule polling_interval { Dir::monitor_ev(dir, current_files, callback) };
|
||||
}
|
||||
}
|
||||
|
||||
function monitor(dir: string, callback: function(fname: string))
|
||||
{
|
||||
event Dir::monitor_ev(dir, set(), callback);
|
||||
}
|
||||
|
||||
|
||||
207
scripts/base/utils/exec.bro
Normal file
207
scripts/base/utils/exec.bro
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
##! A module for executing external command line programs.
|
||||
##! This requires code that is still in topic branches and
|
||||
##! definitely won't currently work on any released version of Bro.
|
||||
|
||||
@load base/frameworks/input
|
||||
|
||||
module Exec;
|
||||
|
||||
export {
|
||||
type Command: record {
|
||||
## The command line to execute.
|
||||
## Use care to avoid injection attacks!
|
||||
cmd: string;
|
||||
## Provide standard in to the program as a
|
||||
## string.
|
||||
stdin: string &default="";
|
||||
## If additional files are required to be read
|
||||
## in as part of the output of the command they
|
||||
## can be defined here.
|
||||
read_files: set[string] &optional;
|
||||
};
|
||||
|
||||
type Result: record {
|
||||
## Exit code from the program.
|
||||
exit_code: count &default=0;
|
||||
## Each line of standard out.
|
||||
stdout: vector of string &optional;
|
||||
## Each line of standard error.
|
||||
stderr: vector of string &optional;
|
||||
## If additional files were requested to be read in
|
||||
## the content of the files will be available here.
|
||||
files: table[string] of string_vec &optional;
|
||||
};
|
||||
|
||||
## Function for running command line programs and getting
|
||||
## output. This is an asynchronous function which is meant
|
||||
## to be run with the `when` statement.
|
||||
##
|
||||
## cmd: The command to run. Use care to avoid injection attacks!
|
||||
##
|
||||
## returns: A record representing the full results from the
|
||||
## external program execution.
|
||||
global run: function(cmd: Command): Result;
|
||||
}
|
||||
|
||||
redef record Command += {
|
||||
# The prefix name for tracking temp files.
|
||||
prefix_name: string &optional;
|
||||
};
|
||||
|
||||
global results: table[string] of Result = table();
|
||||
global finished_commands: set[string];
|
||||
global tmp_files: set[string] = set();
|
||||
|
||||
type OneLine: record { line: string; };
|
||||
|
||||
event Exec::stdout_line(description: Input::EventDescription, tpe: Input::Event, s: string)
|
||||
{
|
||||
local name = sub(description$name, /_[^_]*$/, "");
|
||||
|
||||
local result = results[name];
|
||||
if ( ! results[name]?$stdout )
|
||||
result$stdout = vector(s);
|
||||
else
|
||||
result$stdout[|result$stdout|] = s;
|
||||
}
|
||||
|
||||
event Exec::stderr_line(description: Input::EventDescription, tpe: Input::Event, s: string)
|
||||
{
|
||||
local name = sub(description$name, /_[^_]*$/, "");
|
||||
|
||||
local result = results[name];
|
||||
if ( ! results[name]?$stderr )
|
||||
result$stderr = vector(s);
|
||||
else
|
||||
result$stderr[|result$stderr|] = s;
|
||||
}
|
||||
|
||||
event Exec::file_line(description: Input::EventDescription, tpe: Input::Event, s: string)
|
||||
{
|
||||
local parts = split1(description$name, /_/);
|
||||
local name = parts[1];
|
||||
local track_file = parts[2];
|
||||
|
||||
local result = results[name];
|
||||
if ( ! result?$files )
|
||||
result$files = table();
|
||||
|
||||
if ( track_file !in result$files )
|
||||
result$files[track_file] = vector(s);
|
||||
else
|
||||
result$files[track_file][|result$files[track_file]|] = s;
|
||||
}
|
||||
|
||||
event Exec::cleanup_and_do_callback(name: string)
|
||||
{
|
||||
Input::remove(fmt("%s_stdout", name));
|
||||
system(fmt("rm %s_stdout", name));
|
||||
delete tmp_files[fmt("%s_stdout", name)];
|
||||
|
||||
Input::remove(fmt("%s_stderr", name));
|
||||
system(fmt("rm %s_stderr", name));
|
||||
delete tmp_files[fmt("%s_stderr", name)];
|
||||
|
||||
Input::remove(fmt("%s_done", name));
|
||||
system(fmt("rm %s_done", name));
|
||||
delete tmp_files[fmt("%s_done", name)];
|
||||
|
||||
# Indicate to the "when" async watcher that this command is done.
|
||||
add finished_commands[name];
|
||||
}
|
||||
|
||||
event Exec::run_done(description: Input::EventDescription, tpe: Input::Event, s: string)
|
||||
{
|
||||
local name = sub(description$name, /_[^_]*$/, "");
|
||||
|
||||
if ( /^exit_code:/ in s )
|
||||
results[name]$exit_code = to_count(split1(s, /:/)[2]);
|
||||
else if ( s == "done" )
|
||||
# Wait one second to allow all threads to read all of their input
|
||||
# and forward it.
|
||||
schedule 1sec { Exec::cleanup_and_do_callback(name) };
|
||||
}
|
||||
|
||||
event Exec::start_watching_files(cmd: Command)
|
||||
{
|
||||
Input::add_event([$source=fmt("%s_done", cmd$prefix_name),
|
||||
$name=fmt("%s_done", cmd$prefix_name),
|
||||
$reader=Input::READER_RAW,
|
||||
$mode=Input::STREAM,
|
||||
$want_record=F,
|
||||
$fields=OneLine,
|
||||
$ev=Exec::run_done]);
|
||||
|
||||
Input::add_event([$source=fmt("%s_stdout", cmd$prefix_name),
|
||||
$name=fmt("%s_stdout", cmd$prefix_name),
|
||||
$reader=Input::READER_RAW,
|
||||
$mode=Input::STREAM,
|
||||
$want_record=F,
|
||||
$fields=OneLine,
|
||||
$ev=Exec::stdout_line]);
|
||||
|
||||
Input::add_event([$source=fmt("%s_stderr", cmd$prefix_name),
|
||||
$name=fmt("%s_stderr", cmd$prefix_name),
|
||||
$reader=Input::READER_RAW,
|
||||
$mode=Input::STREAM,
|
||||
$want_record=F,
|
||||
$fields=OneLine,
|
||||
$ev=Exec::stderr_line]);
|
||||
|
||||
if ( cmd?$read_files )
|
||||
{
|
||||
for ( read_file in cmd$read_files )
|
||||
{
|
||||
Input::add_event([$source=fmt("%s", read_file),
|
||||
$name=fmt("%s_%s", cmd$prefix_name, read_file),
|
||||
$reader=Input::READER_RAW,
|
||||
$mode=Input::STREAM,
|
||||
$want_record=F,
|
||||
$fields=OneLine,
|
||||
$ev=Exec::file_line]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function run(cmd: Command): Result
|
||||
{
|
||||
cmd$prefix_name = "/tmp/bro-exec-" + unique_id("");
|
||||
system(fmt("touch %s_done %s_stdout %s_stderr 2>/dev/null", cmd$prefix_name, cmd$prefix_name, cmd$prefix_name));
|
||||
add tmp_files[fmt("%s_done", cmd$prefix_name)];
|
||||
add tmp_files[fmt("%s_stdout", cmd$prefix_name)];
|
||||
add tmp_files[fmt("%s_stderr", cmd$prefix_name)];
|
||||
|
||||
if ( cmd?$read_files )
|
||||
{
|
||||
for ( read_file in cmd$read_files )
|
||||
{
|
||||
system(fmt("touch %s 2>/dev/null", read_file));
|
||||
add tmp_files[read_file];
|
||||
}
|
||||
}
|
||||
|
||||
piped_exec(fmt("%s 2>> %s_stderr 1>> %s_stdout; echo \"exit_code:${?}\" >> %s_done; echo \"done\" >> %s_done",
|
||||
cmd$cmd, cmd$prefix_name, cmd$prefix_name, cmd$prefix_name, cmd$prefix_name),
|
||||
cmd$stdin);
|
||||
|
||||
results[cmd$prefix_name] = [];
|
||||
|
||||
schedule 1msec { Exec::start_watching_files(cmd) };
|
||||
|
||||
return when ( cmd$prefix_name in finished_commands )
|
||||
{
|
||||
delete finished_commands[cmd$prefix_name];
|
||||
local result = results[cmd$prefix_name];
|
||||
delete results[cmd$prefix_name];
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
event bro_done()
|
||||
{
|
||||
# We are punting here and just deleting any files that haven't been processed yet.
|
||||
for ( fname in tmp_files )
|
||||
{
|
||||
system(fmt("rm \"%s\"", str_shell_escape(fname)));
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue