From 0f99956417425ef20e5592781e3b6335ea4f3f37 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 13 Mar 2013 14:36:27 -0400 Subject: [PATCH] Added Exec, Dir, and ActiveHTTP modules. --- scripts/base/init-default.bro | 3 + scripts/base/utils/active-http.bro | 120 +++++++++++++++++ scripts/base/utils/dir.bro | 51 +++++++ scripts/base/utils/exec.bro | 207 +++++++++++++++++++++++++++++ 4 files changed, 381 insertions(+) create mode 100644 scripts/base/utils/active-http.bro create mode 100644 scripts/base/utils/dir.bro create mode 100644 scripts/base/utils/exec.bro diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro index 8b36899f10..9b62c80014 100644 --- a/scripts/base/init-default.bro +++ b/scripts/base/init-default.bro @@ -5,9 +5,12 @@ ##! you actually want. @load base/utils/site +@load base/utils/active-http @load base/utils/addrs @load base/utils/conn-ids +@load base/utils/dir @load base/utils/directions-and-hosts +@load base/utils/exec @load base/utils/files @load base/utils/numbers @load base/utils/paths diff --git a/scripts/base/utils/active-http.bro b/scripts/base/utils/active-http.bro new file mode 100644 index 0000000000..5522cc108a --- /dev/null +++ b/scripts/base/utils/active-http.bro @@ -0,0 +1,120 @@ +##! A module for performing active HTTP requests and +##! getting the reply at runtime. + +@load ./exec + +module ActiveHTTP; + +export { + ## The default timeout for HTTP requests. + const default_max_time = 1min &redef; + + ## The default HTTP method/verb to use for requests. + const default_method = "GET" &redef; + + type Response: record { + ## Numeric response code from the server. + code: count; + ## String response messgae from the server. + msg: string; + ## Full body of the response. + body: string &optional; + ## All headers returned by the server. + headers: table[string] of string &optional; + }; + + type Request: record { + ## The URL being requested. + url: string; + ## The HTTP method/verb to use for the request. + method: string &default=default_method; + ## Data to send to the server in the client body. Keep in + ## mind that you will probably need to set the $method field + ## to "POST" or "PUT". + client_data: string &optional; + ## Arbitrary headers to pass to the server. Some headers + ## will be included by libCurl. + #custom_headers: table[string] of string &optional; + ## Timeout for the request. + max_time: interval &default=default_max_time; + ## Additional curl command line arguments. Be very careful + ## with this option since shell injection could take place + ## if careful handling of untrusted data is not applied. + addl_curl_args: string &optional; + }; + + ## Perform an HTTP request according to the :bro:type:`Request` record. + ## This is an asynchronous function and must be called within a "when" + ## statement. + ## + ## req: A record instance representing all options for an HTTP request. + ## + ## Returns: A record with the full response message. + global request: function(req: ActiveHTTP::Request): ActiveHTTP::Response; +} + +function request2curl(r: Request, bodyfile: string, headersfile: string): string + { + local cmd = fmt("curl -s -g -o \"%s\" -D \"%s\" -X \"%s\"", + str_shell_escape(bodyfile), + str_shell_escape(headersfile), + str_shell_escape(r$method)); + + cmd = fmt("%s -m %.0f", cmd, r$max_time); + + if ( r?$client_data ) + cmd = fmt("%s -d -", cmd); + + if ( r?$addl_curl_args ) + cmd = fmt("%s %s", cmd, r$addl_curl_args); + + cmd = fmt("%s \"%s\"", cmd, str_shell_escape(r$url)); + return cmd; + } + +function request(req: Request): ActiveHTTP::Response + { + local tmpfile = "/tmp/bro-activehttp-" + unique_id(""); + local bodyfile = fmt("%s_body", tmpfile); + local headersfile = fmt("%s_headers", tmpfile); + + local cmd = request2curl(req, bodyfile, headersfile); + local stdin_data = req?$client_data ? req$client_data : ""; + + local resp: Response; + resp$code = 0; + resp$msg = ""; + resp$body = ""; + resp$headers = table(); + return when ( local result = Exec::run([$cmd=cmd, $stdin=stdin_data, $read_files=set(bodyfile, headersfile)]) ) + { + # If there is no response line then nothing else will work either. + if ( ! (result?$files && headersfile in result$files) ) + Reporter::error(fmt("There was a failure when requesting \"%s\" with ActiveHTTP.", req$url)); + + local headers = result$files[headersfile]; + for ( i in headers ) + { + # The reply is the first line. + if ( i == 0 ) + { + local response_line = split_n(headers[0], /[[:blank:]]+/, F, 2); + if ( |response_line| != 3 ) + return resp; + + resp$code = to_count(response_line[2]); + resp$msg = response_line[3]; + resp$body = join_string_vec(result$files[bodyfile], ""); + } + else + { + local line = headers[i]; + local h = split1(line, /:/); + if ( |h| != 2 ) + next; + resp$headers[h[1]] = sub_bytes(h[2], 0, |h[2]|-1); + } + } + return resp; + } + } diff --git a/scripts/base/utils/dir.bro b/scripts/base/utils/dir.bro new file mode 100644 index 0000000000..2ed1c8e6e9 --- /dev/null +++ b/scripts/base/utils/dir.bro @@ -0,0 +1,51 @@ +@load base/utils/exec +@load base/frameworks/reporter +@load base/utils/paths + +module Dir; + +export { + ## Register a directory to monitor with a callback that is called + ## every time a previously unseen file is seen. If a file is deleted + ## and seen to be gone, the file is available for being seen again in + ## the future. + ## + ## dir: The directory to monitor for files. + ## + ## callback: Callback that gets executed with each file name + ## that is found. Filenames are provided with the full path. + global monitor: function(dir: string, callback: function(fname: string)); + + ## The interval this module checks for files in directories when using + ## the :bro:see:`Dir::monitor` function. + const polling_interval = 30sec &redef; +} + +event Dir::monitor_ev(dir: string, last_files: set[string], callback: function(fname: string)) + { + when ( local result = Exec::run([$cmd=fmt("ls \"%s\"", str_shell_escape(dir))]) ) + { + if ( result$exit_code != 0 ) + { + Reporter::warning("Requested monitoring of non-existent directory."); + return; + } + + local current_files: set[string] = set(); + local files = result$stdout; + for ( i in files ) + { + if ( files[i] !in last_files ) + callback(build_path_compressed(dir, files[i])); + add current_files[files[i]]; + } + schedule polling_interval { Dir::monitor_ev(dir, current_files, callback) }; + } + } + +function monitor(dir: string, callback: function(fname: string)) + { + event Dir::monitor_ev(dir, set(), callback); + } + + diff --git a/scripts/base/utils/exec.bro b/scripts/base/utils/exec.bro new file mode 100644 index 0000000000..fe353cf590 --- /dev/null +++ b/scripts/base/utils/exec.bro @@ -0,0 +1,207 @@ +##! A module for executing external command line programs. +##! This requires code that is still in topic branches and +##! definitely won't currently work on any released version of Bro. + +@load base/frameworks/input + +module Exec; + +export { + type Command: record { + ## The command line to execute. + ## Use care to avoid injection attacks! + cmd: string; + ## Provide standard in to the program as a + ## string. + stdin: string &default=""; + ## If additional files are required to be read + ## in as part of the output of the command they + ## can be defined here. + read_files: set[string] &optional; + }; + + type Result: record { + ## Exit code from the program. + exit_code: count &default=0; + ## Each line of standard out. + stdout: vector of string &optional; + ## Each line of standard error. + stderr: vector of string &optional; + ## If additional files were requested to be read in + ## the content of the files will be available here. + files: table[string] of string_vec &optional; + }; + + ## Function for running command line programs and getting + ## output. This is an asynchronous function which is meant + ## to be run with the `when` statement. + ## + ## cmd: The command to run. Use care to avoid injection attacks! + ## + ## returns: A record representing the full results from the + ## external program execution. + global run: function(cmd: Command): Result; +} + +redef record Command += { + # The prefix name for tracking temp files. + prefix_name: string &optional; +}; + +global results: table[string] of Result = table(); +global finished_commands: set[string]; +global tmp_files: set[string] = set(); + +type OneLine: record { line: string; }; + +event Exec::stdout_line(description: Input::EventDescription, tpe: Input::Event, s: string) + { + local name = sub(description$name, /_[^_]*$/, ""); + + local result = results[name]; + if ( ! results[name]?$stdout ) + result$stdout = vector(s); + else + result$stdout[|result$stdout|] = s; + } + +event Exec::stderr_line(description: Input::EventDescription, tpe: Input::Event, s: string) + { + local name = sub(description$name, /_[^_]*$/, ""); + + local result = results[name]; + if ( ! results[name]?$stderr ) + result$stderr = vector(s); + else + result$stderr[|result$stderr|] = s; + } + +event Exec::file_line(description: Input::EventDescription, tpe: Input::Event, s: string) + { + local parts = split1(description$name, /_/); + local name = parts[1]; + local track_file = parts[2]; + + local result = results[name]; + if ( ! result?$files ) + result$files = table(); + + if ( track_file !in result$files ) + result$files[track_file] = vector(s); + else + result$files[track_file][|result$files[track_file]|] = s; + } + +event Exec::cleanup_and_do_callback(name: string) + { + Input::remove(fmt("%s_stdout", name)); + system(fmt("rm %s_stdout", name)); + delete tmp_files[fmt("%s_stdout", name)]; + + Input::remove(fmt("%s_stderr", name)); + system(fmt("rm %s_stderr", name)); + delete tmp_files[fmt("%s_stderr", name)]; + + Input::remove(fmt("%s_done", name)); + system(fmt("rm %s_done", name)); + delete tmp_files[fmt("%s_done", name)]; + + # Indicate to the "when" async watcher that this command is done. + add finished_commands[name]; + } + +event Exec::run_done(description: Input::EventDescription, tpe: Input::Event, s: string) + { + local name = sub(description$name, /_[^_]*$/, ""); + + if ( /^exit_code:/ in s ) + results[name]$exit_code = to_count(split1(s, /:/)[2]); + else if ( s == "done" ) + # Wait one second to allow all threads to read all of their input + # and forward it. + schedule 1sec { Exec::cleanup_and_do_callback(name) }; + } + +event Exec::start_watching_files(cmd: Command) + { + Input::add_event([$source=fmt("%s_done", cmd$prefix_name), + $name=fmt("%s_done", cmd$prefix_name), + $reader=Input::READER_RAW, + $mode=Input::STREAM, + $want_record=F, + $fields=OneLine, + $ev=Exec::run_done]); + + Input::add_event([$source=fmt("%s_stdout", cmd$prefix_name), + $name=fmt("%s_stdout", cmd$prefix_name), + $reader=Input::READER_RAW, + $mode=Input::STREAM, + $want_record=F, + $fields=OneLine, + $ev=Exec::stdout_line]); + + Input::add_event([$source=fmt("%s_stderr", cmd$prefix_name), + $name=fmt("%s_stderr", cmd$prefix_name), + $reader=Input::READER_RAW, + $mode=Input::STREAM, + $want_record=F, + $fields=OneLine, + $ev=Exec::stderr_line]); + + if ( cmd?$read_files ) + { + for ( read_file in cmd$read_files ) + { + Input::add_event([$source=fmt("%s", read_file), + $name=fmt("%s_%s", cmd$prefix_name, read_file), + $reader=Input::READER_RAW, + $mode=Input::STREAM, + $want_record=F, + $fields=OneLine, + $ev=Exec::file_line]); + } + } + } + +function run(cmd: Command): Result + { + cmd$prefix_name = "/tmp/bro-exec-" + unique_id(""); + system(fmt("touch %s_done %s_stdout %s_stderr 2>/dev/null", cmd$prefix_name, cmd$prefix_name, cmd$prefix_name)); + add tmp_files[fmt("%s_done", cmd$prefix_name)]; + add tmp_files[fmt("%s_stdout", cmd$prefix_name)]; + add tmp_files[fmt("%s_stderr", cmd$prefix_name)]; + + if ( cmd?$read_files ) + { + for ( read_file in cmd$read_files ) + { + system(fmt("touch %s 2>/dev/null", read_file)); + add tmp_files[read_file]; + } + } + + piped_exec(fmt("%s 2>> %s_stderr 1>> %s_stdout; echo \"exit_code:${?}\" >> %s_done; echo \"done\" >> %s_done", + cmd$cmd, cmd$prefix_name, cmd$prefix_name, cmd$prefix_name, cmd$prefix_name), + cmd$stdin); + + results[cmd$prefix_name] = []; + + schedule 1msec { Exec::start_watching_files(cmd) }; + + return when ( cmd$prefix_name in finished_commands ) + { + delete finished_commands[cmd$prefix_name]; + local result = results[cmd$prefix_name]; + delete results[cmd$prefix_name]; + return result; + } + } + +event bro_done() + { + # We are punting here and just deleting any files that haven't been processed yet. + for ( fname in tmp_files ) + { + system(fmt("rm \"%s\"", str_shell_escape(fname))); + } + } \ No newline at end of file