Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-14 04:28:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge remote-tracking branch 'origin/master' into topic/vladg/ssh

Browse source
This commit is contained in:
Vlad Grigorescu 2015-03-17 12:36:30 -04:00
commit 092a78d14b
256 changed files with 11215 additions and 1544 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/3rdparty

@ -1 +1 @@
Subproject commit 7e15efe9d28d46bfa662fcdd1cbb15ce1db285c9
Subproject commit f2e34d731ed29bb993fbb065846faa342a8c824f

20
src/CMakeLists.txt
View file

@ -161,6 +161,14 @@ add_subdirectory(iosource)
add_subdirectory(logging)
add_subdirectory(probabilistic)
if ( ENABLE_BROKER )
add_subdirectory(broker)
else ()
# Just to satisfy coverage unit tests until new Broker-based
# communication is enabled by default.
add_subdirectory(broker-dummy)
endif ()
set(bro_SUBDIRS
# Order is important here.
${bro_PLUGIN_LIBS}
@ -408,6 +416,18 @@ add_dependencies(bro bif_loader_plugins)
# Install *.bif.bro.
install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/base/bif DESTINATION ${BRO_SCRIPT_INSTALL_PATH}/base)
# Create plugin directory at install time.
install(DIRECTORY DESTINATION ${BRO_PLUGIN_INSTALL_PATH})
# Make clean removes the bif directory.
set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${CMAKE_BINARY_DIR}/scripts/base/bif)
# Remove some stale files and scripts that previous Bro versions put in
# place, yet make confuse us now. This makes upgrading easier.
install(CODE "
file(REMOVE_RECURSE
${BRO_SCRIPT_INSTALL_PATH}/base/frameworks/logging/writers/dataseries.bro
${BRO_SCRIPT_INSTALL_PATH}/base/frameworks/logging/writers/elasticsearch.bro
${BRO_SCRIPT_INSTALL_PATH}/policy/tuning/logs-to-elasticsearch.bro
)
")

3
src/Conn.h
View file

@ -263,6 +263,9 @@ public:
void CheckFlowLabel(bool is_orig, uint32 flow_label);
uint32 GetOrigFlowLabel() { return orig_flow_label; }
uint32 GetRespFlowLabel() { return resp_flow_label; }
protected:
Connection() { persistent = 0; }

2
src/DebugLogger.cc
View file

@ -19,7 +19,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = {
{ "logging", 0, false }, {"input", 0, false },
{ "threading", 0, false }, { "file_analysis", 0, false },
{ "plugins", 0, false }, { "broxygen", 0, false },
{ "pktio", 0, false}
{ "pktio", 0, false }, { "broker", 0, false }
};
DebugLogger::DebugLogger(const char* filename)

1
src/DebugLogger.h
View file

@ -32,6 +32,7 @@ enum DebugStream {
DBG_PLUGINS, // Plugin system
DBG_BROXYGEN, // Broxygen
DBG_PKTIO, // Packet sources and dumpers.
DBG_BROKER, // Broker communication
NUM_DBGS // Has to be last
};

52
src/EventHandler.cc
View file

@ -5,6 +5,11 @@
#include "RemoteSerializer.h"
#include "NetVar.h"
#ifdef ENABLE_BROKER
#include "broker/Manager.h"
#include "broker/Data.h"
#endif
EventHandler::EventHandler(const char* arg_name)
{
name = copy_string(arg_name);
@ -26,7 +31,12 @@ EventHandler::operator bool() const
{
return enabled && ((local && local->HasBodies())
|| receivers.length()
|| generate_always);
|| generate_always
#ifdef ENABLE_BROKER
|| ! auto_remote_send.empty()
// TODO: and require a subscriber interested in a topic or unsolicited flags?
#endif
);
}
FuncType* EventHandler::FType()
@ -73,6 +83,46 @@ void EventHandler::Call(val_list* vl, bool no_remote)
SerialInfo info(remote_serializer);
remote_serializer->SendCall(&info, receivers[i], name, vl);
}
#ifdef ENABLE_BROKER
if ( ! auto_remote_send.empty() )
{
// TODO: also short-circuit based on interested subscribers/flags?
broker::message msg;
msg.reserve(vl->length() + 1);
msg.emplace_back(Name());
bool valid_args = true;
for ( auto i = 0; i < vl->length(); ++i )
{
auto opt_data = bro_broker::val_to_data((*vl)[i]);
if ( opt_data )
msg.emplace_back(move(*opt_data));
else
{
valid_args = false;
auto_remote_send.clear();
reporter->Error("failed auto-remote event '%s', disabled",
Name());
break;
}
}
if ( valid_args )
{
for ( auto it = auto_remote_send.begin();
it != auto_remote_send.end(); ++it )
{
if ( std::next(it) == auto_remote_send.end() )
broker_mgr->Event(it->first, move(msg), it->second);
else
broker_mgr->Event(it->first, msg, it->second);
}
}
}
#endif
}
if ( local )

19
src/EventHandler.h
View file

@ -4,7 +4,8 @@
#define EVENTHANDLER
#include <assert.h>
#include <map>
#include <string>
#include "List.h"
#include "BroList.h"
@ -28,6 +29,18 @@ public:
void AddRemoteHandler(SourceID peer);
void RemoveRemoteHandler(SourceID peer);
#ifdef ENABLE_BROKER
void AutoRemote(std::string topic, int flags)
{
auto_remote_send[std::move(topic)] = flags;
}
void AutoRemoteStop(const std::string& topic)
{
auto_remote_send.erase(topic);
}
#endif
void Call(val_list* vl, bool no_remote = false);
// Returns true if there is at least one local or remote handler.
@ -67,6 +80,10 @@ private:
declare(List, SourceID);
typedef List(SourceID) receiver_list;
receiver_list receivers;
#ifdef ENABLE_BROKER
std::map<std::string, int> auto_remote_send; // topic -> flags
#endif
};
// Encapsulates a ptr to an event handler to overload the boolean operator.

66
src/Func.cc
View file

@ -54,6 +54,7 @@ const Expr* calling_expr = 0;
bool did_builtin_init = false;
vector<Func*> Func::unique_ids;
static const std::pair<bool, Val*> empty_hook_result(false, NULL);
Func::Func() : scope(0), type(0)
{
@ -245,20 +246,31 @@ TraversalCode Func::Traverse(TraversalCallback* cb) const
HANDLE_TC_STMT_POST(tc);
}
Val* Func::HandlePluginResult(Val* plugin_result, val_list* args, function_flavor flavor) const
std::pair<bool, Val*> Func::HandlePluginResult(std::pair<bool, Val*> plugin_result, val_list* args, function_flavor flavor) const
{
// Helper function factoring out this code from BroFunc:Call() for better
// readability.
// Helper function factoring out this code from BroFunc:Call() for
// better readability.
if( ! plugin_result.first )
{
if( plugin_result.second )
reporter->InternalError("plugin set processed flag to false but actually returned a value");
// The plugin result hasn't been processed yet (read: fall
// into ::Call method).
return plugin_result;
}
switch ( flavor ) {
case FUNC_FLAVOR_EVENT:
Unref(plugin_result);
plugin_result = 0;
if( plugin_result.second )
reporter->InternalError("plugin returned non-void result for event %s", this->Name());
break;
case FUNC_FLAVOR_HOOK:
if ( plugin_result->Type()->Tag() != TYPE_BOOL )
reporter->InternalError("plugin returned non-bool for hook");
if ( plugin_result.second->Type()->Tag() != TYPE_BOOL )
reporter->InternalError("plugin returned non-bool for hook %s", this->Name());
break;
@ -268,14 +280,14 @@ Val* Func::HandlePluginResult(Val* plugin_result, val_list* args, function_flavo
if ( (! yt) || yt->Tag() == TYPE_VOID )
{
Unref(plugin_result);
plugin_result = 0;
if( plugin_result.second )
reporter->InternalError("plugin returned non-void result for void method %s", this->Name());
}
else
else if ( plugin_result.second && plugin_result.second->Type()->Tag() != yt->Tag() && yt->Tag() != TYPE_ANY)
{
if ( plugin_result->Type()->Tag() != yt->Tag() )
reporter->InternalError("plugin returned wrong type for function call");
reporter->InternalError("plugin returned wrong type (got %d, expecting %d) for %s",
plugin_result.second->Type()->Tag(), yt->Tag(), this->Name());
}
break;
@ -331,10 +343,15 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const
if ( sample_logger )
sample_logger->FunctionSeen(this);
Val* plugin_result = PLUGIN_HOOK_WITH_RESULT(HOOK_CALL_FUNCTION, HookCallFunction(this, args), 0);
std::pair<bool, Val*> plugin_result = PLUGIN_HOOK_WITH_RESULT(HOOK_CALL_FUNCTION, HookCallFunction(this, parent, args), empty_hook_result);
if ( plugin_result )
return HandlePluginResult(plugin_result, args, Flavor());
plugin_result = HandlePluginResult(plugin_result, args, Flavor());
if( plugin_result.first )
{
Val *result = plugin_result.second;
return result;
}
if ( bodies.empty() )
{
@ -425,11 +442,11 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const
// Warn if the function returns something, but we returned from
// the function without an explicit return, or without a value.
else if ( FType()->YieldType() && FType()->YieldType()->Tag() != TYPE_VOID &&
(flow != FLOW_RETURN /* we fell off the end */ ||
! result /* explicit return with no result */) &&
! f->HasDelayed() )
(flow != FLOW_RETURN /* we fell off the end */ ||
! result /* explicit return with no result */) &&
! f->HasDelayed() )
reporter->Warning("non-void function returns without a value: %s",
Name());
Name());
if ( result && g_trace_state.DoTrace() )
{
@ -548,10 +565,15 @@ Val* BuiltinFunc::Call(val_list* args, Frame* parent) const
if ( sample_logger )
sample_logger->FunctionSeen(this);
Val* plugin_result = PLUGIN_HOOK_WITH_RESULT(HOOK_CALL_FUNCTION, HookCallFunction(this, args), 0);
std::pair<bool, Val*> plugin_result = PLUGIN_HOOK_WITH_RESULT(HOOK_CALL_FUNCTION, HookCallFunction(this, parent, args), empty_hook_result);
if ( plugin_result )
return HandlePluginResult(plugin_result, args, FUNC_FLAVOR_FUNCTION);
plugin_result = HandlePluginResult(plugin_result, args, FUNC_FLAVOR_FUNCTION);
if ( plugin_result.first )
{
Val *result = plugin_result.second;
return result;
}
if ( g_trace_state.DoTrace() )
{

4
src/Func.h
View file

@ -3,6 +3,8 @@
#ifndef func_h
#define func_h
#include <utility>
#include "BroList.h"
#include "Obj.h"
#include "Debug.h"
@ -71,7 +73,7 @@ protected:
Func();
// Helper function for handling result of plugin hook.
Val* HandlePluginResult(Val* plugin_result, val_list* args, function_flavor flavor) const;
std::pair<bool, Val*> HandlePluginResult(std::pair<bool, Val*> plugin_result, val_list* args, function_flavor flavor) const;
DECLARE_ABSTRACT_SERIAL(Func);

13
src/Net.cc
View file

@ -34,6 +34,10 @@
#include "iosource/PktDumper.h"
#include "plugin/Manager.h"
#ifdef ENABLE_BROKER
#include "broker/Manager.h"
#endif
extern "C" {
#include "setsignal.h"
};
@ -315,6 +319,11 @@ void net_run()
}
#endif
current_iosrc = src;
bool communication_enabled = using_communication;
#ifdef ENABLE_BROKER
communication_enabled |= broker_mgr->Enabled();
#endif
if ( src )
src->Process(); // which will call net_packet_dispatch()
@ -332,7 +341,7 @@ void net_run()
}
}
else if ( (have_pending_timers || using_communication) &&
else if ( (have_pending_timers || communication_enabled) &&
! pseudo_realtime )
{
// Take advantage of the lull to get up to
@ -347,7 +356,7 @@ void net_run()
// us a lot of idle time, but doesn't delay near-term
// timers too much. (Delaying them somewhat is okay,
// since Bro timers are not high-precision anyway.)
if ( ! using_communication )
if ( ! communication_enabled )
usleep(100000);
else
usleep(1000);

13
src/Reporter.cc
View file

@ -123,6 +123,19 @@ void Reporter::ExprRuntimeError(const Expr* expr, const char* fmt, ...)
throw InterpreterException();
}
void Reporter::RuntimeError(const Location* location, const char* fmt, ...)
{
++errors;
PushLocation(location);
va_list ap;
va_start(ap, fmt);
FILE* out = errors_to_stderr ? stderr : 0;
DoLog("runtime error", reporter_error, out, 0, 0, true, true, "", fmt, ap);
va_end(ap);
PopLocation();
throw InterpreterException();
}
void Reporter::InternalError(const char* fmt, ...)
{
va_list ap;

4
src/Reporter.h
View file

@ -73,6 +73,10 @@ public:
// function will not return but raise an InterpreterException.
void ExprRuntimeError(const Expr* expr, const char* fmt, ...);
// Report a runtime error in evaluating a Bro script expression. This
// function will not return but raise an InterpreterException.
void RuntimeError(const Location* location, const char* fmt, ...);
// Report a traffic weirdness, i.e., an unexpected protocol situation
// that may lead to incorrectly processing a connnection.
void Weird(const char* name); // Raises net_weird().

3
src/SerialTypes.h
View file

@ -113,6 +113,8 @@ SERIAL_VAL(TOPK_VAL, 20)
SERIAL_VAL(BLOOMFILTER_VAL, 21)
SERIAL_VAL(CARDINALITY_VAL, 22)
SERIAL_VAL(X509_VAL, 23)
SERIAL_VAL(COMM_STORE_HANDLE_VAL, 24)
SERIAL_VAL(COMM_DATA_VAL, 25)
#define SERIAL_EXPR(name, val) SERIAL_CONST(name, val, EXPR)
SERIAL_EXPR(EXPR, 1)
@ -181,6 +183,7 @@ SERIAL_STMT(INIT_STMT, 17)
SERIAL_STMT(NULL_STMT, 18)
SERIAL_STMT(WHEN_STMT, 19)
SERIAL_STMT(FALLTHROUGH_STMT, 20)
SERIAL_STMT(WHILE_STMT, 21)
#define SERIAL_TYPE(name, val) SERIAL_CONST(name, val, BRO_TYPE)
SERIAL_TYPE(BRO_TYPE, 1)

6
src/Sessions.cc
View file

@ -466,6 +466,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
id.src_addr = ip_hdr->SrcAddr();
id.dst_addr = ip_hdr->DstAddr();
Dictionary* d = 0;
BifEnum::Tunnel::Type tunnel_type = BifEnum::Tunnel::IP;
switch ( proto ) {
case IPPROTO_TCP:
@ -606,6 +607,8 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
// Treat GRE tunnel like IP tunnels, fallthrough to logic below now
// that GRE header is stripped and only payload packet remains.
// The only thing different is the tunnel type enum value to use.
tunnel_type = BifEnum::Tunnel::GRE;
}
case IPPROTO_IPV4:
@ -653,7 +656,8 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
if ( it == ip_tunnels.end() )
{
EncapsulatingConn ec(ip_hdr->SrcAddr(), ip_hdr->DstAddr());
EncapsulatingConn ec(ip_hdr->SrcAddr(), ip_hdr->DstAddr(),
tunnel_type);
ip_tunnels[tunnel_idx] = TunnelActivity(ec, network_time);
timer_mgr->Add(new IPTunnelTimer(network_time, tunnel_idx));
}

24
src/Stats.cc
View file

@ -10,6 +10,10 @@
#include "Trigger.h"
#include "threading/Manager.h"
#ifdef ENABLE_BROKER
#include "broker/Manager.h"
#endif
int killed_by_inactivity = 0;
uint64 tot_ack_events = 0;
@ -222,6 +226,26 @@ void ProfileLogger::Log()
));
}
#ifdef ENABLE_BROKER
auto cs = broker_mgr->ConsumeStatistics();
file->Write(fmt("%0.6f Comm: peers=%zu stores=%zu "
"store_queries=%zu store_responses=%zu "
"outgoing_conn_status=%zu incoming_conn_status=%zu "
"reports=%zu\n",
network_time, cs.outgoing_peer_count, cs.data_store_count,
cs.pending_query_count, cs.response_count,
cs.outgoing_conn_status_count, cs.incoming_conn_status_count,
cs.report_count));
for ( const auto& s : cs.print_count )
file->Write(fmt(" %-25s prints dequeued=%zu\n", s.first.data(), s.second));
for ( const auto& s : cs.event_count )
file->Write(fmt(" %-25s events dequeued=%zu\n", s.first.data(), s.second));
for ( const auto& s : cs.log_count )
file->Write(fmt(" %-25s logs dequeued=%zu\n", s.first.data(), s.second));
#endif
// Script-level state.
unsigned int size, mem = 0;
PDict(ID)* globals = global_scope()->Vars();

122
src/Stmt.cc
View file

@ -23,7 +23,7 @@ const char* stmt_name(BroStmtTag t)
"print", "event", "expr", "if", "when", "switch",
"for", "next", "break", "return", "add", "delete",
"list", "bodylist",
"<init>", "fallthrough",
"<init>", "fallthrough", "while",
"null",
};
@ -1127,6 +1127,126 @@ bool EventStmt::DoUnserialize(UnserialInfo* info)
return event_expr != 0;
}
WhileStmt::WhileStmt(Expr* arg_loop_condition, Stmt* arg_body)
: loop_condition(arg_loop_condition), body(arg_body)
{
if ( ! loop_condition->IsError() &&
! IsBool(loop_condition->Type()->Tag()) )
loop_condition->Error("while conditional must be boolean");
}
WhileStmt::~WhileStmt()
{
Unref(loop_condition);
Unref(body);
}
int WhileStmt::IsPure() const
{
return loop_condition->IsPure() && body->IsPure();
}
void WhileStmt::Describe(ODesc* d) const
{
Stmt::Describe(d);
if ( d->IsReadable() )
d->Add("(");
loop_condition->Describe(d);
if ( d->IsReadable() )
d->Add(")");
d->SP();
d->PushIndent();
body->AccessStats(d);
body->Describe(d);
d->PopIndent();
}
TraversalCode WhileStmt::Traverse(TraversalCallback* cb) const
{
TraversalCode tc = cb->PreStmt(this);
HANDLE_TC_STMT_PRE(tc);
tc = loop_condition->Traverse(cb);
HANDLE_TC_STMT_PRE(tc);
tc = body->Traverse(cb);
HANDLE_TC_STMT_PRE(tc);
tc = cb->PostStmt(this);
HANDLE_TC_STMT_POST(tc);
}
Val* WhileStmt::Exec(Frame* f, stmt_flow_type& flow) const
{
RegisterAccess();
flow = FLOW_NEXT;
Val* rval = 0;
for ( ; ; )
{
Val* cond = loop_condition->Eval(f);
if ( ! cond )
break;
bool cont = cond->AsBool();
Unref(cond);
if ( ! cont )
break;
flow = FLOW_NEXT;
rval = body->Exec(f, flow);
if ( flow == FLOW_BREAK || flow == FLOW_RETURN )
break;
}
if ( flow == FLOW_LOOP || flow == FLOW_BREAK )
flow = FLOW_NEXT;
return rval;
}
Stmt* WhileStmt::Simplify()
{
loop_condition = simplify_expr(loop_condition, SIMPLIFY_GENERAL);
if ( loop_condition->IsConst() && loop_condition->IsZero() )
return new NullStmt();
body = simplify_stmt(body);
return this;
}
IMPLEMENT_SERIAL(WhileStmt, SER_WHILE_STMT);
bool WhileStmt::DoSerialize(SerialInfo* info) const
{
DO_SERIALIZE(SER_WHILE_STMT, Stmt);
if ( ! loop_condition->Serialize(info) )
return false;
return body->Serialize(info);
}
bool WhileStmt::DoUnserialize(UnserialInfo* info)
{
DO_UNSERIALIZE(Stmt);
loop_condition = Expr::Unserialize(info);
if ( ! loop_condition )
return false;
body = Stmt::Unserialize(info);
return body != 0;
}
ForStmt::ForStmt(id_list* arg_loop_vars, Expr* loop_expr)
: ExprStmt(STMT_FOR, loop_expr)
{

27
src/Stmt.h
View file

@ -310,6 +310,33 @@ protected:
EventExpr* event_expr;
};
class WhileStmt : public Stmt {
public:
WhileStmt(Expr* loop_condition, Stmt* body);
~WhileStmt();
int IsPure() const;
void Describe(ODesc* d) const;
TraversalCode Traverse(TraversalCallback* cb) const;
protected:
friend class Stmt;
WhileStmt()
{ loop_condition = 0; body = 0; }
Val* Exec(Frame* f, stmt_flow_type& flow) const;
Stmt* Simplify();
DECLARE_SERIAL(WhileStmt);
Expr* loop_condition;
Stmt* body;
};
class ForStmt : public ExprStmt {
public:
ForStmt(id_list* loop_vars, Expr* loop_expr);

1
src/StmtEnums.h
View file

@ -17,6 +17,7 @@ typedef enum {
STMT_LIST, STMT_EVENT_BODY_LIST,
STMT_INIT,
STMT_FALLTHROUGH,
STMT_WHILE,
STMT_NULL
#define NUM_STMTS (int(STMT_NULL) + 1)
} BroStmtTag;

12
src/Trigger.cc
View file

@ -112,6 +112,7 @@ Trigger::Trigger(Expr* arg_cond, Stmt* arg_body, Stmt* arg_timeout_stmts,
attached = 0;
is_return = arg_is_return;
location = arg_location;
timeout_value = -1;
++total_triggers;
@ -133,17 +134,22 @@ Trigger::Trigger(Expr* arg_cond, Stmt* arg_body, Stmt* arg_timeout_stmts,
Val* timeout_val = arg_timeout ? arg_timeout->Eval(arg_frame) : 0;
if ( timeout_val )
{
Unref(timeout_val);
timeout_value = timeout_val->AsInterval();
}
// Make sure we don't get deleted if somebody calls a method like
// Timeout() while evaluating the trigger.
Ref(this);
if ( ! Eval() && timeout_val )
if ( ! Eval() && timeout_value >= 0 )
{
timer = new TriggerTimer(timeout_val->AsInterval(), this);
timer = new TriggerTimer(timeout_value, this);
timer_mgr->Add(timer);
}
Unref(timeout_val);
Unref(this);
}

7
src/Trigger.h
View file

@ -32,6 +32,10 @@ public:
// Executes timeout code and deletes the object.
void Timeout();
// Return the timeout interval (negative if none was specified).
double TimeoutValue() const
{ return timeout_value; }
// Called if another entity needs to complete its operations first
// in any case before this trigger can proceed.
void Hold() { delayed = true; }
@ -51,6 +55,8 @@ public:
// may not immediately delete it as other references may still exist.
void Disable();
bool Disabled() const { return disabled; }
virtual void Describe(ODesc* d) const { d->Add("<trigger>"); }
// Overidden from Notifier. We queue the trigger and evaluate it
@ -87,6 +93,7 @@ private:
Stmt* body;
Stmt* timeout_stmts;
Expr* timeout;
double timeout_value;
Frame* frame;
bool is_return;
const Location* location;

9
src/TunnelEncapsulation.h
View file

@ -37,10 +37,12 @@ public:
*
* @param s The tunnel source address, likely taken from an IP header.
* @param d The tunnel destination address, likely taken from an IP header.
* @param t The type of IP tunnel.
*/
EncapsulatingConn(const IPAddr& s, const IPAddr& d)
EncapsulatingConn(const IPAddr& s, const IPAddr& d,
BifEnum::Tunnel::Type t = BifEnum::Tunnel::IP)
: src_addr(s), dst_addr(d), src_port(0), dst_port(0),
proto(TRANSPORT_UNKNOWN), type(BifEnum::Tunnel::IP),
proto(TRANSPORT_UNKNOWN), type(t),
uid(Bro::UID(bits_per_uid))
{
}
@ -85,7 +87,8 @@ public:
if ( ec1.type != ec2.type )
return false;
if ( ec1.type == BifEnum::Tunnel::IP )
if ( ec1.type == BifEnum::Tunnel::IP ||
ec1.type == BifEnum::Tunnel::GRE )
// Reversing endpoints is still same tunnel.
return ec1.uid == ec2.uid && ec1.proto == ec2.proto &&
((ec1.src_addr == ec2.src_addr && ec1.dst_addr == ec2.dst_addr) ||

3
src/analyzer/protocol/socks/SOCKS.cc
View file

@ -57,8 +57,7 @@ void SOCKS_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
// with the rest of the conneciton.
//
// Note that we assume that no payload data arrives before both endpoints
// are done with there part of the SOCKS protocol.
// are done with their part of the SOCKS protocol.
if ( ! pia )
{
pia = new pia::PIA_TCP(Conn());

16
src/analyzer/protocol/socks/events.bif
View file

@ -27,3 +27,19 @@ event socks_request%(c: connection, version: count, request_type: count, sa: SOC
## p: The destination port for the proxied traffic.
event socks_reply%(c: connection, version: count, reply: count, sa: SOCKS::Address, p: port%);
## Generated when a SOCKS client performs username and password based login.
##
## c: The parent connection of the proxy.
##
## user: The given username.
##
## password: The given password.
event socks_login_userpass_request%(c: connection, user: string, password: string%);
## Generated when a SOCKS server replies to a username/password login attempt.
##
## c: The parent connection of the proxy.
##
## code: The response code for the attempted login.
event socks_login_userpass_reply%(c: connection, code: count%);

50
src/analyzer/protocol/socks/socks-analyzer.pac
View file

@ -148,6 +148,37 @@ refine connection SOCKS_Conn += {
return true;
%}
function socks5_auth_request_userpass(request: SOCKS5_Auth_Request_UserPass_v1): bool
%{
StringVal* user = new StringVal(${request.username}.length(), (const char*) ${request.username}.begin());
StringVal* pass = new StringVal(${request.password}.length(), (const char*) ${request.password}.begin());
BifEvent::generate_socks_login_userpass_request(bro_analyzer(),
bro_analyzer()->Conn(),
user, pass);
return true;
%}
function socks5_unsupported_authentication_method(auth_method: uint8): bool
%{
reporter->Weird(bro_analyzer()->Conn(), fmt("socks5_unsupported_authentication_method_%d", auth_method));
return true;
%}
function socks5_unsupported_authentication_version(auth_method: uint8, version: uint8): bool
%{
reporter->Weird(bro_analyzer()->Conn(), fmt("socks5_unsupported_authentication_%d_%d", auth_method, version));
return true;
%}
function socks5_auth_reply_userpass(reply: SOCKS5_Auth_Reply_UserPass_v1): bool
%{
BifEvent::generate_socks_login_userpass_reply(bro_analyzer(),
bro_analyzer()->Conn(),
${reply.code});
return true;
%}
function version_error(version: uint8): bool
%{
bro_analyzer()->ProtocolViolation(fmt("unsupported/unknown SOCKS version %d", version));
@ -176,3 +207,22 @@ refine typeattr SOCKS5_Request += &let {
refine typeattr SOCKS5_Reply += &let {
proc: bool = $context.connection.socks5_reply(this);
};
refine typeattr SOCKS5_Auth_Negotiation_Reply += &let {
};
refine typeattr SOCKS5_Auth_Request_UserPass_v1 += &let {
proc: bool = $context.connection.socks5_auth_request_userpass(this);
};
refine typeattr SOCKS5_Auth_Reply_UserPass_v1 += &let {
proc: bool = $context.connection.socks5_auth_reply_userpass(this);
};
refine typeattr SOCKS5_Unsupported_Authentication_Method += &let {
proc: bool = $context.connection.socks5_unsupported_authentication_method($context.connection.v5_auth_method());
};
refine typeattr SOCKS5_Unsupported_Authentication_Version += &let {
proc: bool = $context.connection.socks5_unsupported_authentication_version($context.connection.v5_auth_method(), version);
};

102
src/analyzer/protocol/socks/socks-protocol.pac
View file

@ -1,10 +1,15 @@
type SOCKS_Message(is_orig: bool) = case $context.connection.v5_in_auth_sub_negotiation() of {
true -> auth: SOCKS5_Auth_Message(is_orig);
false -> msg: SOCKS_Version(is_orig);
};
type SOCKS_Version(is_orig: bool) = record {
version: uint8;
msg: case version of {
4 -> socks4_msg: SOCKS4_Message(is_orig);
5 -> socks5_msg: SOCKS5_Message(is_orig);
default -> socks_msg_fail: SOCKS_Version_Error(version);
4 -> socks4_msg: SOCKS4_Message(is_orig);
5 -> socks5_msg: SOCKS5_Message(is_orig);
default -> socks_msg_fail: SOCKS_Version_Error(version);
};
};
@ -14,10 +19,11 @@ type SOCKS_Version_Error(version: uint8) = record {
# SOCKS5 Implementation
type SOCKS5_Message(is_orig: bool) = case $context.connection.v5_past_authentication() of {
true -> msg: SOCKS5_Real_Message(is_orig);
false -> auth: SOCKS5_Auth_Negotiation(is_orig);
true -> msg: SOCKS5_Real_Message(is_orig);
};
type SOCKS5_Auth_Negotiation(is_orig: bool) = case is_orig of {
true -> req: SOCKS5_Auth_Negotiation_Request;
false -> rep: SOCKS5_Auth_Negotiation_Reply;
@ -31,7 +37,61 @@ type SOCKS5_Auth_Negotiation_Request = record {
type SOCKS5_Auth_Negotiation_Reply = record {
selected_auth_method: uint8;
} &let {
in_auth_sub_neg = $context.connection.set_v5_in_auth_sub_negotiation(selected_auth_method == 0 || selected_auth_method == 0xff ? false : true);
past_auth = $context.connection.set_v5_past_authentication();
set_auth = $context.connection.set_v5_auth_method(selected_auth_method);
};
type SOCKS5_Auth_Message(is_orig: bool) = case is_orig of {
true -> req: SOCKS5_Auth_Request;
false -> rep: SOCKS5_Auth_Reply;
};
type SOCKS5_Auth_Request = case $context.connection.v5_auth_method() of {
0x02 -> userpass : SOCKS5_Auth_Request_UserPass;
default -> unsupported : SOCKS5_Unsupported_Authentication_Method;
};
type SOCKS5_Unsupported_Authentication_Method = record {
crap: bytestring &restofdata;
};
type SOCKS5_Unsupported_Authentication_Version(version: uint8) = record {
crap: bytestring &restofdata;
};
type SOCKS5_Auth_Request_UserPass = record {
version: uint8;
msg: case version of {
1 -> v1: SOCKS5_Auth_Request_UserPass_v1;
default -> unsupported: SOCKS5_Unsupported_Authentication_Version(version);
};
};
type SOCKS5_Auth_Request_UserPass_v1 = record {
ulen : uint8;
username : bytestring &length=ulen;
plen : uint8;
password : bytestring &length=plen;
};
type SOCKS5_Auth_Reply = case $context.connection.v5_auth_method() of {
0x02 -> userpass : SOCKS5_Auth_Reply_UserPass;
default -> unsupported : SOCKS5_Unsupported_Authentication_Method;
} &let {
in_auth_sub_neg = $context.connection.set_v5_in_auth_sub_negotiation(false);
};
type SOCKS5_Auth_Reply_UserPass = record {
version: uint8;
msg: case version of {
1 -> v1: SOCKS5_Auth_Reply_UserPass_v1;
default -> unsupported: SOCKS5_Unsupported_Authentication_Version(version);
};
};
type SOCKS5_Auth_Reply_UserPass_v1 = record {
code : uint8;
};
type SOCKS5_Real_Message(is_orig: bool) = case is_orig of {
@ -55,10 +115,10 @@ type SOCKS5_Address = record {
} &byteorder = bigendian;
type SOCKS5_Request = record {
command: uint8;
reserved: uint8;
remote_name: SOCKS5_Address;
port: uint16;
command : uint8;
reserved : uint8;
remote_name : SOCKS5_Address;
port : uint16;
} &byteorder = bigendian;
type SOCKS5_Reply = record {
@ -98,13 +158,28 @@ type SOCKS4_Reply = record {
refine connection SOCKS_Conn += {
%member{
bool v5_in_auth_sub_negotiation_;
bool v5_authenticated_;
uint8 selected_auth_method_;
%}
%init{
v5_in_auth_sub_negotiation_ = false;
v5_authenticated_ = false;
selected_auth_method_ = 255;
%}
function v5_in_auth_sub_negotiation(): bool
%{
return v5_in_auth_sub_negotiation_;
%}
function set_v5_in_auth_sub_negotiation(b: bool): bool
%{
v5_in_auth_sub_negotiation_ = b;
return true;
%}
function v5_past_authentication(): bool
%{
return v5_authenticated_;
@ -115,5 +190,16 @@ refine connection SOCKS_Conn += {
v5_authenticated_ = true;
return true;
%}
function set_v5_auth_method(method: uint8): bool
%{
selected_auth_method_ = method;
return true;
%}
function v5_auth_method(): uint8
%{
return selected_auth_method_;
%}
};

2
src/analyzer/protocol/socks/socks.pac
View file

@ -20,7 +20,7 @@ connection SOCKS_Conn(bro_analyzer: BroAnalyzer) {
%include socks-protocol.pac
flow SOCKS_Flow(is_orig: bool) {
datagram = SOCKS_Version(is_orig) withcontext(connection, this);
datagram = SOCKS_Message(is_orig) withcontext(connection, this);
};
%include socks-analyzer.pac

2
src/analyzer/protocol/ssl/ssl-analyzer.pac
View file

@ -207,7 +207,7 @@ refine connection SSL_Conn += {
{
// This should be impossible due to the binpac parser
// and protocol description
bro_analyzer()->ProtocolViolation(fmt("Impossible extension length: %lu", length));
bro_analyzer()->ProtocolViolation(fmt("Impossible extension length: %zu", length));
bro_analyzer()->SetSkip(true);
return true;
}

3
src/bro.bif
View file

@ -1675,6 +1675,7 @@ function net_stats%(%): NetStats
unsigned int recv = 0;
unsigned int drop = 0;
unsigned int link = 0;
unsigned int bytes_recv = 0;
const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
@ -1688,12 +1689,14 @@ function net_stats%(%): NetStats
recv += stat.received;
drop += stat.dropped;
link += stat.link;
bytes_recv += stat.bytes_received;
}
RecordVal* ns = new RecordVal(net_stats);
ns->Assign(0, new Val(recv, TYPE_COUNT));
ns->Assign(1, new Val(drop, TYPE_COUNT));
ns->Assign(2, new Val(link, TYPE_COUNT));
ns->Assign(3, new Val(bytes_recv, TYPE_COUNT));
return ns;
%}

13
src/broker-dummy/CMakeLists.txt Normal file
View file

@ -0,0 +1,13 @@
# Placeholder for Broker-based communication functionality, not enabled
# by default. This helps satisfy coverage unit tests pass regardless of
# whether Broker is enabled or not.
include(BroSubdir)
bif_target(comm.bif)
bif_target(data.bif)
bif_target(messaging.bif)
bif_target(store.bif)
bro_add_subdir_library(broker_dummy ${BIF_OUTPUT_CC})
add_dependencies(bro_broker_dummy generate_outputs)

3
src/broker-dummy/comm.bif Normal file
View file

@ -0,0 +1,3 @@
##! Placeholder for Broker-based communication functionality, not enabled
##! by default.

3
src/broker-dummy/data.bif Normal file
View file

@ -0,0 +1,3 @@
##! Placeholder for Broker-based communication functionality, not enabled
##! by default

3
src/broker-dummy/messaging.bif Normal file
View file

@ -0,0 +1,3 @@
##! Placeholder for Broker-based communication functionality, not enabled
##! by default

3
src/broker-dummy/store.bif Normal file
View file

@ -0,0 +1,3 @@
##! Placeholder for Broker-based communication functionality, not enabled
##! by default

28
src/broker/CMakeLists.txt Normal file
View file

@ -0,0 +1,28 @@
include(BroSubdir)
include_directories(BEFORE
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR}
)
if ( ROCKSDB_INCLUDE_DIR )
add_definitions(-DHAVE_ROCKSDB)
include_directories(BEFORE ${ROCKSDB_INCLUDE_DIR})
endif ()
include_directories(BEFORE ${LIBCAF_INCLUDE_DIR_CORE})
include_directories(BEFORE ${LIBCAF_INCLUDE_DIR_IO})
set(comm_SRCS
Data.cc
Manager.cc
Store.cc
)
bif_target(comm.bif)
bif_target(data.bif)
bif_target(messaging.bif)
bif_target(store.bif)
bro_add_subdir_library(brokercomm ${comm_SRCS} ${BIF_OUTPUT_CC})
add_dependencies(bro_brokercomm generate_outputs)

708
src/broker/Data.cc Normal file
View file

@ -0,0 +1,708 @@
#include "Data.h"
#include "broker/data.bif.h"
#include <caf/binary_serializer.hpp>
#include <caf/binary_deserializer.hpp>
using namespace std;
OpaqueType* bro_broker::opaque_of_data_type;
OpaqueType* bro_broker::opaque_of_set_iterator;
OpaqueType* bro_broker::opaque_of_table_iterator;
OpaqueType* bro_broker::opaque_of_vector_iterator;
OpaqueType* bro_broker::opaque_of_record_iterator;
static broker::port::protocol to_broker_port_proto(TransportProto tp)
{
switch ( tp ) {
case TRANSPORT_TCP:
return broker::port::protocol::tcp;
case TRANSPORT_UDP:
return broker::port::protocol::udp;
case TRANSPORT_ICMP:
return broker::port::protocol::icmp;
case TRANSPORT_UNKNOWN:
default:
return broker::port::protocol::unknown;
}
}
TransportProto bro_broker::to_bro_port_proto(broker::port::protocol tp)
{
switch ( tp ) {
case broker::port::protocol::tcp:
return TRANSPORT_TCP;
case broker::port::protocol::udp:
return TRANSPORT_UDP;
case broker::port::protocol::icmp:
return TRANSPORT_ICMP;
case broker::port::protocol::unknown:
default:
return TRANSPORT_UNKNOWN;
}
}
struct val_converter {
using result_type = Val*;
BroType* type;
bool require_log_attr;
result_type operator()(bool a)
{
if ( type->Tag() == TYPE_BOOL )
return new Val(a, TYPE_BOOL);
return nullptr;
}
result_type operator()(uint64_t a)
{
if ( type->Tag() == TYPE_COUNT )
return new Val(a, TYPE_COUNT);
if ( type->Tag() == TYPE_COUNTER )
return new Val(a, TYPE_COUNTER);
return nullptr;
}
result_type operator()(int64_t a)
{
if ( type->Tag() == TYPE_INT )
return new Val(a, TYPE_INT);
return nullptr;
}
result_type operator()(double a)
{
if ( type->Tag() == TYPE_DOUBLE )
return new Val(a, TYPE_DOUBLE);
return nullptr;
}
result_type operator()(std::string& a)
{
switch ( type->Tag() ) {
case TYPE_STRING:
return new StringVal(a.size(), a.data());
case TYPE_FILE:
{
auto file = BroFile::GetFile(a.data());
if ( file )
{
Ref(file);
return new Val(file);
}
return nullptr;
}
case TYPE_FUNC:
{
auto id = lookup_ID(a.data(), GLOBAL_MODULE_NAME);
auto rval = id ? id->ID_Val() : nullptr;
Unref(id);
if ( rval && rval->Type()->Tag() == TYPE_FUNC )
return rval;
return nullptr;
}
default:
return nullptr;
}
}
result_type operator()(broker::address& a)
{
if ( type->Tag() == TYPE_ADDR )
{
auto bits = reinterpret_cast<const in6_addr*>(&a.bytes());
return new AddrVal(IPAddr(*bits));
}
return nullptr;
}
result_type operator()(broker::subnet& a)
{
if ( type->Tag() == TYPE_SUBNET )
{
auto bits = reinterpret_cast<const in6_addr*>(&a.network().bytes());
return new SubNetVal(IPPrefix(IPAddr(*bits), a.length()));
}
return nullptr;
}
result_type operator()(broker::port& a)
{
if ( type->Tag() == TYPE_PORT )
return new PortVal(a.number(), bro_broker::to_bro_port_proto(a.type()));
return nullptr;
}
result_type operator()(broker::time_point& a)
{
if ( type->Tag() == TYPE_TIME )
return new Val(a.value, TYPE_TIME);
return nullptr;
}
result_type operator()(broker::time_duration& a)
{
if ( type->Tag() == TYPE_INTERVAL )
return new Val(a.value, TYPE_INTERVAL);
return nullptr;
}
result_type operator()(broker::enum_value& a)
{
if ( type->Tag() == TYPE_ENUM )
{
auto etype = type->AsEnumType();
auto i = etype->Lookup(GLOBAL_MODULE_NAME, a.name.data());
if ( i == -1 )
return nullptr;
return new EnumVal(i, etype);
}
return nullptr;
}
result_type operator()(broker::set& a)
{
if ( ! type->IsSet() )
return nullptr;
auto tt = type->AsTableType();
auto rval = new TableVal(tt);
for ( auto& item : a )
{
broker::vector composite_key;
auto indices = broker::get<broker::vector>(item);
if ( ! indices )
{
composite_key.emplace_back(move(item));
indices = &composite_key;
}
auto expected_index_types = tt->Indices()->Types();
if ( static_cast<size_t>(expected_index_types->length()) !=
indices->size() )
{
Unref(rval);
return nullptr;
}
auto list_val = new ListVal(TYPE_ANY);
for ( auto i = 0u; i < indices->size(); ++i )
{
auto index_val = bro_broker::data_to_val(move((*indices)[i]),
(*expected_index_types)[i]);
if ( ! index_val )
{
Unref(rval);
Unref(list_val);
return nullptr;
}
list_val->Append(index_val);
}
rval->Assign(list_val, nullptr);
Unref(list_val);
}
return rval;
}
result_type operator()(broker::table& a)
{
if ( ! type->IsTable() )
return nullptr;
auto tt = type->AsTableType();
auto rval = new TableVal(tt);
for ( auto& item : a )
{
broker::vector composite_key;
auto indices = broker::get<broker::vector>(item.first);
if ( ! indices )
{
composite_key.emplace_back(move(item.first));
indices = &composite_key;
}
auto expected_index_types = tt->Indices()->Types();
if ( static_cast<size_t>(expected_index_types->length()) !=
indices->size() )
{
Unref(rval);
return nullptr;
}
auto list_val = new ListVal(TYPE_ANY);
for ( auto i = 0u; i < indices->size(); ++i )
{
auto index_val = bro_broker::data_to_val(move((*indices)[i]),
(*expected_index_types)[i]);
if ( ! index_val )
{
Unref(rval);
Unref(list_val);
return nullptr;
}
list_val->Append(index_val);
}
auto value_val = bro_broker::data_to_val(move(item.second),
tt->YieldType());
if ( ! value_val )
{
Unref(rval);
Unref(list_val);
return nullptr;
}
rval->Assign(list_val, value_val);
Unref(list_val);
}
return rval;
}
result_type operator()(broker::vector& a)
{
if ( type->Tag() != TYPE_VECTOR )
return nullptr;
auto vt = type->AsVectorType();
auto rval = new VectorVal(vt);
for ( auto& item : a )
{
auto item_val = bro_broker::data_to_val(move(item), vt->YieldType());
if ( ! item_val )
{
Unref(rval);
return nullptr;
}
rval->Assign(rval->Size(), item_val);
}
return rval;
}
result_type operator()(broker::record& a)
{
if ( type->Tag() != TYPE_RECORD )
return nullptr;
auto rt = type->AsRecordType();
auto rval = new RecordVal(rt);
for ( auto i = 0u; i < static_cast<size_t>(rt->NumFields()); ++i )
{
if ( require_log_attr && ! rt->FieldDecl(i)->FindAttr(ATTR_LOG) )
continue;
if ( i >= a.fields.size() )
{
Unref(rval);
return nullptr;
}
if ( ! a.fields[i] )
{
rval->Assign(i, nullptr);
continue;
}
auto item_val = bro_broker::data_to_val(move(*a.fields[i]),
rt->FieldType(i));
if ( ! item_val )
{
Unref(rval);
return nullptr;
}
rval->Assign(i, item_val);
}
return rval;
}
};
Val* bro_broker::data_to_val(broker::data d, BroType* type, bool require_log_attr)
{
return broker::visit(val_converter{type, require_log_attr}, d);
}
broker::util::optional<broker::data> bro_broker::val_to_data(Val* v)
{
switch ( v->Type()->Tag() ) {
case TYPE_BOOL:
return {v->AsBool()};
case TYPE_INT:
return {v->AsInt()};
case TYPE_COUNT:
return {v->AsCount()};
case TYPE_COUNTER:
return {v->AsCounter()};
case TYPE_PORT:
{
auto p = v->AsPortVal();
return {broker::port(p->Port(), to_broker_port_proto(p->PortType()))};
}
case TYPE_ADDR:
{
auto a = v->AsAddr();
in6_addr tmp;
a.CopyIPv6(&tmp);
return {broker::address(reinterpret_cast<const uint32_t*>(&tmp),
broker::address::family::ipv6,
broker::address::byte_order::network)};
}
break;
case TYPE_SUBNET:
{
auto s = v->AsSubNet();
in6_addr tmp;
s.Prefix().CopyIPv6(&tmp);
auto a = broker::address(reinterpret_cast<const uint32_t*>(&tmp),
broker::address::family::ipv6,
broker::address::byte_order::network);
return {broker::subnet(a, s.Length())};
}
break;
case TYPE_DOUBLE:
return {v->AsDouble()};
case TYPE_TIME:
return {broker::time_point(v->AsTime())};
case TYPE_INTERVAL:
return {broker::time_duration(v->AsInterval())};
case TYPE_ENUM:
{
auto enum_type = v->Type()->AsEnumType();
auto enum_name = enum_type->Lookup(v->AsEnum());
return {broker::enum_value(enum_name ? enum_name : "<unknown enum>")};
}
case TYPE_STRING:
{
auto s = v->AsString();
return {string(reinterpret_cast<const char*>(s->Bytes()), s->Len())};
}
case TYPE_FILE:
return {string(v->AsFile()->Name())};
case TYPE_FUNC:
return {string(v->AsFunc()->Name())};
case TYPE_TABLE:
{
auto is_set = v->Type()->IsSet();
auto table = v->AsTable();
auto table_val = v->AsTableVal();
broker::data rval;
if ( is_set )
rval = broker::set();
else
rval = broker::table();
struct iter_guard {
iter_guard(HashKey* arg_k, ListVal* arg_lv)
: k(arg_k), lv(arg_lv)
{}
~iter_guard()
{
delete k;
Unref(lv);
}
HashKey* k;
ListVal* lv;
};
HashKey* k;
TableEntryVal* entry;
auto c = table->InitForIteration();
while ( (entry = table->NextEntry(k, c)) )
{
auto vl = table_val->RecoverIndex(k);
iter_guard ig(k, vl);
broker::vector composite_key;
composite_key.reserve(vl->Length());
for ( auto k = 0; k < vl->Length(); ++k )
{
auto key_part = val_to_data((*vl->Vals())[k]);
if ( ! key_part )
return {};
composite_key.emplace_back(move(*key_part));
}
broker::data key;
if ( composite_key.size() == 1 )
key = move(composite_key[0]);
else
key = move(composite_key);
if ( is_set )
broker::get<broker::set>(rval)->emplace(move(key));
else
{
auto val = val_to_data(entry->Value());
if ( ! val )
return {};
broker::get<broker::table>(rval)->emplace(move(key),
move(*val));
}
}
return {rval};
}
case TYPE_VECTOR:
{
auto vec = v->AsVectorVal();
broker::vector rval;
rval.reserve(vec->Size());
for ( auto i = 0u; i < vec->Size(); ++i )
{
auto item_val = vec->Lookup(i);
if ( ! item_val )
continue;
auto item = val_to_data(item_val);
if ( ! item )
return {};
rval.emplace_back(move(*item));
}
return {rval};
}
case TYPE_RECORD:
{
auto rec = v->AsRecordVal();
broker::record rval;
size_t num_fields = v->Type()->AsRecordType()->NumFields();
rval.fields.reserve(num_fields);
for ( auto i = 0u; i < num_fields; ++i )
{
auto item_val = rec->LookupWithDefault(i);
if ( ! item_val )
{
rval.fields.emplace_back(broker::record::field{});
continue;
}
auto item = val_to_data(item_val);
Unref(item_val);
if ( ! item )
return {};
rval.fields.emplace_back(broker::record::field{move(*item)});
}
return {rval};
}
default:
reporter->Error("unsupported BrokerComm::Data type: %s",
type_name(v->Type()->Tag()));
break;
}
return {};
}
RecordVal* bro_broker::make_data_val(Val* v)
{
auto rval = new RecordVal(BifType::Record::BrokerComm::Data);
auto data = val_to_data(v);
if ( data )
rval->Assign(0, new DataVal(move(*data)));
return rval;
}
RecordVal* bro_broker::make_data_val(broker::data d)
{
auto rval = new RecordVal(BifType::Record::BrokerComm::Data);
rval->Assign(0, new DataVal(move(d)));
return rval;
}
struct data_type_getter {
using result_type = EnumVal*;
result_type operator()(bool a)
{
return new EnumVal(BifEnum::BrokerComm::BOOL,
BifType::Enum::BrokerComm::DataType);
}
result_type operator()(uint64_t a)
{
return new EnumVal(BifEnum::BrokerComm::COUNT,
BifType::Enum::BrokerComm::DataType);
}
result_type operator()(int64_t a)
{
return new EnumVal(BifEnum::BrokerComm::INT,
BifType::Enum::BrokerComm::DataType);
}
result_type operator()(double a)
{
return new EnumVal(BifEnum::BrokerComm::DOUBLE,
BifType::Enum::BrokerComm::DataType);
}
result_type operator()(const std::string& a)
{
return new EnumVal(BifEnum::BrokerComm::STRING,
BifType::Enum::BrokerComm::DataType);
}
result_type operator()(const broker::address& a)
{
return new EnumVal(BifEnum::BrokerComm::ADDR,
BifType::Enum::BrokerComm::DataType);
}
result_type operator()(const broker::subnet& a)
{
return new EnumVal(BifEnum::BrokerComm::SUBNET,
BifType::Enum::BrokerComm::DataType);
}
result_type operator()(const broker::port& a)
{
return new EnumVal(BifEnum::BrokerComm::PORT,
BifType::Enum::BrokerComm::DataType);
}
result_type operator()(const broker::time_point& a)
{
return new EnumVal(BifEnum::BrokerComm::TIME,
BifType::Enum::BrokerComm::DataType);
}
result_type operator()(const broker::time_duration& a)
{
return new EnumVal(BifEnum::BrokerComm::INTERVAL,
BifType::Enum::BrokerComm::DataType);
}
result_type operator()(const broker::enum_value& a)
{
return new EnumVal(BifEnum::BrokerComm::ENUM,
BifType::Enum::BrokerComm::DataType);
}
result_type operator()(const broker::set& a)
{
return new EnumVal(BifEnum::BrokerComm::SET,
BifType::Enum::BrokerComm::DataType);
}
result_type operator()(const broker::table& a)
{
return new EnumVal(BifEnum::BrokerComm::TABLE,
BifType::Enum::BrokerComm::DataType);
}
result_type operator()(const broker::vector& a)
{
return new EnumVal(BifEnum::BrokerComm::VECTOR,
BifType::Enum::BrokerComm::DataType);
}
result_type operator()(const broker::record& a)
{
return new EnumVal(BifEnum::BrokerComm::RECORD,
BifType::Enum::BrokerComm::DataType);
}
};
EnumVal* bro_broker::get_data_type(RecordVal* v, Frame* frame)
{
return broker::visit(data_type_getter{}, opaque_field_to_data(v, frame));
}
broker::data& bro_broker::opaque_field_to_data(RecordVal* v, Frame* f)
{
Val* d = v->Lookup(0);
if ( ! d )
reporter->RuntimeError(f->GetCall()->GetLocationInfo(),
"BrokerComm::Data's opaque field is not set");
return static_cast<DataVal*>(d)->data;
}
IMPLEMENT_SERIAL(bro_broker::DataVal, SER_COMM_DATA_VAL);
bool bro_broker::DataVal::DoSerialize(SerialInfo* info) const
{
DO_SERIALIZE(SER_COMM_DATA_VAL, OpaqueVal);
std::string serial;
caf::binary_serializer bs(std::back_inserter(serial));
bs << data;
if ( ! SERIALIZE_STR(serial.data(), serial.size()) )
return false;
return true;
}
bool bro_broker::DataVal::DoUnserialize(UnserialInfo* info)
{
DO_UNSERIALIZE(OpaqueVal);
const char* serial;
int len;
if ( ! UNSERIALIZE_STR(&serial, &len) )
return false;
caf::binary_deserializer bd(serial, len);
caf::uniform_typeid<broker::data>()->deserialize(&data, &bd);
delete [] serial;
return true;
}

256
src/broker/Data.h Normal file
View file

@ -0,0 +1,256 @@
#ifndef BRO_COMM_DATA_H
#define BRO_COMM_DATA_H
#include <broker/data.hh>
#include "Val.h"
#include "Reporter.h"
#include "Frame.h"
#include "Expr.h"
namespace bro_broker {
extern OpaqueType* opaque_of_data_type;
extern OpaqueType* opaque_of_set_iterator;
extern OpaqueType* opaque_of_table_iterator;
extern OpaqueType* opaque_of_vector_iterator;
extern OpaqueType* opaque_of_record_iterator;
/**
* Convert a broker port protocol to a bro port protocol.
*/
TransportProto to_bro_port_proto(broker::port::protocol tp);
/**
* Create a BrokerComm::Data value from a Bro value.
* @param v the Bro value to convert to a Broker data value.
* @return a BrokerComm::Data value, where the optional field is set if the conversion
* was possible, else it is unset.
*/
RecordVal* make_data_val(Val* v);
/**
* Create a BrokerComm::Data value from a Broker data value.
* @param d the Broker value to wrap in an opaque type.
* @return a BrokerComm::Data value that wraps the Broker value.
*/
RecordVal* make_data_val(broker::data d);
/**
* Get the type of Broker data that BrokerComm::Data wraps.
* @param v a BrokerComm::Data value.
* @param frame used to get location info upon error.
* @return a BrokerComm::DataType value.
*/
EnumVal* get_data_type(RecordVal* v, Frame* frame);
/**
* Convert a Bro value to a Broker data value.
* @param v a Bro value.
* @return a Broker data value if the Bro value could be converted to one.
*/
broker::util::optional<broker::data> val_to_data(Val* v);
/**
* Convert a Broker data value to a Bro value.
* @param d a Broker data value.
* @param type the expected type of the value to return.
* @param require_log_attr if true, skip over record fields that don't have the
* &log attribute.
* @return a pointer to a new Bro value or a nullptr if the conversion was not
* possible.
*/
Val* data_to_val(broker::data d, BroType* type, bool require_log_attr = false);
/**
* A Bro value which wraps a Broker data value.
*/
class DataVal : public OpaqueVal {
public:
DataVal(broker::data arg_data)
: OpaqueVal(bro_broker::opaque_of_data_type), data(std::move(arg_data))
{}
void ValDescribe(ODesc* d) const override
{
d->Add("broker::data{");
d->Add(broker::to_string(data));
d->Add("}");
}
DECLARE_SERIAL(DataVal);
broker::data data;
protected:
DataVal()
{}
};
/**
* Visitor for retrieving type names a Broker data value.
*/
struct type_name_getter {
using result_type = const char*;
result_type operator()(bool a)
{ return "bool"; }
result_type operator()(uint64_t a)
{ return "uint64_t"; }
result_type operator()(int64_t a)
{ return "int64_t"; }
result_type operator()(double a)
{ return "double"; }
result_type operator()(const std::string& a)
{ return "string"; }
result_type operator()(const broker::address& a)
{ return "address"; }
result_type operator()(const broker::subnet& a)
{ return "subnet"; }
result_type operator()(const broker::port& a)
{ return "port"; }
result_type operator()(const broker::time_point& a)
{ return "time"; }
result_type operator()(const broker::time_duration& a)
{ return "interval"; }
result_type operator()(const broker::enum_value& a)
{ return "enum"; }
result_type operator()(const broker::set& a)
{ return "set"; }
result_type operator()(const broker::table& a)
{ return "table"; }
result_type operator()(const broker::vector& a)
{ return "vector"; }
result_type operator()(const broker::record& a)
{ return "record"; }
};
/**
* Retrieve Broker data value associated with a BrokerComm::Data Bro value.
* @param v a BrokerComm::Data value.
* @param f used to get location information on error.
* @return a reference to the wrapped Broker data value. A runtime interpreter
* exception is thrown if the the optional opaque value of \a v is not set.
*/
broker::data& opaque_field_to_data(RecordVal* v, Frame* f);
/**
* Retrieve variant data from a Broker data value.
* @tparam T a type that the variant may contain.
* @param d a Broker data value to get variant data out of.
* @param tag a Bro tag which corresponds to T (just used for error reporting).
* @param f used to get location information on error.
* @return a refrence to the requested type in the variant Broker data.
* A runtime interpret exception is thrown if trying to access a type which
* is not currently stored in the Broker data.
*/
template <typename T>
T& require_data_type(broker::data& d, TypeTag tag, Frame* f)
{
auto ptr = broker::get<T>(d);
if ( ! ptr )
reporter->RuntimeError(f->GetCall()->GetLocationInfo(),
"data is of type '%s' not of type '%s'",
broker::visit(type_name_getter{}, d),
type_name(tag));
return *ptr;
}
/**
* @see require_data_type() and opaque_field_to_data().
*/
template <typename T>
inline T& require_data_type(RecordVal* v, TypeTag tag, Frame* f)
{
return require_data_type<T>(opaque_field_to_data(v, f), tag, f);
}
/**
* Convert a BrokerComm::Data Bro value to a Bro value of a given type.
* @tparam a type that a Broker data variant may contain.
* @param v a BrokerComm::Data value.
* @param tag a Bro type to convert to.
* @param f used to get location information on error.
* A runtime interpret exception is thrown if trying to access a type which
* is not currently stored in the Broker data.
*/
template <typename T>
inline Val* refine(RecordVal* v, TypeTag tag, Frame* f)
{
return new Val(require_data_type<T>(v, tag, f), tag);
}
// Copying data in to iterator vals is not the fastest approach, but safer...
class SetIterator : public OpaqueVal {
public:
SetIterator(RecordVal* v, TypeTag tag, Frame* f)
: OpaqueVal(bro_broker::opaque_of_set_iterator),
dat(require_data_type<broker::set>(v, TYPE_TABLE, f)),
it(dat.begin())
{}
broker::set dat;
broker::set::iterator it;
};
class TableIterator : public OpaqueVal {
public:
TableIterator(RecordVal* v, TypeTag tag, Frame* f)
: OpaqueVal(bro_broker::opaque_of_table_iterator),
dat(require_data_type<broker::table>(v, TYPE_TABLE, f)),
it(dat.begin())
{}
broker::table dat;
broker::table::iterator it;
};
class VectorIterator : public OpaqueVal {
public:
VectorIterator(RecordVal* v, TypeTag tag, Frame* f)
: OpaqueVal(bro_broker::opaque_of_vector_iterator),
dat(require_data_type<broker::vector>(v, TYPE_VECTOR, f)),
it(dat.begin())
{}
broker::vector dat;
broker::vector::iterator it;
};
class RecordIterator : public OpaqueVal {
public:
RecordIterator(RecordVal* v, TypeTag tag, Frame* f)
: OpaqueVal(bro_broker::opaque_of_record_iterator),
dat(require_data_type<broker::record>(v, TYPE_VECTOR, f)),
it(dat.fields.begin())
{}
broker::record dat;
decltype(broker::record::fields)::iterator it;
};
} // namespace bro_broker
#endif // BRO_COMM_DATA_H

1078
src/broker/Manager.cc Normal file
View file

File diff suppressed because it is too large Load diff

366
src/broker/Manager.h Normal file
View file

@ -0,0 +1,366 @@
#ifndef BRO_COMM_MANAGER_H
#define BRO_COMM_MANAGER_H
#include <broker/endpoint.hh>
#include <broker/message_queue.hh>
#include <memory>
#include <string>
#include <map>
#include <unordered_set>
#include "broker/Store.h"
#include "Reporter.h"
#include "iosource/IOSource.h"
#include "Val.h"
namespace bro_broker {
/**
* Communication statistics. Some are tracked in relation to last
* sample (bro_broker::Manager::ConsumeStatistics()).
*/
struct Stats {
// Number of outgoing peer connections (at time of sample).
size_t outgoing_peer_count = 0;
// Number of data stores (at time of sample).
size_t data_store_count = 0;
// Number of pending data store queries (at time of sample).
size_t pending_query_count = 0;
// Number of data store responses received (since last sample).
size_t response_count = 0;
// Number of outgoing connection updates received (since last sample).
size_t outgoing_conn_status_count = 0;
// Number of incoming connection updates received (since last sample).
size_t incoming_conn_status_count = 0;
// Number of broker report messages (e.g. debug, warning, errors) received
// (since last sample).
size_t report_count = 0;
// Number of print messages received per topic-prefix (since last sample).
std::map<std::string, size_t> print_count;
// Number of event messages received per topic-prefix (since last sample).
std::map<std::string, size_t> event_count;
// Number of log messages received per topic-prefix (since last sample).
std::map<std::string, size_t> log_count;
};
/**
* Manages various forms of communication between peer Bro processes
* or other external applications via use of the Broker messaging library.
*/
class Manager : public iosource::IOSource {
friend class StoreHandleVal;
public:
/**
* Destructor. Any still-pending data store queries are aborted.
*/
~Manager();
/**
* Enable use of communication.
* @param flags used to tune the local Broker endpoint's behavior.
* See the BrokerComm::EndpointFlags record type.
* @return true if communication is successfully initialized.
*/
bool Enable(Val* flags);
/**
* Changes endpoint flags originally supplied to bro_broker::Manager::Enable().
* @param flags the new behavior flags to use.
* @return true if flags were changed.
*/
bool SetEndpointFlags(Val* flags);
/**
* @return true if bro_broker::Manager::Enable() has previously been called and
* it succeeded.
*/
bool Enabled()
{ return endpoint != nullptr; }
/**
* Listen for remote connections.
* @param port the TCP port to listen on.
* @param addr an address string on which to accept connections, e.g.
* "127.0.0.1". A nullptr refers to @p INADDR_ANY.
* @param reuse_addr equivalent to behavior of SO_REUSEADDR.
* @return true if the local endpoint is now listening for connections.
*/
bool Listen(uint16_t port, const char* addr = nullptr,
bool reuse_addr = true);
/**
* Initiate a remote connection.
* @param addr an address to connect to, e.g. "localhost" or "127.0.0.1".
* @param port the TCP port on which the remote side is listening.
* @param retry_interval an interval at which to retry establishing the
* connection with the remote peer.
* @return true if it's possible to try connecting with the peer and
* it's a new peer. The actual connection may not be established until a
* later point in time.
*/
bool Connect(std::string addr, uint16_t port,
std::chrono::duration<double> retry_interval);
/**
* Remove a remote connection.
* @param addr the address used in bro_broker::Manager::Connect().
* @param port the port used in bro_broker::Manager::Connect().
* @return true if the arguments match a previously successful call to
* bro_broker::Manager::Connect().
*/
bool Disconnect(const std::string& addr, uint16_t port);
/**
* Print a simple message to any interested peers.
* @param topic a topic string associated with the print message.
* Peers advertise interest by registering a subscription to some prefix
* of this topic name.
* @param msg the string to send to peers.
* @param flags tune the behavior of how the message is send.
* See the BrokerComm::SendFlags record type.
* @return true if the message is sent successfully.
*/
bool Print(std::string topic, std::string msg, Val* flags);
/**
* Send an event to any interested peers.
* @param topic a topic string associated with the print message.
* Peers advertise interest by registering a subscription to some prefix
* of this topic name.
* @param msg the event to send to peers, which is the name of the event
* as a string followed by all of its arguments.
* @param flags tune the behavior of how the message is send.
* See the BrokerComm::SendFlags record type.
* @return true if the message is sent successfully.
*/
bool Event(std::string topic, broker::message msg, int flags);
/**
* Send an event to any interested peers.
* @param topic a topic string associated with the print message.
* Peers advertise interest by registering a subscription to some prefix
* of this topic name.
* @param args the event and its arguments to send to peers. See the
* BrokerComm::EventArgs record type.
* @param flags tune the behavior of how the message is send.
* See the BrokerComm::SendFlags record type.
* @return true if the message is sent successfully.
*/
bool Event(std::string topic, RecordVal* args, Val* flags);
/**
* Send a log entry to any interested peers. The topic name used is
* implicitly "bro/log/<stream-name>".
* @param stream_id the stream to which the log entry belongs.
* @param columns the data which comprises the log entry.
* @param info the record type corresponding to the log's columns.
* @param flags tune the behavior of how the message is send.
* See the BrokerComm::SendFlags record type.
* @return true if the message is sent successfully.
*/
bool Log(EnumVal* stream_id, RecordVal* columns, RecordType* info,
int flags);
/**
* Automatically send an event to any interested peers whenever it is
* locally dispatched (e.g. using "event my_event(...);" in a script).
* @param topic a topic string associated with the event message.
* Peers advertise interest by registering a subscription to some prefix
* of this topic name.
* @param event a Bro event value.
* @param flags tune the behavior of how the message is send.
* See the BrokerComm::SendFlags record type.
* @return true if automatic event sending is now enabled.
*/
bool AutoEvent(std::string topic, Val* event, Val* flags);
/**
* Stop automatically sending an event to peers upon local dispatch.
* @param topic a topic originally given to bro_broker::Manager::AutoEvent().
* @param event an event originally given to bro_broker::Manager::AutoEvent().
* @return true if automatic events will no occur for the topic/event pair.
*/
bool AutoEventStop(const std::string& topic, Val* event);
/**
* Create an EventArgs record value from an event and its arguments.
* @param args the event and its arguments. The event is always the first
* elements in the list.
* @return an EventArgs record value. If an invalid event or arguments
* were supplied the optional "name" field will not be set.
*/
RecordVal* MakeEventArgs(val_list* args);
/**
* Register interest in peer print messages that use a certain topic prefix.
* @param topic_prefix a prefix to match against remote message topics.
* e.g. an empty prefix will match everything and "a" will match "alice"
* and "amy" but not "bob".
* @return true if it's a new print subscriptions and it is now registered.
*/
bool SubscribeToPrints(std::string topic_prefix);
/**
* Unregister interest in peer print messages.
* @param topic_prefix a prefix previously supplied to a successful call
* to bro_broker::Manager::SubscribeToPrints().
* @return true if interest in topic prefix is no longer advertised.
*/
bool UnsubscribeToPrints(const std::string& topic_prefix);
/**
* Register interest in peer event messages that use a certain topic prefix.
* @param topic_prefix a prefix to match against remote message topics.
* e.g. an empty prefix will match everything and "a" will match "alice"
* and "amy" but not "bob".
* @return true if it's a new event subscription and it is now registered.
*/
bool SubscribeToEvents(std::string topic_prefix);
/**
* Unregister interest in peer event messages.
* @param topic_prefix a prefix previously supplied to a successful call
* to bro_broker::Manager::SubscribeToEvents().
* @return true if interest in topic prefix is no longer advertised.
*/
bool UnsubscribeToEvents(const std::string& topic_prefix);
/**
* Register interest in peer log messages that use a certain topic prefix.
* @param topic_prefix a prefix to match against remote message topics.
* e.g. an empty prefix will match everything and "a" will match "alice"
* and "amy" but not "bob".
* @return true if it's a new log subscription and it is now registered.
*/
bool SubscribeToLogs(std::string topic_prefix);
/**
* Unregister interest in peer log messages.
* @param topic_prefix a prefix previously supplied to a successful call
* to bro_broker::Manager::SubscribeToLogs().
* @return true if interest in topic prefix is no longer advertised.
*/
bool UnsubscribeToLogs(const std::string& topic_prefix);
/**
* Allow sending messages to peers if associated with the given topic.
* This has no effect if auto publication behavior is enabled via the flags
* supplied to bro_broker::Manager::Enable() or bro_broker::Manager::SetEndpointFlags().
* @param t a topic to allow messages to be published under.
* @return true if successful.
*/
bool PublishTopic(broker::topic t);
/**
* Disallow sending messages to peers if associated with the given topic.
* This has no effect if auto publication behavior is enabled via the flags
* supplied to bro_broker::Manager::Enable() or bro_broker::Manager::SetEndpointFlags().
* @param t a topic to disallow messages to be published under.
* @return true if successful.
*/
bool UnpublishTopic(broker::topic t);
/**
* Allow advertising interest in the given topic to peers.
* This has no effect if auto advertise behavior is enabled via the flags
* supplied to bro_broker::Manager::Enable() or bro_broker::Manager::SetEndpointFlags().
* @param t a topic to allow advertising interest/subscription to peers.
* @return true if successful.
*/
bool AdvertiseTopic(broker::topic t);
/**
* Disallow advertising interest in the given topic to peers.
* This has no effect if auto advertise behavior is enabled via the flags
* supplied to bro_broker::Manager::Enable() or bro_broker::Manager::SetEndpointFlags().
* @param t a topic to disallow advertising interest/subscription to peers.
* @return true if successful.
*/
bool UnadvertiseTopic(broker::topic t);
/**
* Register the availability of a data store.
* @param handle the data store.
* @return true if the store was valid and not already away of it.
*/
bool AddStore(StoreHandleVal* handle);
/**
* Lookup a data store by it's identifier name and type.
* @param id the store's name.
* @param type the type of data store.
* @return a pointer to the store handle if it exists else nullptr.
*/
StoreHandleVal* LookupStore(const broker::store::identifier& id, StoreType type);
/**
* Close and unregister a data store. Any existing references to the
* store handle will not be able to be used for any data store operations.
* @param id the stores' name.
* @param type the type of the data store.
* @return true if such a store existed and is now closed.
*/
bool CloseStore(const broker::store::identifier& id, StoreType type);
/**
* Register a data store query callback.
* @param cb the callback info to use when the query completes or times out.
* @return true if now tracking a data store query.
*/
bool TrackStoreQuery(StoreQueryCallback* cb);
/**
* @return communication statistics.
*/
Stats ConsumeStatistics();
/**
* Convert BrokerComm::SendFlags to int flags for use with broker::send().
*/
static int send_flags_to_int(Val* flags);
private:
// IOSource interface overrides:
void GetFds(iosource::FD_Set* read, iosource::FD_Set* write,
iosource::FD_Set* except) override;
double NextTimestamp(double* local_network_time) override;
void Process() override;
const char* Tag() override
{ return "BrokerComm::Manager"; }
broker::endpoint& Endpoint()
{ return *endpoint; }
struct QueueWithStats {
broker::message_queue q;
size_t received = 0;
};
std::unique_ptr<broker::endpoint> endpoint;
std::map<std::pair<std::string, uint16_t>, broker::peering> peers;
std::map<std::string, QueueWithStats> print_subscriptions;
std::map<std::string, QueueWithStats> event_subscriptions;
std::map<std::string, QueueWithStats> log_subscriptions;
std::map<std::pair<broker::store::identifier, StoreType>,
StoreHandleVal*> data_stores;
std::unordered_set<StoreQueryCallback*> pending_queries;
Stats statistics;
static VectorType* vector_of_data_type;
static EnumType* log_id_type;
static int send_flags_self_idx;
static int send_flags_peers_idx;
static int send_flags_unsolicited_idx;
};
} // namespace bro_broker
extern bro_broker::Manager* broker_mgr;
#endif // BRO_COMM_MANAGER_H

204
src/broker/Store.cc Normal file
View file

@ -0,0 +1,204 @@
#include "Store.h"
#include "broker/Manager.h"
#include <broker/store/master.hh>
#include <broker/store/clone.hh>
#include <broker/store/sqlite_backend.hh>
#ifdef HAVE_ROCKSDB
#include <broker/store/rocksdb_backend.hh>
#include <rocksdb/db.h>
#endif
OpaqueType* bro_broker::opaque_of_store_handle;
bro_broker::StoreHandleVal::StoreHandleVal(broker::store::identifier id,
bro_broker::StoreType arg_type,
broker::util::optional<BifEnum::BrokerStore::BackendType> arg_back,
RecordVal* backend_options, std::chrono::duration<double> resync)
: OpaqueVal(opaque_of_store_handle),
store(), store_type(arg_type), backend_type(arg_back)
{
using BifEnum::BrokerStore::BackendType;
std::unique_ptr<broker::store::backend> backend;
if ( backend_type )
switch ( *backend_type ) {
case BackendType::MEMORY:
backend.reset(new broker::store::memory_backend);
break;
case BackendType::SQLITE:
{
auto sqlite = new broker::store::sqlite_backend;
std::string path = backend_options->Lookup(0)->AsRecordVal()
->Lookup(0)->AsStringVal()->CheckString();
if ( sqlite->open(path) )
backend.reset(sqlite);
else
{
reporter->Error("failed to open sqlite backend at path %s: %s",
path.data(), sqlite->last_error().data());
delete sqlite;
}
}
break;
case BackendType::ROCKSDB:
{
#ifdef HAVE_ROCKSDB
std::string path = backend_options->Lookup(1)->AsRecordVal()
->Lookup(0)->AsStringVal()->CheckString();
rocksdb::Options rock_op;
rock_op.create_if_missing = true;
auto rocksdb = new broker::store::rocksdb_backend;
if ( rocksdb->open(path, options).ok() )
backend.reset(rocksdb);
else
{
reporter->Error("failed to open rocksdb backend at path %s: %s",
path.data(), rocksdb->last_error().data());
delete rocksdb;
}
#else
reporter->Error("rocksdb backend support is not enabled");
#endif
}
break;
default:
reporter->FatalError("unknown data store backend: %d",
static_cast<int>(*backend_type));
}
switch ( store_type ) {
case StoreType::FRONTEND:
store = new broker::store::frontend(broker_mgr->Endpoint(), move(id));
break;
case StoreType::MASTER:
store = new broker::store::master(broker_mgr->Endpoint(), move(id),
move(backend));
break;
case StoreType::CLONE:
store = new broker::store::clone(broker_mgr->Endpoint(), move(id), resync,
move(backend));
break;
default:
reporter->FatalError("unknown data store type: %d",
static_cast<int>(store_type));
}
}
void bro_broker::StoreHandleVal::ValDescribe(ODesc* d) const
{
using BifEnum::BrokerStore::BackendType;
d->Add("broker::store::");
switch ( store_type ) {
case StoreType::FRONTEND:
d->Add("frontend");
break;
case StoreType::MASTER:
d->Add("master");
break;
case StoreType::CLONE:
d->Add("clone");
break;
default:
d->Add("unknown");
}
d->Add("{");
d->Add(store->id());
if ( backend_type )
{
d->Add(", ");
switch ( *backend_type ) {
case BackendType::MEMORY:
d->Add("memory");
break;
case BackendType::SQLITE:
d->Add("sqlite");
break;
case BackendType::ROCKSDB:
d->Add("rocksdb");
break;
default:
d->Add("unknown");
}
}
d->Add("}");
}
IMPLEMENT_SERIAL(bro_broker::StoreHandleVal, SER_COMM_STORE_HANDLE_VAL);
bool bro_broker::StoreHandleVal::DoSerialize(SerialInfo* info) const
{
DO_SERIALIZE(SER_COMM_STORE_HANDLE_VAL, OpaqueVal);
bool have_store = store != nullptr;
if ( ! SERIALIZE(have_store) )
return false;
if ( ! have_store )
return true;
if ( ! SERIALIZE(static_cast<int>(store_type)) )
return false;
if ( ! SERIALIZE_STR(store->id().data(), store->id().size()) )
return false;
return true;
}
bool bro_broker::StoreHandleVal::DoUnserialize(UnserialInfo* info)
{
DO_UNSERIALIZE(OpaqueVal);
bool have_store;
if ( ! UNSERIALIZE(&have_store) )
return false;
if ( ! have_store )
{
store = nullptr;
return true;
}
int type;
if ( ! UNSERIALIZE(&type) )
return false;
const char* id_str;
int len;
if ( ! UNSERIALIZE_STR(&id_str, &len) )
return false;
broker::store::identifier id(id_str, len);
delete [] id_str;
auto handle = broker_mgr->LookupStore(id, static_cast<bro_broker::StoreType>(type));
if ( ! handle )
{
// Passing serialized version of store handles to other Bro processes
// doesn't make sense, only allow local clones of the handle val.
reporter->Error("failed to look up unserialized store handle %s, %d",
id.data(), type);
store = nullptr;
return false;
}
store = handle->store;
store_type = handle->store_type;
backend_type = handle->backend_type;
return true;
}

153
src/broker/Store.h Normal file
View file

@ -0,0 +1,153 @@
#ifndef BRO_COMM_STORE_H
#define BRO_COMM_STORE_H
#include "broker/store.bif.h"
#include "broker/data.bif.h"
#include "Reporter.h"
#include "Type.h"
#include "Val.h"
#include "Trigger.h"
#include <broker/store/frontend.hh>
namespace bro_broker {
extern OpaqueType* opaque_of_store_handle;
/**
* Enumerates the possible types of data stores.
*/
enum StoreType {
// Just a view in to a remote store, contains no data itself.
FRONTEND,
MASTER,
CLONE,
};
/**
* Create a BrokerStore::QueryStatus value.
* @param success whether the query status should be set to success or failure.
* @return a BrokerStore::QueryStatus value.
*/
inline EnumVal* query_status(bool success)
{
static EnumType* store_query_status = nullptr;
static int success_val;
static int failure_val;
if ( ! store_query_status )
{
store_query_status = internal_type("BrokerStore::QueryStatus")->AsEnumType();
success_val = store_query_status->Lookup("BrokerStore", "SUCCESS");
failure_val = store_query_status->Lookup("BrokerStore", "FAILURE");
}
return new EnumVal(success ? success_val : failure_val, store_query_status);
}
/**
* @return a BrokerStore::QueryResult value that has a BrokerStore::QueryStatus indicating
* a failure.
*/
inline RecordVal* query_result()
{
auto rval = new RecordVal(BifType::Record::BrokerStore::QueryResult);
rval->Assign(0, query_status(false));
rval->Assign(1, new RecordVal(BifType::Record::BrokerComm::Data));
return rval;
}
/**
* @param data the result of the query.
* @return a BrokerStore::QueryResult value that has a BrokerStore::QueryStatus indicating
* a success.
*/
inline RecordVal* query_result(RecordVal* data)
{
auto rval = new RecordVal(BifType::Record::BrokerStore::QueryResult);
rval->Assign(0, query_status(true));
rval->Assign(1, data);
return rval;
}
/**
* Used for asynchronous data store queries which use "when" statements.
*/
class StoreQueryCallback {
public:
StoreQueryCallback(Trigger* arg_trigger, const CallExpr* arg_call,
broker::store::identifier arg_store_id,
StoreType arg_store_type)
: trigger(arg_trigger), call(arg_call), store_id(move(arg_store_id)),
store_type(arg_store_type)
{
Ref(trigger);
}
~StoreQueryCallback()
{
Unref(trigger);
}
void Result(RecordVal* result)
{
trigger->Cache(call, result);
trigger->Release();
Unref(result);
}
void Abort()
{
auto result = query_result();
trigger->Cache(call, result);
trigger->Release();
Unref(result);
}
bool Disabled() const
{ return trigger->Disabled(); }
const broker::store::identifier& StoreID() const
{ return store_id; }
StoreType GetStoreType() const
{ return store_type; }
private:
Trigger* trigger;
const CallExpr* call;
broker::store::identifier store_id;
StoreType store_type;
};
/**
* An opaque handle which wraps a Broker data store.
*/
class StoreHandleVal : public OpaqueVal {
public:
StoreHandleVal(broker::store::identifier id,
bro_broker::StoreType arg_type,
broker::util::optional<BifEnum::BrokerStore::BackendType> arg_back,
RecordVal* backend_options,
std::chrono::duration<double> resync = std::chrono::seconds(1));
void ValDescribe(ODesc* d) const override;
DECLARE_SERIAL(StoreHandleVal);
broker::store::frontend* store;
bro_broker::StoreType store_type;
broker::util::optional<BifEnum::BrokerStore::BackendType> backend_type;
protected:
StoreHandleVal()
{}
};
} // namespace bro_broker
#endif // BRO_COMM_STORE_H

199
src/broker/comm.bif Normal file
View file

@ -0,0 +1,199 @@
##! General functions regarding Bro's broker communication mechanisms.
%%{
#include "broker/Manager.h"
%%}
module BrokerComm;
type BrokerComm::EndpointFlags: record;
## Enable use of communication.
##
## flags: used to tune the local Broker endpoint behavior.
##
## Returns: true if communication is successfully initialized.
function BrokerComm::enable%(flags: EndpointFlags &default = EndpointFlags()%): bool
%{
return new Val(broker_mgr->Enable(flags), TYPE_BOOL);
%}
## Changes endpoint flags originally supplied to :bro:see:`BrokerComm::enable`.
##
## flags: the new endpoint behavior flags to use.
##
## Returns: true of flags were changed.
function BrokerComm::set_endpoint_flags%(flags: EndpointFlags &default = EndpointFlags()%): bool
%{
return new Val(broker_mgr->SetEndpointFlags(flags), TYPE_BOOL);
%}
## Allow sending messages to peers if associated with the given topic.
## This has no effect if auto publication behavior is enabled via the flags
## supplied to :bro:see:`BrokerComm::enable` or :bro:see:`BrokerComm::set_endpoint_flags`.
##
## topic: a topic to allow messages to be published under.
##
## Returns: true if successful.
function BrokerComm::publish_topic%(topic: string%): bool
%{
return new Val(broker_mgr->PublishTopic(topic->CheckString()), TYPE_BOOL);
%}
## Disallow sending messages to peers if associated with the given topic.
## This has no effect if auto publication behavior is enabled via the flags
## supplied to :bro:see:`BrokerComm::enable` or :bro:see:`BrokerComm::set_endpoint_flags`.
##
## topic: a topic to disallow messages to be published under.
##
## Returns: true if successful.
function BrokerComm::unpublish_topic%(topic: string%): bool
%{
return new Val(broker_mgr->UnpublishTopic(topic->CheckString()), TYPE_BOOL);
%}
## Allow advertising interest in the given topic to peers.
## This has no effect if auto advertise behavior is enabled via the flags
## supplied to :bro:see:`BrokerComm::enable` or :bro:see:`BrokerComm::set_endpoint_flags`.
##
## topic: a topic to allow advertising interest/subscription to peers.
##
## Returns: true if successful.
function BrokerComm::advertise_topic%(topic: string%): bool
%{
return new Val(broker_mgr->AdvertiseTopic(topic->CheckString()), TYPE_BOOL);
%}
## Disallow advertising interest in the given topic to peers.
## This has no effect if auto advertise behavior is enabled via the flags
## supplied to :bro:see:`BrokerComm::enable` or :bro:see:`BrokerComm::set_endpoint_flags`.
##
## topic: a topic to disallow advertising interest/subscription to peers.
##
## Returns: true if successful.
function BrokerComm::unadvertise_topic%(topic: string%): bool
%{
return new Val(broker_mgr->UnadvertiseTopic(topic->CheckString()), TYPE_BOOL);
%}
## Generated when a connection has been established due to a previous call
## to :bro:see:`BrokerComm::connect`.
##
## peer_address: the address used to connect to the peer.
##
## peer_port: the port used to connect to the peer.
##
## peer_name: the name by which the peer identified itself.
event BrokerComm::outgoing_connection_established%(peer_address: string,
peer_port: port,
peer_name: string%);
## Generated when a previously established connection becomes broken.
## Reconnection will automatically be attempted at a frequency given
## by the original call to :bro:see:`BrokerComm::connect`.
##
## peer_address: the address used to connect to the peer.
##
## peer_port: the port used to connect to the peer.
##
## .. bro:see:: BrokerComm::outgoing_connection_established
event BrokerComm::outgoing_connection_broken%(peer_address: string,
peer_port: port%);
## Generated when a connection via :bro:see:`BrokerComm::connect` has failed
## because the remote side is incompatible.
##
## peer_address: the address used to connect to the peer.
##
## peer_port: the port used to connect to the peer.
event BrokerComm::outgoing_connection_incompatible%(peer_address: string,
peer_port: port%);
## Generated when a peer has established a connection with this process
## as a result of previously performing a :bro:see:`BrokerComm::listen`.
##
## peer_name: the name by which the peer identified itself.
event BrokerComm::incoming_connection_established%(peer_name: string%);
## Generated when a peer that previously established a connection with this
## process becomes disconnected.
##
## peer_name: the name by which the peer identified itself.
##
## .. bro:see:: BrokerComm::incoming_connection_established
event BrokerComm::incoming_connection_broken%(peer_name: string%);
## Listen for remote connections.
##
## p: the TCP port to listen on.
##
## a: an address string on which to accept connections, e.g.
## "127.0.0.1". An empty string refers to @p INADDR_ANY.
##
## reuse: equivalent to behavior of SO_REUSEADDR.
##
## Returns: true if the local endpoint is now listening for connections.
##
## .. bro:see:: BrokerComm::incoming_connection_established
function BrokerComm::listen%(p: port, a: string &default = "",
reuse: bool &default = T%): bool
%{
if ( ! p->IsTCP() )
{
reporter->Error("listen port must use tcp");
return new Val(false, TYPE_BOOL);
}
auto rval = broker_mgr->Listen(p->Port(), a->Len() ? a->CheckString() : 0,
reuse);
return new Val(rval, TYPE_BOOL);
%}
## Initiate a remote connection.
##
## a: an address to connect to, e.g. "localhost" or "127.0.0.1".
##
## p: the TCP port on which the remote side is listening.
##
## retry: an interval at which to retry establishing the
## connection with the remote peer if it cannot be made initially, or
## if it ever becomes disconnected.
##
## Returns: true if it's possible to try connecting with the peer and
## it's a new peer. The actual connection may not be established
## a later point in time.
##
## .. bro:see:: BrokerComm::outgoing_connection_established
function BrokerComm::connect%(a: string, p: port, retry: interval%): bool
%{
if ( ! p->IsTCP() )
{
reporter->Error("remote connection port must use tcp");
return new Val(false, TYPE_BOOL);
}
auto rval = broker_mgr->Connect(a->CheckString(), p->Port(),
std::chrono::duration<double>(retry));
return new Val(rval, TYPE_BOOL);
%}
## Remove a remote connection.
##
## a: the address used in previous successful call to :bro:see:`BrokerComm::connect`.
##
## p: the port used in previous successful call to :bro:see:`BrokerComm::connect`.
##
## Returns: true if the arguments match a previously successful call to
## :bro:see:`BrokerComm::connect`.
function BrokerComm::disconnect%(a: string, p: port%): bool
%{
if ( ! p->IsTCP() )
{
reporter->Error("remote connection port must use tcp");
return new Val(false, TYPE_BOOL);
}
auto rval = broker_mgr->Disconnect(a->CheckString(), p->Port());
return new Val(rval, TYPE_BOOL);
%}

834
src/broker/data.bif Normal file
View file

@ -0,0 +1,834 @@
##! Functions for inspecting and manipulating broker data.
%%{
#include "broker/Data.h"
%%}
module BrokerComm;
## Enumerates the possible types that :bro:see:`BrokerComm::Data` may be in terms of
## Bro data types.
enum DataType %{
BOOL,
INT,
COUNT,
DOUBLE,
STRING,
ADDR,
SUBNET,
PORT,
TIME,
INTERVAL,
ENUM,
SET,
TABLE,
VECTOR,
RECORD,
%}
type BrokerComm::Data: record;
type BrokerComm::TableItem: record;
## Convert any Bro value in to communication data.
##
## d: any Bro value to attempt to convert (not all types are supported).
##
## Returns: the converted communication data which may not set its only
## opaque field of the the conversion was not possible (the Bro data
## type does not support being converted to communicaiton data).
function BrokerComm::data%(d: any%): BrokerComm::Data
%{
return bro_broker::make_data_val(d);
%}
## Retrieve the type of data associated with communication data.
##
## d: the communication data.
##
## Returns: the data type associated with the communication data.
function BrokerComm::data_type%(d: BrokerComm::Data%): BrokerComm::DataType
%{
return bro_broker::get_data_type(d->AsRecordVal(), frame);
%}
## Convert communication data with a type of :bro:see:`BrokerComm::BOOL` to
## an actual Bro value.
##
## d: the communication data to convert.
##
## Returns: the value retrieved from the communication data.
function BrokerComm::refine_to_bool%(d: BrokerComm::Data%): bool
%{
return bro_broker::refine<bool>(d->AsRecordVal(), TYPE_BOOL, frame);
%}
## Convert communication data with a type of :bro:see:`BrokerComm::INT` to
## an actual Bro value.
##
## d: the communication data to convert.
##
## Returns: the value retrieved from the communication data.
function BrokerComm::refine_to_int%(d: BrokerComm::Data%): int
%{
return bro_broker::refine<int64_t>(d->AsRecordVal(), TYPE_INT, frame);
%}
## Convert communication data with a type of :bro:see:`BrokerComm::COUNT` to
## an actual Bro value.
##
## d: the communication data to convert.
##
## Returns: the value retrieved from the communication data.
function BrokerComm::refine_to_count%(d: BrokerComm::Data%): count
%{
return bro_broker::refine<uint64_t>(d->AsRecordVal(), TYPE_COUNT, frame);
%}
## Convert communication data with a type of :bro:see:`BrokerComm::DOUBLE` to
## an actual Bro value.
##
## d: the communication data to convert.
##
## Returns: the value retrieved from the communication data.
function BrokerComm::refine_to_double%(d: BrokerComm::Data%): double
%{
return bro_broker::refine<double>(d->AsRecordVal(), TYPE_DOUBLE, frame);
%}
## Convert communication data with a type of :bro:see:`BrokerComm::STRING` to
## an actual Bro value.
##
## d: the communication data to convert.
##
## Returns: the value retrieved from the communication data.
function BrokerComm::refine_to_string%(d: BrokerComm::Data%): string
%{
return new StringVal(bro_broker::require_data_type<std::string>(d->AsRecordVal(),
TYPE_STRING,
frame));
%}
## Convert communication data with a type of :bro:see:`BrokerComm::ADDR` to
## an actual Bro value.
##
## d: the communication data to convert.
##
## Returns: the value retrieved from the communication data.
function BrokerComm::refine_to_addr%(d: BrokerComm::Data%): addr
%{
auto& a = bro_broker::require_data_type<broker::address>(d->AsRecordVal(),
TYPE_ADDR, frame);
auto bits = reinterpret_cast<const in6_addr*>(&a.bytes());
return new AddrVal(IPAddr(*bits));
%}
## Convert communication data with a type of :bro:see:`BrokerComm::SUBNET` to
## an actual Bro value.
##
## d: the communication data to convert.
##
## Returns: the value retrieved from the communication data.
function BrokerComm::refine_to_subnet%(d: BrokerComm::Data%): subnet
%{
auto& a = bro_broker::require_data_type<broker::subnet>(d->AsRecordVal(),
TYPE_SUBNET, frame);
auto bits = reinterpret_cast<const in6_addr*>(&a.network().bytes());
return new SubNetVal(IPPrefix(IPAddr(*bits), a.length()));
%}
## Convert communication data with a type of :bro:see:`BrokerComm::PORT` to
## an actual Bro value.
##
## d: the communication data to convert.
##
## Returns: the value retrieved from the communication data.
function BrokerComm::refine_to_port%(d: BrokerComm::Data%): port
%{
auto& a = bro_broker::require_data_type<broker::port>(d->AsRecordVal(),
TYPE_SUBNET, frame);
return new PortVal(a.number(), bro_broker::to_bro_port_proto(a.type()));
%}
## Convert communication data with a type of :bro:see:`BrokerComm::TIME` to
## an actual Bro value.
##
## d: the communication data to convert.
##
## Returns: the value retrieved from the communication data.
function BrokerComm::refine_to_time%(d: BrokerComm::Data%): time
%{
auto v = bro_broker::require_data_type<broker::time_point>(d->AsRecordVal(),
TYPE_TIME, frame).value;
return new Val(v, TYPE_TIME);
%}
## Convert communication data with a type of :bro:see:`BrokerComm::INTERVAL` to
## an actual Bro value.
##
## d: the communication data to convert.
##
## Returns: the value retrieved from the communication data.
function BrokerComm::refine_to_interval%(d: BrokerComm::Data%): interval
%{
auto v = bro_broker::require_data_type<broker::time_duration>(d->AsRecordVal(),
TYPE_TIME, frame).value;
return new Val(v, TYPE_INTERVAL);
%}
## Convert communication data with a type of :bro:see:`BrokerComm::ENUM` to
## the name of the enum value. :bro:see:`lookup_ID` may be used to convert
## the name to the actual enum value.
##
## d: the communication data to convert.
##
## Returns: the enum name retrieved from the communication data.
function BrokerComm::refine_to_enum_name%(d: BrokerComm::Data%): string
%{
auto& v = bro_broker::require_data_type<broker::enum_value>(d->AsRecordVal(),
TYPE_ENUM, frame).name;
return new StringVal(v);
%}
## Create communication data of type "set".
function BrokerComm::set_create%(%): BrokerComm::Data
%{
return bro_broker::make_data_val(broker::set());
%}
## Remove all elements within a set.
##
## s: the set to clear.
##
## Returns: always true.
function BrokerComm::set_clear%(s: BrokerComm::Data%): bool
%{
auto& v = bro_broker::require_data_type<broker::set>(s->AsRecordVal(), TYPE_TABLE,
frame);
v.clear();
return new Val(true, TYPE_BOOL);
%}
## Get the number of elements within a set.
##
## s: the set to query.
##
## Returns: the number of elements in the set.
function BrokerComm::set_size%(s: BrokerComm::Data%): count
%{
auto& v = bro_broker::require_data_type<broker::set>(s->AsRecordVal(), TYPE_TABLE,
frame);
return new Val(static_cast<uint64_t>(v.size()), TYPE_COUNT);
%}
## Check if a set contains a particular element.
##
## s: the set to query.
##
## key: the element to check for existence.
##
## Returns: true if the key exists in the set.
function BrokerComm::set_contains%(s: BrokerComm::Data, key: BrokerComm::Data%): bool
%{
auto& v = bro_broker::require_data_type<broker::set>(s->AsRecordVal(), TYPE_TABLE,
frame);
auto& k = bro_broker::opaque_field_to_data(key->AsRecordVal(), frame);
return new Val(v.find(k) != v.end(), TYPE_BOOL);
%}
### Insert an element into a set.
##
## s: the set to modify.
##
## key: the element to insert.
##
## Returns: true if the key was inserted, or false if it already existed.
function BrokerComm::set_insert%(s: BrokerComm::Data, key: BrokerComm::Data%): bool
%{
auto& v = bro_broker::require_data_type<broker::set>(s->AsRecordVal(), TYPE_TABLE,
frame);
auto& k = bro_broker::opaque_field_to_data(key->AsRecordVal(), frame);
return new Val(v.insert(k).second, TYPE_BOOL);
%}
## Remove an element from a set.
##
## s: the set to modify.
##
## key: the element to remove.
##
## Returns: true if the element existed in the set and is now removed.
function BrokerComm::set_remove%(s: BrokerComm::Data, key: BrokerComm::Data%): bool
%{
auto& v = bro_broker::require_data_type<broker::set>(s->AsRecordVal(), TYPE_TABLE,
frame);
auto& k = bro_broker::opaque_field_to_data(key->AsRecordVal(), frame);
return new Val(v.erase(k) > 0, TYPE_BOOL);
%}
## Create an iterator for a set. Note that this makes a copy of the set
## internally to ensure the iterator is always valid.
##
## s: the set to iterate over.
##
## Returns: an iterator.
function BrokerComm::set_iterator%(s: BrokerComm::Data%): opaque of BrokerComm::SetIterator
%{
return new bro_broker::SetIterator(s->AsRecordVal(), TYPE_TABLE, frame);
%}
## Check if there are no more elements to iterate over.
##
## it: an iterator.
##
## Returns: true if there are no more elements to iterator over, i.e.
## the iterator is one-past-the-final-element.
function BrokerComm::set_iterator_last%(it: opaque of BrokerComm::SetIterator%): bool
%{
auto set_it = static_cast<bro_broker::SetIterator*>(it);
return new Val(set_it->it == set_it->dat.end(), TYPE_BOOL);
%}
## Advance an iterator.
##
## it: an iterator.
##
## Returns: true if the iterator, after advancing, still references an element
## in the collection. False if the iterator, after advancing, is
## one-past-the-final-element.
function BrokerComm::set_iterator_next%(it: opaque of BrokerComm::SetIterator%): bool
%{
auto set_it = static_cast<bro_broker::SetIterator*>(it);
if ( set_it->it == set_it->dat.end() )
return new Val(false, TYPE_BOOL);
++set_it->it;
return new Val(set_it->it != set_it->dat.end(), TYPE_BOOL);
%}
## Retrieve the data at an iterator's current position.
##
## it: an iterator.
##
## Returns: element in the collection that the iterator currently references.
function BrokerComm::set_iterator_value%(it: opaque of BrokerComm::SetIterator%): BrokerComm::Data
%{
auto set_it = static_cast<bro_broker::SetIterator*>(it);
auto rval = new RecordVal(BifType::Record::BrokerComm::Data);
if ( set_it->it == set_it->dat.end() )
{
reporter->PushLocation(frame->GetCall()->GetLocationInfo());
reporter->Warning("attempt to retrieve value of invalid set iterator");
reporter->PopLocation();
return rval;
}
rval->Assign(0, new bro_broker::DataVal(*set_it->it));
return rval;
%}
## Create communication data of type "table".
function BrokerComm::table_create%(%): BrokerComm::Data
%{
return bro_broker::make_data_val(broker::table());
%}
## Remove all elements within a table.
##
## t: the table to clear.
##
## Returns: always true.
function BrokerComm::table_clear%(t: BrokerComm::Data%): bool
%{
auto& v = bro_broker::require_data_type<broker::table>(t->AsRecordVal(),
TYPE_TABLE, frame);
v.clear();
return new Val(true, TYPE_BOOL);
%}
## Get the number of elements within a table.
##
## t: the table to query.
##
## Returns: the number of elements in the table.
function BrokerComm::table_size%(t: BrokerComm::Data%): count
%{
auto& v = bro_broker::require_data_type<broker::table>(t->AsRecordVal(),
TYPE_TABLE, frame);
return new Val(static_cast<uint64_t>(v.size()), TYPE_COUNT);
%}
## Check if a table contains a particular key.
##
## t: the table to query.
##
## key: the key to check for existence.
##
## Returns: true if the key exists in the set.
function BrokerComm::table_contains%(t: BrokerComm::Data, key: BrokerComm::Data%): bool
%{
auto& v = bro_broker::require_data_type<broker::table>(t->AsRecordVal(),
TYPE_TABLE, frame);
auto& k = bro_broker::opaque_field_to_data(key->AsRecordVal(), frame);
return new Val(v.find(k) != v.end(), TYPE_BOOL);
%}
## Insert a key-value pair into a table.
##
## t: the table to modify.
##
## key: the key at which to insert the value.
##
## val: the value to insert.
##
## Returns: true if the key-value pair was inserted, or false if the key
## already existed in the table.
function BrokerComm::table_insert%(t: BrokerComm::Data, key: BrokerComm::Data, val: BrokerComm::Data%): BrokerComm::Data
%{
auto& table = bro_broker::require_data_type<broker::table>(t->AsRecordVal(),
TYPE_TABLE, frame);
auto& k = bro_broker::opaque_field_to_data(key->AsRecordVal(), frame);
auto& v = bro_broker::opaque_field_to_data(val->AsRecordVal(), frame);
try
{
auto& prev = table.at(k);
auto rval = bro_broker::make_data_val(move(prev));
prev = v;
return rval;
}
catch (const std::out_of_range&)
{
table[k] = v;
return new RecordVal(BifType::Record::BrokerComm::Data);
}
%}
## Remove a key-value pair from a table.
##
## t: the table to modify.
##
## key: the key to remove from the table.
##
## Returns: the value associated with the key. If the key did not exist, then
## the optional field of the returned record is not set.
function BrokerComm::table_remove%(t: BrokerComm::Data, key: BrokerComm::Data%): BrokerComm::Data
%{
auto& table = bro_broker::require_data_type<broker::table>(t->AsRecordVal(),
TYPE_TABLE, frame);
auto& k = bro_broker::opaque_field_to_data(key->AsRecordVal(), frame);
auto it = table.find(k);
if ( it == table.end() )
return new RecordVal(BifType::Record::BrokerComm::Data);
else
{
auto rval = bro_broker::make_data_val(move(it->second));
table.erase(it);
return rval;
}
%}
## Retrieve a value from a table.
##
## t: the table to query.
##
## key: the key to lookup.
##
## Returns: the value associated with the key. If the key did not exist, then
## the optional field of the returned record is not set.
function BrokerComm::table_lookup%(t: BrokerComm::Data, key: BrokerComm::Data%): BrokerComm::Data
%{
auto& table = bro_broker::require_data_type<broker::table>(t->AsRecordVal(),
TYPE_TABLE, frame);
auto& k = bro_broker::opaque_field_to_data(key->AsRecordVal(), frame);
auto it = table.find(k);
if ( it == table.end() )
return new RecordVal(BifType::Record::BrokerComm::Data);
else
return bro_broker::make_data_val(it->second);
%}
## Create an iterator for a table. Note that this makes a copy of the table
## internally to ensure the iterator is always valid.
##
## t: the table to iterate over.
##
## Returns: an iterator.
function BrokerComm::table_iterator%(t: BrokerComm::Data%): opaque of BrokerComm::TableIterator
%{
return new bro_broker::TableIterator(t->AsRecordVal(), TYPE_TABLE, frame);
%}
## Check if there are no more elements to iterate over.
##
## it: an iterator.
##
## Returns: true if there are no more elements to iterator over, i.e.
## the iterator is one-past-the-final-element.
function BrokerComm::table_iterator_last%(it: opaque of BrokerComm::TableIterator%): bool
%{
auto ti = static_cast<bro_broker::TableIterator*>(it);
return new Val(ti->it == ti->dat.end(), TYPE_BOOL);
%}
## Advance an iterator.
##
## it: an iterator.
##
## Returns: true if the iterator, after advancing, still references an element
## in the collection. False if the iterator, after advancing, is
## one-past-the-final-element.
function BrokerComm::table_iterator_next%(it: opaque of BrokerComm::TableIterator%): bool
%{
auto ti = static_cast<bro_broker::TableIterator*>(it);
if ( ti->it == ti->dat.end() )
return new Val(false, TYPE_BOOL);
++ti->it;
return new Val(ti->it != ti->dat.end(), TYPE_BOOL);
%}
## Retrieve the data at an iterator's current position.
##
## it: an iterator.
##
## Returns: element in the collection that the iterator currently references.
function BrokerComm::table_iterator_value%(it: opaque of BrokerComm::TableIterator%): BrokerComm::TableItem
%{
auto ti = static_cast<bro_broker::TableIterator*>(it);
auto rval = new RecordVal(BifType::Record::BrokerComm::TableItem);
auto key_val = new RecordVal(BifType::Record::BrokerComm::Data);
auto val_val = new RecordVal(BifType::Record::BrokerComm::Data);
rval->Assign(0, key_val);
rval->Assign(1, val_val);
if ( ti->it == ti->dat.end() )
{
reporter->PushLocation(frame->GetCall()->GetLocationInfo());
reporter->Warning("attempt to retrieve value of invalid table iterator");
reporter->PopLocation();
return rval;
}
key_val->Assign(0, new bro_broker::DataVal(ti->it->first));
val_val->Assign(0, new bro_broker::DataVal(ti->it->second));
return rval;
%}
## Create communication data of type "vector".
function BrokerComm::vector_create%(%): BrokerComm::Data
%{
return bro_broker::make_data_val(broker::vector());
%}
## Remove all elements within a vector.
##
## v: the vector to clear.
##
## Returns: always true.
function BrokerComm::vector_clear%(v: BrokerComm::Data%): bool
%{
auto& vec = bro_broker::require_data_type<broker::vector>(v->AsRecordVal(),
TYPE_VECTOR, frame);
vec.clear();
return new Val(true, TYPE_BOOL);
%}
## Get the number of elements within a vector.
##
## v: the vector to query.
##
## Returns: the number of elements in the vector.
function BrokerComm::vector_size%(v: BrokerComm::Data%): count
%{
auto& vec = bro_broker::require_data_type<broker::vector>(v->AsRecordVal(),
TYPE_VECTOR, frame);
return new Val(static_cast<uint64_t>(vec.size()), TYPE_COUNT);
%}
## Insert an element into a vector at a particular position, possibly displacing
## existing elements (insertion always grows the size of the vector by one).
##
## v: the vector to modify.
##
## d: the element to insert.
##
## idx: the index at which to insert the data. If it is greater than the
## current size of the vector, the element is inserted at the end.
##
## Returns: always true.
function BrokerComm::vector_insert%(v: BrokerComm::Data, d: BrokerComm::Data, idx: count%): bool
%{
auto& vec = bro_broker::require_data_type<broker::vector>(v->AsRecordVal(),
TYPE_VECTOR, frame);
auto& item = bro_broker::opaque_field_to_data(d->AsRecordVal(), frame);
idx = min(idx, static_cast<uint64_t>(vec.size()));
vec.insert(vec.begin() + idx, item);
return new Val(true, TYPE_BOOL);
%}
## Replace an element in a vector at a particular position.
##
## v: the vector to modify.
##
## d: the element to insert.
##
## idx: the index to replace.
##
## Returns: the value that was just evicted. If the index was larger than any
## valid index, the optional field of the returned record is not set.
function BrokerComm::vector_replace%(v: BrokerComm::Data, d: BrokerComm::Data, idx: count%): BrokerComm::Data
%{
auto& vec = bro_broker::require_data_type<broker::vector>(v->AsRecordVal(),
TYPE_VECTOR, frame);
auto& item = bro_broker::opaque_field_to_data(d->AsRecordVal(), frame);
if ( idx >= vec.size() )
return new RecordVal(BifType::Record::BrokerComm::Data);
auto rval = bro_broker::make_data_val(move(vec[idx]));
vec[idx] = item;
return rval;
%}
## Remove an element from a vector at a particular position.
##
## v: the vector to modify.
##
## idx: the index to remove.
##
## Returns: the value that was just evicted. If the index was larger than any
## valid index, the optional field of the returned record is not set.
function BrokerComm::vector_remove%(v: BrokerComm::Data, idx: count%): BrokerComm::Data
%{
auto& vec = bro_broker::require_data_type<broker::vector>(v->AsRecordVal(),
TYPE_VECTOR, frame);
if ( idx >= vec.size() )
return new RecordVal(BifType::Record::BrokerComm::Data);
auto rval = bro_broker::make_data_val(move(vec[idx]));
vec.erase(vec.begin() + idx);
return rval;
%}
## Lookup an element in a vector at a particular position.
##
## v: the vector to query.
##
## idx: the index to lookup.
##
## Returns: the value at the index. If the index was larger than any
## valid index, the optional field of the returned record is not set.
function BrokerComm::vector_lookup%(v: BrokerComm::Data, idx: count%): BrokerComm::Data
%{
auto& vec = bro_broker::require_data_type<broker::vector>(v->AsRecordVal(),
TYPE_VECTOR, frame);
if ( idx >= vec.size() )
return new RecordVal(BifType::Record::BrokerComm::Data);
return bro_broker::make_data_val(vec[idx]);
%}
## Create an iterator for a vector. Note that this makes a copy of the vector
## internally to ensure the iterator is always valid.
##
## v: the vector to iterate over.
##
## Returns: an iterator.
function BrokerComm::vector_iterator%(v: BrokerComm::Data%): opaque of BrokerComm::VectorIterator
%{
return new bro_broker::VectorIterator(v->AsRecordVal(), TYPE_VECTOR, frame);
%}
## Check if there are no more elements to iterate over.
##
## it: an iterator.
##
## Returns: true if there are no more elements to iterator over, i.e.
## the iterator is one-past-the-final-element.
function BrokerComm::vector_iterator_last%(it: opaque of BrokerComm::VectorIterator%): bool
%{
auto vi = static_cast<bro_broker::VectorIterator*>(it);
return new Val(vi->it == vi->dat.end(), TYPE_BOOL);
%}
## Advance an iterator.
##
## it: an iterator.
##
## Returns: true if the iterator, after advancing, still references an element
## in the collection. False if the iterator, after advancing, is
## one-past-the-final-element.
function BrokerComm::vector_iterator_next%(it: opaque of BrokerComm::VectorIterator%): bool
%{
auto vi = static_cast<bro_broker::VectorIterator*>(it);
if ( vi->it == vi->dat.end() )
return new Val(false, TYPE_BOOL);
++vi->it;
return new Val(vi->it != vi->dat.end(), TYPE_BOOL);
%}
## Retrieve the data at an iterator's current position.
##
## it: an iterator.
##
## Returns: element in the collection that the iterator currently references.
function BrokerComm::vector_iterator_value%(it: opaque of BrokerComm::VectorIterator%): BrokerComm::Data
%{
auto vi = static_cast<bro_broker::VectorIterator*>(it);
auto rval = new RecordVal(BifType::Record::BrokerComm::Data);
if ( vi->it == vi->dat.end() )
{
reporter->PushLocation(frame->GetCall()->GetLocationInfo());
reporter->Warning("attempt to retrieve value of invalid vector iterator");
reporter->PopLocation();
return rval;
}
rval->Assign(0, new bro_broker::DataVal(*vi->it));
return rval;
%}
## Create communication data of type "record".
##
## sz: the number of fields in the record.
##
## Returns: record data, with all fields uninitialized.
function BrokerComm::record_create%(sz: count%): BrokerComm::Data
%{
return bro_broker::make_data_val(broker::record(std::vector<broker::record::field>(sz)));
%}
## Get the number of fields within a record.
##
## r: the record to query.
##
## Returns: the number of fields in the record.
function BrokerComm::record_size%(r: BrokerComm::Data%): count
%{
auto& v = bro_broker::require_data_type<broker::record>(r->AsRecordVal(),
TYPE_RECORD, frame);
return new Val(static_cast<uint64_t>(v.fields.size()), TYPE_COUNT);
%}
## Replace a field in a record at a particular position.
##
## t: the table to modify.
##
## d: the new field value to assign.
##
## idx: the index to replace.
##
## Returns: false if the index was larger than any valid index, else true.
function BrokerComm::record_assign%(r: BrokerComm::Data, d: BrokerComm::Data, idx: count%): bool
%{
auto& v = bro_broker::require_data_type<broker::record>(r->AsRecordVal(),
TYPE_RECORD, frame);
auto& item = bro_broker::opaque_field_to_data(d->AsRecordVal(), frame);
if ( idx >= v.fields.size() )
return new Val(false, TYPE_BOOL);
v.fields[idx] = item;
return new Val(true, TYPE_BOOL);
%}
## Lookup a field in a record at a particular position.
##
## r: the record to query.
##
## idx: the index to lookup.
##
## Returns: the value at the index. The optional field of the returned record
## may not be set if the field of the record has no value or if the
## the index was not valid.
function BrokerComm::record_lookup%(r: BrokerComm::Data, idx: count%): BrokerComm::Data
%{
auto& v = bro_broker::require_data_type<broker::record>(r->AsRecordVal(),
TYPE_RECORD, frame);
if ( idx >= v.size() )
return new RecordVal(BifType::Record::BrokerComm::Data);
if ( ! v.fields[idx] )
return new RecordVal(BifType::Record::BrokerComm::Data);
return bro_broker::make_data_val(*v.fields[idx]);
%}
## Create an iterator for a record. Note that this makes a copy of the record
## internally to ensure the iterator is always valid.
##
## r: the record to iterate over.
##
## Returns: an iterator.
function BrokerComm::record_iterator%(r: BrokerComm::Data%): opaque of BrokerComm::RecordIterator
%{
return new bro_broker::RecordIterator(r->AsRecordVal(), TYPE_RECORD, frame);
%}
## Check if there are no more elements to iterate over.
##
## it: an iterator.
##
## Returns: true if there are no more elements to iterator over, i.e.
## the iterator is one-past-the-final-element.
function BrokerComm::record_iterator_last%(it: opaque of BrokerComm::RecordIterator%): bool
%{
auto ri = static_cast<bro_broker::RecordIterator*>(it);
return new Val(ri->it == ri->dat.fields.end(), TYPE_BOOL);
%}
## Advance an iterator.
##
## it: an iterator.
##
## Returns: true if the iterator, after advancing, still references an element
## in the collection. False if the iterator, after advancing, is
## one-past-the-final-element.
function BrokerComm::record_iterator_next%(it: opaque of BrokerComm::RecordIterator%): bool
%{
auto ri = static_cast<bro_broker::RecordIterator*>(it);
if ( ri->it == ri->dat.fields.end() )
return new Val(false, TYPE_BOOL);
++ri->it;
return new Val(ri->it != ri->dat.fields.end(), TYPE_BOOL);
%}
## Retrieve the data at an iterator's current position.
##
## it: an iterator.
##
## Returns: element in the collection that the iterator currently references.
function BrokerComm::record_iterator_value%(it: opaque of BrokerComm::RecordIterator%): BrokerComm::Data
%{
auto ri = static_cast<bro_broker::RecordIterator*>(it);
auto rval = new RecordVal(BifType::Record::BrokerComm::Data);
if ( ri->it == ri->dat.fields.end() )
{
reporter->PushLocation(frame->GetCall()->GetLocationInfo());
reporter->Warning("attempt to retrieve value of invalid record iterator");
reporter->PopLocation();
return rval;
}
if ( ! *ri->it )
return rval; // field isn't set
rval->Assign(0, new bro_broker::DataVal(**ri->it));
return rval;
%}

211
src/broker/messaging.bif Normal file
View file

@ -0,0 +1,211 @@
##! Functions for peering and various messaging patterns (e.g. print/log/event).
%%{
#include "broker/Manager.h"
#include "logging/Manager.h"
%%}
module BrokerComm;
type BrokerComm::SendFlags: record;
type BrokerComm::EventArgs: record;
## Used to handle remote print messages from peers that call
## :bro:see:`BrokerComm::print`.
event BrokerComm::print_handler%(msg: string%);
## Print a simple message to any interested peers. The receiver can use
## :bro:see:`BrokerComm::print_handler` to handle messages.
##
## topic: a topic associated with the printed message.
##
## msg: the print message to send to peers.
##
## flags: tune the behavior of how the message is sent.
##
## Returns: true if the message is sent.
function BrokerComm::print%(topic: string, msg: string,
flags: SendFlags &default = SendFlags()%): bool
%{
auto rval = broker_mgr->Print(topic->CheckString(), msg->CheckString(),
flags);
return new Val(rval, TYPE_BOOL);
%}
## Register interest in all peer print messages that use a certain topic prefix.
## use :bro:see:`BrokerComm::print_handler` to handle received messages.
##
## topic_prefix: a prefix to match against remote message topics.
## e.g. an empty prefix matches everything and "a" matches
## "alice" and "amy" but not "bob".
##
## Returns: true if it's a new print subscription and it is now registered.
function BrokerComm::subscribe_to_prints%(topic_prefix: string%): bool
%{
auto rval = broker_mgr->SubscribeToPrints(topic_prefix->CheckString());
return new Val(rval, TYPE_BOOL);
%}
## Unregister interest in all peer print messages that use a topic prefix.
##
## topic_prefix: a prefix previously supplied to a successful call to
## :bro:see:`BrokerComm::subscribe_to_prints`.
##
## Returns: true if interest in the topic prefix is no longer advertised.
function BrokerComm::unsubscribe_to_prints%(topic_prefix: string%): bool
%{
auto rval = broker_mgr->UnsubscribeToPrints(topic_prefix->CheckString());
return new Val(rval, TYPE_BOOL);
%}
## Create a data structure that may be used to send a remote event via
## :bro:see:`BrokerComm::event`.
##
## args: an event, followed by a list of argument values that may be used
## to call it.
##
## Returns: opaque communication data that may be used to send a remote event.
function BrokerComm::event_args%(...%): BrokerComm::EventArgs
%{
auto rval = broker_mgr->MakeEventArgs(@ARGS@);
return rval;
%}
## Send an event to any interested peers.
##
## topic: a topic associated with the event message.
##
## args: event arguments as made by :bro:see:`BrokerComm::event_args`.
##
## flags: tune the behavior of how the message is sent.
##
## Returns: true if the message is sent.
function BrokerComm::event%(topic: string, args: BrokerComm::EventArgs,
flags: SendFlags &default = SendFlags()%): bool
%{
auto rval = broker_mgr->Event(topic->CheckString(), args->AsRecordVal(),
flags);
return new Val(rval, TYPE_BOOL);
%}
## Automatically send an event to any interested peers whenever it is
## locally dispatched (e.g. using "event my_event(...);" in a script).
##
## topic: a topic string associated with the event message.
## Peers advertise interest by registering a subscription to some prefix
## of this topic name.
##
## ev: a Bro event value.
##
## flags: tune the behavior of how the message is send.
##
## Returns: true if automatic event sending is now enabled.
function BrokerComm::auto_event%(topic: string, ev: any,
flags: SendFlags &default = SendFlags()%): bool
%{
auto rval = broker_mgr->AutoEvent(topic->CheckString(), ev, flags);
return new Val(rval, TYPE_BOOL);
%}
## Stop automatically sending an event to peers upon local dispatch.
##
## topic: a topic originally given to :bro:see:`BrokerComm::auto_event`.
##
## ev: an event originally given to :bro:see:`BrokerComm::auto_event`.
##
## Returns: true if automatic events will no occur for the topic/event pair.
function BrokerComm::auto_event_stop%(topic: string, ev: any%): bool
%{
auto rval = broker_mgr->AutoEventStop(topic->CheckString(), ev);
return new Val(rval, TYPE_BOOL);
%}
## Register interest in all peer event messages that use a certain topic prefix.
##
## topic_prefix: a prefix to match against remote message topics.
## e.g. an empty prefix matches everything and "a" matches
## "alice" and "amy" but not "bob".
##
## Returns: true if it's a new event subscription and it is now registered.
function BrokerComm::subscribe_to_events%(topic_prefix: string%): bool
%{
auto rval = broker_mgr->SubscribeToEvents(topic_prefix->CheckString());
return new Val(rval, TYPE_BOOL);
%}
## Unregister interest in all peer event messages that use a topic prefix.
##
## topic_prefix: a prefix previously supplied to a successful call to
## :bro:see:`BrokerComm::subscribe_to_events`.
##
## Returns: true if interest in the topic prefix is no longer advertised.
function BrokerComm::unsubscribe_to_events%(topic_prefix: string%): bool
%{
auto rval = broker_mgr->UnsubscribeToEvents(topic_prefix->CheckString());
return new Val(rval, TYPE_BOOL);
%}
## Enable remote logs for a given log stream.
##
## id: the log stream to enable remote logs for.
##
## flags: tune the behavior of how log entry messages are sent.
##
## Returns: true if remote logs are enabled for the stream.
function
BrokerComm::enable_remote_logs%(id: Log::ID,
flags: SendFlags &default = SendFlags()%): bool
%{
auto rval = log_mgr->EnableRemoteLogs(id->AsEnumVal(),
bro_broker::Manager::send_flags_to_int(flags));
return new Val(rval, TYPE_BOOL);
%}
## Disable remote logs for a given log stream.
##
## id: the log stream to disable remote logs for.
##
## Returns: true if remote logs are disabled for the stream.
function BrokerComm::disable_remote_logs%(id: Log::ID%): bool
%{
auto rval = log_mgr->DisableRemoteLogs(id->AsEnumVal());
return new Val(rval, TYPE_BOOL);
%}
## Returns: true if remote logs are enabled for the given stream.
function BrokerComm::remote_logs_enabled%(id: Log::ID%): bool
%{
auto rval = log_mgr->RemoteLogsAreEnabled(id->AsEnumVal());
return new Val(rval, TYPE_BOOL);
%}
## Register interest in all peer log messages that use a certain topic prefix.
## Logs are implicitly sent with topic "bro/log/<stream-name>" and the
## receiving side processes them through the logging framework as usual.
##
## topic_prefix: a prefix to match against remote message topics.
## e.g. an empty prefix matches everything and "a" matches
## "alice" and "amy" but not "bob".
##
## Returns: true if it's a new log subscription and it is now registered.
function BrokerComm::subscribe_to_logs%(topic_prefix: string%): bool
%{
auto rval = broker_mgr->SubscribeToLogs(topic_prefix->CheckString());
return new Val(rval, TYPE_BOOL);
%}
## Unregister interest in all peer log messages that use a topic prefix.
## Logs are implicitly sent with topic "bro/log/<stream-name>" and the
## receiving side processes them through the logging framework as usual.
##
## topic_prefix: a prefix previously supplied to a successful call to
## :bro:see:`BrokerComm::subscribe_to_logs`.
##
## Returns: true if interest in the topic prefix is no longer advertised.
function BrokerComm::unsubscribe_to_logs%(topic_prefix: string%): bool
%{
auto rval = broker_mgr->UnsubscribeToLogs(topic_prefix->CheckString());
return new Val(rval, TYPE_BOOL);
%}

597
src/broker/store.bif Normal file
View file

@ -0,0 +1,597 @@
##! Functions to interface with broker's distributed data store.
%%{
#include "broker/Manager.h"
#include "broker/Store.h"
#include "broker/Data.h"
#include "Trigger.h"
%%}
module BrokerStore;
type BrokerStore::ExpiryTime: record;
type BrokerStore::QueryResult: record;
type BrokerStore::BackendOptions: record;
## Enumerates the possible storage backends.
enum BackendType %{
MEMORY,
SQLITE,
ROCKSDB,
%}
## Create a master data store which contains key-value pairs.
##
## id: a unique name for the data store.
##
## b: the storage backend to use.
##
## options: tunes how some storage backends operate.
##
## Returns: a handle to the data store.
function BrokerStore::create_master%(id: string, b: BackendType &default = MEMORY,
options: BackendOptions &default = BackendOptions()%): opaque of BrokerStore::Handle
%{
auto id_str = id->CheckString();
auto type = bro_broker::StoreType::MASTER;
auto rval = broker_mgr->LookupStore(id_str, type);
if ( rval )
{
Ref(rval);
return rval;
}
rval = new bro_broker::StoreHandleVal(id_str, type,
static_cast<BifEnum::BrokerStore::BackendType>(b->AsEnum()),
options->AsRecordVal());
auto added = broker_mgr->AddStore(rval);
assert(added);
return rval;
%}
## Create a clone of a master data store which may live with a remote peer.
## A clone automatically synchronizes to the master by automatically receiving
## modifications and applying them locally. Direct modifications are not
## possible, they must be sent through the master store, which then
## automatically broadcasts the changes out to clones. But queries may be made
## directly against the local cloned copy, which may be resolved quicker than
## reaching out to a remote master store.
##
## id: the unique name which identifies the master data store.
##
## b: the storage backend to use.
##
## options: tunes how some storage backends operate.
##
## resync: the interval at which to re-attempt synchronizing with the master
## store should the connection be lost. If the clone has not yet
## synchronized for the first time, updates and queries queue up until
## the synchronization completes. After, if the connection to the
## master store is lost, queries continue to use the clone's version,
## but updates will be lost until the master is once again available.
##
## Returns: a handle to the data store.
function BrokerStore::create_clone%(id: string, b: BackendType &default = MEMORY,
options: BackendOptions &default = BackendOptions(),
resync: interval &default = 1sec%): opaque of BrokerStore::Handle
%{
auto id_str = id->CheckString();
auto type = bro_broker::StoreType::CLONE;
auto rval = broker_mgr->LookupStore(id_str, type);
if ( rval )
{
Ref(rval);
return rval;
}
rval = new bro_broker::StoreHandleVal(id_str, type,
static_cast<BifEnum::BrokerStore::BackendType>(b->AsEnum()),
options->AsRecordVal(),
std::chrono::duration<double>(resync));
auto added = broker_mgr->AddStore(rval);
assert(added);
return rval;
%}
## Create a frontend interface to an existing master data store that allows
## querying and updating its contents.
##
## id: the unique name which identifies the master data store.
##
## Returns: a handle to the data store.
function BrokerStore::create_frontend%(id: string%): opaque of BrokerStore::Handle
%{
auto id_str = id->CheckString();
auto type = bro_broker::StoreType::FRONTEND;
auto rval = broker_mgr->LookupStore(id_str, type);
if ( rval )
{
Ref(rval);
return rval;
}
rval = new bro_broker::StoreHandleVal(id_str, type, {}, nullptr);
auto added = broker_mgr->AddStore(rval);
assert(added);
return rval;
%}
## Close a data store.
##
## h: a data store handle.
##
## Returns: true if store was valid and is now closed. The handle can no
## longer be used for data store operations.
function BrokerStore::close_by_handle%(h: opaque of BrokerStore::Handle%): bool
%{
auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
if ( ! handle->store )
return new Val(false, TYPE_BOOL);
return new Val(broker_mgr->CloseStore(handle->store->id(),
handle->store_type), TYPE_BOOL);
%}
###########################
# non-blocking update API #
###########################
## Insert a key-value pair in to the store.
##
## h: the handle of the store to modify.
##
## k: the key to insert.
##
## v: the value to insert.
##
## e: the expiration time of the key-value pair.
##
## Returns: false if the store handle was not valid.
function BrokerStore::insert%(h: opaque of BrokerStore::Handle,
k: BrokerComm::Data, v: BrokerComm::Data,
e: BrokerStore::ExpiryTime &default = BrokerStore::ExpiryTime()%): bool
%{
auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
if ( ! handle->store )
return new Val(false, TYPE_BOOL);
auto& key = bro_broker::opaque_field_to_data(k->AsRecordVal(), frame);
auto& val = bro_broker::opaque_field_to_data(v->AsRecordVal(), frame);
using broker::store::expiration_time;
auto abs_expiry_val = e->AsRecordVal()->Lookup(0);
if ( abs_expiry_val )
{
auto expiry = expiration_time(abs_expiry_val->AsTime());
handle->store->insert(key, val, expiry);
return new Val(true, TYPE_BOOL);
}
auto rel_expiry_val = e->AsRecordVal()->Lookup(1);
if ( rel_expiry_val )
{
auto ct = broker::time_point::now().value;
auto expiry = expiration_time(rel_expiry_val->AsInterval(), ct);
handle->store->insert(key, val, expiry);
return new Val(true, TYPE_BOOL);
}
handle->store->insert(key, val);
return new Val(true, TYPE_BOOL);
%}
## Remove a key-value pair from the store.
##
## h: the handle of the store to modify.
##
## k: the key to remove.
##
## Returns: false if the store handle was not valid.
function BrokerStore::erase%(h: opaque of BrokerStore::Handle, k: BrokerComm::Data%): bool
%{
auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
if ( ! handle->store )
return new Val(false, TYPE_BOOL);
auto& key = bro_broker::opaque_field_to_data(k->AsRecordVal(), frame);
handle->store->erase(key);
return new Val(true, TYPE_BOOL);
%}
## Remove all key-value pairs from the store.
##
## h: the handle of the store to modify.
##
## Returns: false if the store handle was not valid.
function BrokerStore::clear%(h: opaque of BrokerStore::Handle%): bool
%{
auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
if ( ! handle->store )
return new Val(false, TYPE_BOOL);
handle->store->clear();
return new Val(true, TYPE_BOOL);
%}
## Increment an integer value in a data store.
##
## h: the handle of the store to modify.
##
## k: the key whose associated value is to be modified.
##
## by: the amount to increment the value by. A non-existent key will first
## create it with an implicit value of zero before incrementing.
##
## Returns: false if the store handle was not valid.
function BrokerStore::increment%(h: opaque of BrokerStore::Handle,
k: BrokerComm::Data, by: int &default = +1%): bool
%{
auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
if ( ! handle->store )
return new Val(false, TYPE_BOOL);
auto& key = bro_broker::opaque_field_to_data(k->AsRecordVal(), frame);
handle->store->increment(key, by);
return new Val(true, TYPE_BOOL);
%}
## Decrement an integer value in a data store.
##
## h: the handle of the store to modify.
##
## k: the key whose associated value is to be modified.
##
## by: the amount to decrement the value by. A non-existent key will first
## create it with an implicit value of zero before decrementing.
##
## Returns: false if the store handle was not valid.
function BrokerStore::decrement%(h: opaque of BrokerStore::Handle,
k: BrokerComm::Data, by: int &default = +1%): bool
%{
auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
if ( ! handle->store )
return new Val(false, TYPE_BOOL);
auto& key = bro_broker::opaque_field_to_data(k->AsRecordVal(), frame);
handle->store->decrement(key, by);
return new Val(true, TYPE_BOOL);
%}
## Add an element to a set value in a data store.
##
## h: the handle of the store to modify.
##
## k: the key whose associated value is to be modified.
##
## element: the element to add to the set. A non-existent key will first
## create it with an implicit empty set value before modifying.
##
## Returns: false if the store handle was not valid.
function BrokerStore::add_to_set%(h: opaque of BrokerStore::Handle,
k: BrokerComm::Data, element: BrokerComm::Data%): bool
%{
auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
if ( ! handle->store )
return new Val(false, TYPE_BOOL);
auto& key = bro_broker::opaque_field_to_data(k->AsRecordVal(), frame);
auto& ele = bro_broker::opaque_field_to_data(element->AsRecordVal(), frame);
handle->store->add_to_set(key, ele);
return new Val(true, TYPE_BOOL);
%}
## Remove an element from a set value in a data store.
##
## h: the handle of the store to modify.
##
## k: the key whose associated value is to be modified.
##
## element: the element to remove from the set. A non-existent key will
## implicitly create an empty set value associated with the key.
##
## Returns: false if the store handle was not valid.
function BrokerStore::remove_from_set%(h: opaque of BrokerStore::Handle,
k: BrokerComm::Data, element: BrokerComm::Data%): bool
%{
auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
if ( ! handle->store )
return new Val(false, TYPE_BOOL);
auto& key = bro_broker::opaque_field_to_data(k->AsRecordVal(), frame);
auto& ele = bro_broker::opaque_field_to_data(element->AsRecordVal(), frame);
handle->store->remove_from_set(key, ele);
return new Val(true, TYPE_BOOL);
%}
## Add a new item to the head of a vector value in a data store.
##
## h: the handle of store to modify.
##
## k: the key whose associated value is to be modified.
##
## item: the element to insert in to the vector. A non-existent key will first
## create empty vector value before modifying.
##
## Returns: the handle of store to modify.
function BrokerStore::push_left%(h: opaque of BrokerStore::Handle, k: BrokerComm::Data,
items: BrokerComm::DataVector%): bool
%{
auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
if ( ! handle->store )
return new Val(false, TYPE_BOOL);
auto& key = bro_broker::opaque_field_to_data(k->AsRecordVal(), frame);
broker::vector items_vector;
auto items_vv = items->AsVector();
for ( auto i = 0u; i < items_vv->size(); ++i )
{
auto& item = bro_broker::opaque_field_to_data((*items_vv)[i]->AsRecordVal(),
frame);
items_vector.emplace_back(item);
}
handle->store->push_left(key, move(items_vector));
return new Val(true, TYPE_BOOL);
%}
## Add a new item to the tail of a vector value in a data store.
##
## h: the handle of store to modify.
##
## k: the key whose associated value is to be modified.
##
## item: the element to insert in to the vector. A non-existent key will first
## create empty vector value before modifying.
##
## Returns: the handle of store to modify.
function BrokerStore::push_right%(h: opaque of BrokerStore::Handle, k: BrokerComm::Data,
items: BrokerComm::DataVector%): bool
%{
auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
if ( ! handle->store )
return new Val(false, TYPE_BOOL);
auto& key = bro_broker::opaque_field_to_data(k->AsRecordVal(), frame);
broker::vector items_vector;
auto items_vv = items->AsVector();
for ( auto i = 0u; i < items_vv->size(); ++i )
{
auto& item = bro_broker::opaque_field_to_data((*items_vv)[i]->AsRecordVal(),
frame);
items_vector.emplace_back(item);
}
handle->store->push_right(key, move(items_vector));
return new Val(true, TYPE_BOOL);
%}
##########################
# non-blocking query API #
##########################
%%{
static bool prepare_for_query(Val* opaque, Frame* frame,
bro_broker::StoreHandleVal** handle,
double* timeout,
bro_broker::StoreQueryCallback** cb)
{
*handle = static_cast<bro_broker::StoreHandleVal*>(opaque);
if ( ! (*handle)->store )
{
reporter->PushLocation(frame->GetCall()->GetLocationInfo());
reporter->Error("BrokerStore query has an invalid data store");
reporter->PopLocation();
return false;
}
Trigger* trigger = frame->GetTrigger();
if ( ! trigger )
{
reporter->PushLocation(frame->GetCall()->GetLocationInfo());
reporter->Error("BrokerStore queries can only be called inside when-condition");
reporter->PopLocation();
return false;
}
*timeout = trigger->TimeoutValue();
if ( *timeout < 0 )
{
reporter->PushLocation(frame->GetCall()->GetLocationInfo());
reporter->Error("BrokerStore queries must specify a timeout block");
reporter->PopLocation();
return false;
}
frame->SetDelayed();
trigger->Hold();
*cb = new bro_broker::StoreQueryCallback(trigger, frame->GetCall(),
(*handle)->store->id(),
(*handle)->store_type);
broker_mgr->TrackStoreQuery(*cb);
return true;
}
%%}
## Pop the head of a data store vector value.
##
## h: the handle of the store to query.
##
## k: the key associated with the vector to modify.
##
## Returns: the result of the query.
function BrokerStore::pop_left%(h: opaque of BrokerStore::Handle,
k: BrokerComm::Data%): BrokerStore::QueryResult
%{
if ( ! broker_mgr->Enabled() )
return bro_broker::query_result();
Val* key = k->AsRecordVal()->Lookup(0);
if ( ! key )
return bro_broker::query_result();
double timeout;
bro_broker::StoreQueryCallback* cb;
bro_broker::StoreHandleVal* handle;
if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) )
return bro_broker::query_result();
handle->store->pop_left(static_cast<bro_broker::DataVal*>(key)->data,
std::chrono::duration<double>(timeout), cb);
return 0;
%}
## Pop the tail of a data store vector value.
##
## h: the handle of the store to query.
##
## k: the key associated with the vector to modify.
##
## Returns: the result of the query.
function BrokerStore::pop_right%(h: opaque of BrokerStore::Handle,
k: BrokerComm::Data%): BrokerStore::QueryResult
%{
if ( ! broker_mgr->Enabled() )
return bro_broker::query_result();
Val* key = k->AsRecordVal()->Lookup(0);
if ( ! key )
return bro_broker::query_result();
double timeout;
bro_broker::StoreQueryCallback* cb;
bro_broker::StoreHandleVal* handle;
if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) )
return bro_broker::query_result();
handle->store->pop_right(static_cast<bro_broker::DataVal*>(key)->data,
std::chrono::duration<double>(timeout), cb);
return 0;
%}
## Lookup the value associated with a key in a data store.
##
## h: the handle of the store to query.
##
## k: the key to lookup.
##
## Returns: the result of the query.
function BrokerStore::lookup%(h: opaque of BrokerStore::Handle,
k: BrokerComm::Data%): BrokerStore::QueryResult
%{
if ( ! broker_mgr->Enabled() )
return bro_broker::query_result();
Val* key = k->AsRecordVal()->Lookup(0);
if ( ! key )
return bro_broker::query_result();
double timeout;
bro_broker::StoreQueryCallback* cb;
bro_broker::StoreHandleVal* handle;
if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) )
return bro_broker::query_result();
handle->store->lookup(static_cast<bro_broker::DataVal*>(key)->data,
std::chrono::duration<double>(timeout), cb);
return 0;
%}
## Check if a data store contains a given key.
##
## h: the handle of the store to query.
##
## k: the key to check for existence.
##
## Returns: the result of the query (uses :bro:see:`BrokerComm::BOOL`).
function BrokerStore::exists%(h: opaque of BrokerStore::Handle,
k: BrokerComm::Data%): BrokerStore::QueryResult
%{
if ( ! broker_mgr->Enabled() )
return bro_broker::query_result();
Val* key = k->AsRecordVal()->Lookup(0);
if ( ! key )
return bro_broker::query_result();
double timeout;
bro_broker::StoreQueryCallback* cb;
bro_broker::StoreHandleVal* handle;
if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) )
return bro_broker::query_result();
handle->store->exists(static_cast<bro_broker::DataVal*>(key)->data,
std::chrono::duration<double>(timeout), cb);
return 0;
%}
## Retrieve all keys in a data store.
##
## h: the handle of the store to query.
##
## Returns: the result of the query (uses :bro:see:`BrokerComm::VECTOR`).
function BrokerStore::keys%(h: opaque of BrokerStore::Handle%): BrokerStore::QueryResult
%{
double timeout;
bro_broker::StoreQueryCallback* cb;
bro_broker::StoreHandleVal* handle;
if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) )
return bro_broker::query_result();
handle->store->keys(std::chrono::duration<double>(timeout), cb);
return 0;
%}
## Get the number of key-value pairs in a data store.
##
## h: the handle of the store to query.
##
## Returns: the result of the query (uses :bro:see:`BrokerComm::COUNT`).
function BrokerStore::size%(h: opaque of BrokerStore::Handle%): BrokerStore::QueryResult
%{
if ( ! broker_mgr->Enabled() )
return bro_broker::query_result();
double timeout;
bro_broker::StoreQueryCallback* cb;
bro_broker::StoreHandleVal* handle;
if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) )
return bro_broker::query_result();
handle->store->size(std::chrono::duration<double>(timeout), cb);
return 0;
%}

42
src/file_analysis/analyzer/x509/X509.cc
View file

@ -104,13 +104,35 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
len = BIO_gets(bio, buf, sizeof(buf));
pX509Cert->Assign(2, new StringVal(len, buf));
BIO_reset(bio);
X509_NAME *subject_name = X509_get_subject_name(ssl_cert);
// extract the most specific (last) common name from the subject
int namepos = -1;
for ( ;; )
{
int j = X509_NAME_get_index_by_NID(subject_name, NID_commonName, namepos);
if ( j == -1 )
break;
namepos = j;
}
if ( namepos != -1 )
{
// we found a common name
ASN1_STRING_print(bio, X509_NAME_ENTRY_get_data(X509_NAME_get_entry(subject_name, namepos)));
len = BIO_gets(bio, buf, sizeof(buf));
pX509Cert->Assign(4, new StringVal(len, buf));
BIO_reset(bio);
}
X509_NAME_print_ex(bio, X509_get_issuer_name(ssl_cert), 0, XN_FLAG_RFC2253);
len = BIO_gets(bio, buf, sizeof(buf));
pX509Cert->Assign(3, new StringVal(len, buf));
BIO_free(bio);
pX509Cert->Assign(4, new Val(GetTimeFromAsn1(X509_get_notBefore(ssl_cert)), TYPE_TIME));
pX509Cert->Assign(5, new Val(GetTimeFromAsn1(X509_get_notAfter(ssl_cert)), TYPE_TIME));
pX509Cert->Assign(5, new Val(GetTimeFromAsn1(X509_get_notBefore(ssl_cert)), TYPE_TIME));
pX509Cert->Assign(6, new Val(GetTimeFromAsn1(X509_get_notAfter(ssl_cert)), TYPE_TIME));
// we only read 255 bytes because byte 256 is always 0.
// if the string is longer than 255, that will be our null-termination,
@ -118,28 +140,28 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
if ( ! i2t_ASN1_OBJECT(buf, 255, ssl_cert->cert_info->key->algor->algorithm) )
buf[0] = 0;
pX509Cert->Assign(6, new StringVal(buf));
pX509Cert->Assign(7, new StringVal(buf));
if ( ! i2t_ASN1_OBJECT(buf, 255, ssl_cert->sig_alg->algorithm) )
buf[0] = 0;
pX509Cert->Assign(7, new StringVal(buf));
pX509Cert->Assign(8, new StringVal(buf));
// Things we can do when we have the key...
EVP_PKEY *pkey = X509_extract_key(ssl_cert);
if ( pkey != NULL )
{
if ( pkey->type == EVP_PKEY_DSA )
pX509Cert->Assign(8, new StringVal("dsa"));
pX509Cert->Assign(9, new StringVal("dsa"));
else if ( pkey->type == EVP_PKEY_RSA )
{
pX509Cert->Assign(8, new StringVal("rsa"));
pX509Cert->Assign(9, new StringVal("rsa"));
char *exponent = BN_bn2dec(pkey->pkey.rsa->e);
if ( exponent != NULL )
{
pX509Cert->Assign(10, new StringVal(exponent));
pX509Cert->Assign(11, new StringVal(exponent));
OPENSSL_free(exponent);
exponent = NULL;
}
@ -147,14 +169,14 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
#ifndef OPENSSL_NO_EC
else if ( pkey->type == EVP_PKEY_EC )
{
pX509Cert->Assign(8, new StringVal("ecdsa"));
pX509Cert->Assign(11, KeyCurve(pkey));
pX509Cert->Assign(9, new StringVal("ecdsa"));
pX509Cert->Assign(12, KeyCurve(pkey));
}
#endif
unsigned int length = KeyLength(pkey);
if ( length > 0 )
pX509Cert->Assign(9, new Val(length, TYPE_COUNT));
pX509Cert->Assign(10, new Val(length, TYPE_COUNT));
EVP_PKEY_free(pkey);
}

11
src/iosource/PktSrc.cc
View file

@ -13,6 +13,15 @@
using namespace iosource;
PktSrc::Properties::Properties()
{
selectable_fd = -1;
link_type = -1;
hdr_size = -1;
netmask = NETMASK_UNKNOWN;
is_live = false;
}
PktSrc::PktSrc()
{
have_packet = false;
@ -50,7 +59,7 @@ int PktSrc::LinkType() const
uint32 PktSrc::Netmask() const
{
return IsOpen() ? props.netmask : PCAP_NETMASK_UNKNOWN;
return IsOpen() ? props.netmask : NETMASK_UNKNOWN;
}
bool PktSrc::IsError() const

24
src/iosource/PktSrc.h
View file

@ -16,6 +16,8 @@ namespace iosource {
*/
class PktSrc : public IOSource {
public:
static const int NETMASK_UNKNOWN = 0xffffffff;
/**
* Struct for returning statistics on a packet source.
*/
@ -36,7 +38,12 @@ public:
*/
unsigned int link;
Stats() { received = dropped = link = 0; }
/**
* Bytes received by source after filtering (w/o drops).
*/
uint64 bytes_received;
Stats() { received = dropped = link = bytes_received = 0; }
};
/**
@ -67,7 +74,7 @@ public:
/**
* Returns the netmask associated with the source, or \c
* PCAP_NETMASK_UNKNOWN if unknown.
* NETMASK_UNKNOWN if unknown.
*/
uint32 Netmask() const;
@ -253,8 +260,8 @@ protected:
int hdr_size;
/**
* The netmask associated with the source, or \c
* PCAP_NETMASK_UNKNOWN if unknown.
* Returns the netmask associated with the source, or \c
* NETMASK_UNKNOWN if unknown.
*/
uint32 netmask;
@ -264,14 +271,7 @@ protected:
*/
bool is_live;
Properties()
{
selectable_fd = -1;
link_type = -1;
hdr_size = -1;
netmask = PCAP_NETMASK_UNKNOWN;
is_live = false;
}
Properties();
};
/**

15
src/iosource/pcap/Source.cc
View file

@ -77,6 +77,12 @@ void PcapSource::OpenLive()
props.netmask = 0xffffff00;
}
#ifdef PCAP_NETMASK_UNKNOWN
// Defined in libpcap >= 1.1.1
if ( props.netmask == PCAP_NETMASK_UNKNOWN )
props.netmask = PktSrc::NETMASK_UNKNOWN;
#endif
// We use the smallest time-out possible to return almost immediately if
// no packets are available. (We can't use set_nonblocking() as it's
// broken on FreeBSD: even when select() indicates that we can read
@ -174,6 +180,8 @@ bool PcapSource::ExtractNextPacket(Packet* pkt)
last_hdr = current_hdr;
last_data = data;
++stats.received;
stats.bytes_received += current_hdr.len;
return true;
}
@ -213,7 +221,7 @@ bool PcapSource::SetFilter(int index)
#ifndef HAVE_LINUX
// Linux doesn't clear counters when resetting filter.
stats.received = stats.dropped = stats.link = 0;
stats.received = stats.dropped = stats.link = stats.bytes_received = 0;
#endif
return true;
@ -224,7 +232,7 @@ void PcapSource::Statistics(Stats* s)
char errbuf[PCAP_ERRBUF_SIZE];
if ( ! (props.is_live && pd) )
s->received = s->dropped = s->link = 0;
s->received = s->dropped = s->link = s->bytes_received = 0;
else
{
@ -232,7 +240,7 @@ void PcapSource::Statistics(Stats* s)
if ( pcap_stats(pd, &pstat) < 0 )
{
PcapError();
s->received = s->dropped = s->link = 0;
s->received = s->dropped = s->link = s->bytes_received = 0;
}
else
@ -243,6 +251,7 @@ void PcapSource::Statistics(Stats* s)
}
s->received = stats.received;
s->bytes_received = stats.bytes_received;
if ( ! props.is_live )
s->dropped = 0;

67
src/logging/Manager.cc
View file

@ -16,6 +16,10 @@
#include "WriterBackend.h"
#include "logging.bif.h"
#ifdef ENABLE_BROKER
#include "broker/Manager.h"
#endif
using namespace logging;
struct Manager::Filter {
@ -69,6 +73,11 @@ struct Manager::Stream {
WriterMap writers; // Writers indexed by id/path pair.
#ifdef ENABLE_BROKER
bool enable_remote;
int remote_flags;
#endif
~Stream();
};
@ -287,6 +296,11 @@ bool Manager::CreateStream(EnumVal* id, RecordVal* sval)
streams[idx]->event = event ? event_registry->Lookup(event->Name()) : 0;
streams[idx]->columns = columns->Ref()->AsRecordType();
#ifdef ENABLE_BROKER
streams[idx]->enable_remote = internal_val("Log::enable_remote_logging")->AsBool();
streams[idx]->remote_flags = broker::PEERS;
#endif
DBG_LOG(DBG_LOGGING, "Created new logging stream '%s', raising event %s",
streams[idx]->name.c_str(), event ? streams[idx]->event->Name() : "<none>");
@ -828,6 +842,12 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
#endif
}
#ifdef ENABLE_BROKER
if ( stream->enable_remote &&
! broker_mgr->Log(id, columns, stream->columns, stream->remote_flags) )
stream->enable_remote = false;
#endif
Unref(columns);
if ( error )
@ -1206,6 +1226,53 @@ void Manager::Terminate()
}
}
#ifdef ENABLE_BROKER
bool Manager::EnableRemoteLogs(EnumVal* stream_id, int flags)
{
auto stream = FindStream(stream_id);
if ( ! stream )
return false;
stream->enable_remote = true;
stream->remote_flags = flags;
return true;
}
bool Manager::DisableRemoteLogs(EnumVal* stream_id)
{
auto stream = FindStream(stream_id);
if ( ! stream )
return false;
stream->enable_remote = false;
return true;
}
bool Manager::RemoteLogsAreEnabled(EnumVal* stream_id)
{
auto stream = FindStream(stream_id);
if ( ! stream )
return false;
return stream->enable_remote;
}
RecordType* Manager::StreamColumns(EnumVal* stream_id)
{
auto stream = FindStream(stream_id);
if ( ! stream )
return nullptr;
return stream->columns;
}
#endif
// Timer which on dispatching rotates the filter.
class RotationTimer : public Timer {
public:

28
src/logging/Manager.h
View file

@ -157,6 +157,34 @@ public:
*/
void Terminate();
#ifdef ENABLE_BROKER
/**
* Enable remote logs for a given stream.
* @param stream_id the stream to enable remote logs for.
* @param flags tune behavior of how log entries are sent to peer endpoints.
* @return true if remote logs are enabled.
*/
bool EnableRemoteLogs(EnumVal* stream_id, int flags);
/**
* Disable remote logs for a given stream.
* @param stream_id the stream to disable remote logs for.
* @return true if remote logs are disabled.
*/
bool DisableRemoteLogs(EnumVal* stream_id);
/**
* @return true if remote logs are enabled for a given stream.
*/
bool RemoteLogsAreEnabled(EnumVal* stream_id);
/**
* @return the type which corresponds to the columns in a log entry for
* a given log stream.
*/
RecordType* StreamColumns(EnumVal* stream_id);
#endif
protected:
friend class WriterFrontend;
friend class RotationFinishedMessage;

4
src/logging/WriterBackend.cc
View file

@ -84,12 +84,12 @@ bool WriterBackend::WriterInfo::Read(SerializationFormat* fmt)
config.clear();
while ( size )
while ( size-- )
{
string value;
string key;
if ( ! (fmt->Read(&value, "config-value") && fmt->Read(&value, "config-key")) )
if ( ! (fmt->Read(&value, "config-value") && fmt->Read(&key, "config-key")) )
return false;
config.insert(std::make_pair(copy_string(value.c_str()), copy_string(key.c_str())));

18
src/main.cc
View file

@ -63,6 +63,10 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
#include "3rdparty/sqlite3.h"
#ifdef ENABLE_BROKER
#include "broker/Manager.h"
#endif
Brofiler brofiler;
#ifndef HAVE_STRSEP
@ -81,9 +85,6 @@ int perftools_leaks = 0;
int perftools_profile = 0;
#endif
const char* prog;
char* writefile = 0;
name_list prefixes;
DNS_Mgr* dns_mgr;
TimerMgr* timer_mgr;
logging::Manager* log_mgr = 0;
@ -94,6 +95,13 @@ analyzer::Manager* analyzer_mgr = 0;
file_analysis::Manager* file_mgr = 0;
broxygen::Manager* broxygen_mgr = 0;
iosource::Manager* iosource_mgr = 0;
#ifdef ENABLE_BROKER
bro_broker::Manager* broker_mgr = 0;
#endif
const char* prog;
char* writefile = 0;
name_list prefixes;
Stmt* stmts;
EventHandlerPtr net_done = 0;
RuleMatcher* rule_matcher = 0;
@ -851,6 +859,10 @@ int main(int argc, char** argv)
input_mgr = new input::Manager();
file_mgr = new file_analysis::Manager();
#ifdef ENABLE_BROKER
broker_mgr = new bro_broker::Manager();
#endif
plugin_mgr->InitPreScript();
analyzer_mgr->InitPreScript();
file_mgr->InitPreScript();

6
src/parse.y
View file

@ -16,6 +16,7 @@
%token TOK_REMOVE_FROM TOK_RETURN TOK_SCHEDULE TOK_SET
%token TOK_STRING TOK_SUBNET TOK_SWITCH TOK_TABLE
%token TOK_TIME TOK_TIMEOUT TOK_TIMER TOK_TYPE TOK_UNION TOK_VECTOR TOK_WHEN
%token TOK_WHILE
%token TOK_ATTR_ADD_FUNC TOK_ATTR_ENCRYPT TOK_ATTR_DEFAULT
%token TOK_ATTR_OPTIONAL TOK_ATTR_REDEF TOK_ATTR_ROTATE_INTERVAL
@ -1340,6 +1341,11 @@ stmt:
$1->AsForStmt()->AddBody($2);
}
| TOK_WHILE '(' expr ')' stmt
{
$$ = new WhileStmt($3, $5);
}
| TOK_NEXT ';' opt_no_test
{
set_location(@1, @2);

51
src/plugin/Manager.cc
View file

@ -79,18 +79,19 @@ void Manager::SearchDynamicPlugins(const std::string& dir)
std::string name;
std::getline(in, name);
strstrip(name);
string lower_name = strtolower(name);
if ( name.empty() )
reporter->FatalError("empty plugin magic file %s", magic.c_str());
if ( dynamic_plugins.find(name) != dynamic_plugins.end() )
if ( dynamic_plugins.find(lower_name) != dynamic_plugins.end() )
{
DBG_LOG(DBG_PLUGINS, "Found already known plugin %s in %s, ignoring", name.c_str(), dir.c_str());
return;
}
// Record it, so that we can later activate it.
dynamic_plugins.insert(std::make_pair(name, dir));
dynamic_plugins.insert(std::make_pair(lower_name, dir));
DBG_LOG(DBG_PLUGINS, "Found plugin %s in %s", name.c_str(), dir.c_str());
return;
@ -135,7 +136,7 @@ void Manager::SearchDynamicPlugins(const std::string& dir)
bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_not_found)
{
dynamic_plugin_map::iterator m = dynamic_plugins.find(name);
dynamic_plugin_map::iterator m = dynamic_plugins.find(strtolower(name));
if ( m == dynamic_plugins.end() )
{
@ -172,7 +173,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_
// Load {bif,scripts}/__load__.bro automatically.
string init = dir + "scripts/__load__.bro";
string init = dir + "lib/bif/__load__.bro";
if ( is_file(init) )
{
@ -180,7 +181,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_
scripts_to_load.push_back(init);
}
init = dir + "lib/bif/__load__.bro";
init = dir + "scripts/__load__.bro";
if ( is_file(init) )
{
@ -230,7 +231,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_
// Make sure the name the plugin reports is consistent with
// what we expect from its magic file.
if ( string(current_plugin->Name()) != name )
if ( strtolower(current_plugin->Name()) != strtolower(name) )
reporter->FatalError("inconsistent plugin name: %s vs %s",
current_plugin->Name().c_str(), name.c_str());
@ -297,7 +298,7 @@ void Manager::UpdateInputFiles()
static bool plugin_cmp(const Plugin* a, const Plugin* b)
{
return a->Name() < b->Name();
return strtolower(a->Name()) < strtolower(b->Name());
}
void Manager::RegisterPlugin(Plugin *plugin)
@ -318,10 +319,11 @@ void Manager::RegisterBifFile(const char* plugin, bif_init_func c)
{
bif_init_func_map* bifs = BifFilesInternal();
bif_init_func_map::iterator i = bifs->find(plugin);
std::string lower_plugin = strtolower(plugin);
bif_init_func_map::iterator i = bifs->find(lower_plugin);
if ( i == bifs->end() )
i = bifs->insert(std::make_pair(std::string(plugin), new bif_init_func_list())).first;
i = bifs->insert(std::make_pair(lower_plugin, new bif_init_func_list())).first;
i->second->push_back(c);
}
@ -331,7 +333,7 @@ void Manager::InitPreScript()
assert(! init);
for ( plugin_list::iterator i = Manager::ActivePluginsInternal()->begin();
i != Manager::ActivePluginsInternal()->end(); i++ )
i != Manager::ActivePluginsInternal()->end(); i++ )
{
Plugin* plugin = *i;
plugin->DoConfigure();
@ -346,9 +348,9 @@ void Manager::InitBifs()
bif_init_func_map* bifs = BifFilesInternal();
for ( plugin_list::iterator i = Manager::ActivePluginsInternal()->begin();
i != Manager::ActivePluginsInternal()->end(); i++ )
i != Manager::ActivePluginsInternal()->end(); i++ )
{
bif_init_func_map::const_iterator b = bifs->find((*i)->Name());
bif_init_func_map::const_iterator b = bifs->find(strtolower((*i)->Name()));
if ( b != bifs->end() )
{
@ -363,7 +365,7 @@ void Manager::InitPostScript()
assert(init);
for ( plugin_list::iterator i = Manager::ActivePluginsInternal()->begin();
i != Manager::ActivePluginsInternal()->end(); i++ )
i != Manager::ActivePluginsInternal()->end(); i++ )
(*i)->InitPostScript();
}
@ -372,7 +374,7 @@ void Manager::FinishPlugins()
assert(init);
for ( plugin_list::iterator i = Manager::ActivePluginsInternal()->begin();
i != Manager::ActivePluginsInternal()->end(); i++ )
i != Manager::ActivePluginsInternal()->end(); i++ )
(*i)->Done();
Manager::ActivePluginsInternal()->clear();
@ -397,7 +399,7 @@ Manager::inactive_plugin_list Manager::InactivePlugins() const
for ( plugin_list::const_iterator j = all->begin(); j != all->end(); j++ )
{
if ( (*i).first == (*j)->Name() )
if ( (*i).first == strtolower((*j)->Name()) )
{
found = true;
break;
@ -434,7 +436,7 @@ Manager::bif_init_func_map* Manager::BifFilesInternal()
static bool hook_cmp(std::pair<int, Plugin*> a, std::pair<int, Plugin*> b)
{
if ( a.first == b.first )
return a.second->Name() < a.second->Name();
return strtolower(a.second->Name()) < strtolower(a.second->Name());
// Reverse sort.
return a.first > b.first;
@ -505,13 +507,13 @@ void Manager::DisableHook(HookType hook, Plugin* plugin)
void Manager::RequestEvent(EventHandlerPtr handler, Plugin* plugin)
{
DBG_LOG(DBG_PLUGINS, "Plugin %s requested event %s",
plugin->Name().c_str(), handler->Name());
plugin->Name().c_str(), handler->Name());
handler->SetGenerateAlways();
}
void Manager::RequestBroObjDtor(BroObj* obj, Plugin* plugin)
{
obj->NotifyPluginsOnDtor();
obj->NotifyPluginsOnDtor();
}
int Manager::HookLoadFile(const string& file)
@ -559,31 +561,34 @@ int Manager::HookLoadFile(const string& file)
return rc;
}
Val* Manager::HookCallFunction(const Func* func, val_list* vargs) const
std::pair<bool, Val*> Manager::HookCallFunction(const Func* func, Frame* parent, val_list* vargs) const
{
HookArgumentList args;
if ( HavePluginForHook(META_HOOK_PRE) )
{
args.push_back(HookArgument(func));
args.push_back(HookArgument(parent));
args.push_back(HookArgument(vargs));
MetaHookPre(HOOK_CALL_FUNCTION, args);
}
hook_list* l = hooks[HOOK_CALL_FUNCTION];
Val* v = 0;
std::pair<bool, Val*> v = std::pair<bool, Val*>(false, NULL);
if ( l )
{
for ( hook_list::iterator i = l->begin(); i != l->end(); ++i )
{
Plugin* p = (*i).second;
v = p->HookCallFunction(func, vargs);
v = p->HookCallFunction(func, parent, vargs);
if ( v )
if ( v.first )
break;
}
}
if ( HavePluginForHook(META_HOOK_POST) )
MetaHookPost(HOOK_CALL_FUNCTION, args, HookArgument(v));
@ -671,7 +676,7 @@ void Manager::HookBroObjDtor(void* obj) const
{
HookArgumentList args;
if ( HavePluginForHook(META_HOOK_PRE) )
if ( HavePluginForHook(META_HOOK_PRE) )
{
args.push_back(obj);
MetaHookPre(HOOK_BRO_OBJ_DTOR, args);

3
src/plugin/Manager.h
View file

@ -3,6 +3,7 @@
#ifndef PLUGIN_MANAGER_H
#define PLUGIN_MANAGER_H
#include <utility>
#include <map>
#include "Plugin.h"
@ -244,7 +245,7 @@ public:
* functions and events, it may be any Val and must be ignored). If no
* plugin handled the call, the method returns null.
*/
Val* HookCallFunction(const Func* func, val_list* args) const;
std::pair<bool, Val*> HookCallFunction(const Func* func, Frame *parent, val_list* args) const;
/**
* Hook that filters the queuing of an event.

27
src/plugin/Plugin.cc
View file

@ -83,6 +83,26 @@ void HookArgument::Describe(ODesc* d) const
d->Add("<null>");
break;
case FUNC_RESULT:
if ( func_result.first )
{
if( func_result.second )
func_result.second->Describe(d);
else
d->Add("<null>");
}
else
d->Add("<no result>");
break;
case FRAME:
if ( arg.frame )
d->Add("<frame>");
else
d->Add("<null>");
break;
case FUNC:
if ( arg.func )
d->Add(arg.func->Name());
@ -199,7 +219,7 @@ void Plugin::InitPostScript()
Plugin::bif_item_list Plugin::BifItems() const
{
return bif_items;
return bif_items;
}
void Plugin::Done()
@ -271,9 +291,10 @@ int Plugin::HookLoadFile(const std::string& file, const std::string& ext)
return -1;
}
Val* Plugin::HookCallFunction(const Func* func, val_list* args)
std::pair<bool, Val*> Plugin::HookCallFunction(const Func* func, Frame *parent, val_list* args)
{
return 0;
std::pair<bool, Val*> result(false, NULL);
return result;
}
bool Plugin::HookQueueEvent(Event* event)

46
src/plugin/Plugin.h
View file

@ -5,6 +5,7 @@
#include <list>
#include <string>
#include <utility>
#include "config.h"
#include "analyzer/Component.h"
@ -156,7 +157,7 @@ public:
* Type of the argument.
*/
enum Type {
BOOL, DOUBLE, EVENT, FUNC, INT, STRING, VAL, VAL_LIST, VOID, VOIDP,
BOOL, DOUBLE, EVENT, FRAME, FUNC, FUNC_RESULT, INT, STRING, VAL, VAL_LIST, VOID, VOIDP
};
/**
@ -209,6 +210,16 @@ public:
*/
HookArgument(void* p) { type = VOIDP; arg.voidp = p; }
/**
* Constructor with a function result argument.
*/
HookArgument(std::pair<bool, Val*> fresult) { type = FUNC_RESULT; func_result = fresult; }
/**
* Constructor with a Frame argument.
*/
HookArgument(Frame* f) { type = FRAME; arg.frame = f; }
/**
* Returns the value for a boolen argument. The argument's type must
* match accordingly.
@ -251,6 +262,18 @@ public:
*/
const Val* AsVal() const { assert(type == VAL); return arg.val; }
/**
* Returns the value for a Bro wrapped value argument. The argument's type must
* match accordingly.
*/
const std::pair<bool, Val*> AsFuncResult() const { assert(type == FUNC_RESULT); return func_result; }
/**
* Returns the value for a Bro frame argument. The argument's type must
* match accordingly.
*/
const Frame* AsFrame() const { assert(type == FRAME); return arg.frame; }
/**
* Returns the value for a list of Bro values argument. The argument's type must
* match accordingly.
@ -282,13 +305,16 @@ private:
double double_;
const Event* event;
const Func* func;
const Frame* frame;
int int_;
const Val* val;
const val_list* vals;
const void* voidp;
} arg;
std::string arg_string; // Outside union because it has dtor.
// Outside union because these have dtors.
std::pair<bool, Val*> func_result;
std::string arg_string;
};
typedef std::list<HookArgument> HookArgumentList;
@ -512,7 +538,7 @@ protected:
* actually has code to execute for it. By calling this method, the
* plugin tells Bro to raise the event even if there's no correspondong
* handler; it will then go into HookQueueEvent() just as any other.
*
*
* @param handler The event handler being interested in.
*/
void RequestEvent(EventHandlerPtr handler);
@ -568,13 +594,13 @@ protected:
* in place as long as it ensures matching types and correct reference
* counting.
*
* @return If the plugin handled the call, a Val with +1 reference
* count containixnmg the result value to pass back to the interpreter
* (for void functions and events any \a Val is fine; it will be
* ignored; best to use a \c TYPE_ANY). If the plugin did not handle
* the call, it must return null.
* @return If the plugin handled the call, a std::pair<bool, Val*> with the
* processed flag set to true, and a value set on the object with
* a+1 reference count containing the result value to pass back to the
* interpreter. If the plugin did not handle the call, it must
* return a pair with the processed flag set to 'false'.
*/
virtual Val* HookCallFunction(const Func* func, val_list* args);
virtual std::pair<bool, Val*> HookCallFunction(const Func* func, Frame *parent, val_list* args);
/**
* Hook into raising events. Whenever the script interpreter is about
@ -606,7 +632,7 @@ protected:
* Hook for updates to network time. This method will be called
* whenever network time is advanced.
*
* @param networkt_time The new network time.
* @param network_time The new network time.
*/
virtual void HookUpdateNetworkTime(double network_time);

1
src/scan.l
View file

@ -221,6 +221,7 @@ export return TOK_EXPORT;
fallthrough return TOK_FALLTHROUGH;
file return TOK_FILE;
for return TOK_FOR;
while return TOK_WHILE;
function return TOK_FUNCTION;
global return TOK_GLOBAL;
"?$" return TOK_HAS_FIELD;

13
src/threading/formatters/JSON.cc
View file

@ -15,7 +15,7 @@
using namespace threading::formatter;
JSON::JSON(MsgThread* t, TimeFormat tf) : Formatter(t)
JSON::JSON(MsgThread* t, TimeFormat tf) : Formatter(t), surrounding_braces(true)
{
timestamps = tf;
}
@ -27,7 +27,8 @@ JSON::~JSON()
bool JSON::Describe(ODesc* desc, int num_fields, const Field* const * fields,
Value** vals) const
{
desc->AddRaw("{");
if ( surrounding_braces )
desc->AddRaw("{");
for ( int i = 0; i < num_fields; i++ )
{
@ -41,7 +42,8 @@ bool JSON::Describe(ODesc* desc, int num_fields, const Field* const * fields,
return false;
}
desc->AddRaw("}");
if ( surrounding_braces )
desc->AddRaw("}");
return true;
}
@ -217,3 +219,8 @@ threading::Value* JSON::ParseValue(const string& s, const string& name, TypeTag
GetThread()->Error("JSON formatter does not support parsing yet.");
return NULL;
}
void JSON::SurroundingBraces(bool use_braces)
{
surrounding_braces = use_braces;
}

3
src/threading/formatters/JSON.h
View file

@ -27,8 +27,11 @@ public:
threading::Value** vals) const;
virtual threading::Value* ParseValue(const string& s, const string& name, TypeTag type, TypeTag subtype = TYPE_ERROR) const;
void SurroundingBraces(bool use_braces);
private:
TimeFormat timestamps;
bool surrounding_braces;
};
}}

1
src/types.bif
View file

@ -172,6 +172,7 @@ enum Type %{
SOCKS,
GTPv1,
HTTP,
GRE,
%}
type EncapsulatingConn: record;

7
src/util.cc
View file

@ -541,6 +541,13 @@ bool is_printable(const char* s, int len)
return true;
}
std::string strtolower(const std::string& s)
{
std::string t = s;
std::transform(t.begin(), t.end(), t.begin(), ::tolower);
return t;
}
const char* fmt_bytes(const char* data, int len)
{
static char buf[1024];

7
src/util.h
View file

@ -48,8 +48,8 @@
#endif
#ifdef USE_PERFTOOLS_DEBUG
#include <google/heap-checker.h>
#include <google/heap-profiler.h>
#include <gperftools/heap-checker.h>
#include <gperftools/heap-profiler.h>
extern HeapLeakChecker* heap_checker;
#endif
@ -159,6 +159,9 @@ int strstr_n(const int big_len, const unsigned char* big,
extern int fputs(int len, const char* s, FILE* fp);
extern bool is_printable(const char* s, int len);
// Return a lower-cased version of the string.
extern std::string strtolower(const std::string& s);
extern const char* fmt_bytes(const char* data, int len);
// Note: returns a pointer into a shared buffer.