Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 14:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge remote-tracking branch 'origin/topic/timw/150-to-json'

Browse source 
* origin/topic/timw/150-to-json:
  Update submodules for JSON work
  Update unit tests for JSON logger to match new output
  Modify JSON log writer to use the external JSON library
  Update unit test output to match json.zeek being deprecated and slight format changes to JSON output
  Add proper JSON serialization via C++, deprecate json.zeek
  Add new method for escaping UTF8 strings for JSON output
  Move do_sub method from zeek.bif to StringVal class method
  Move record_fields method from zeek.bif to Val class method
  Add ToStdString method for StringVal
This commit is contained in:
Johanna Amann 2019-07-11 11:06:42 -07:00
commit 1f329ad541
29 changed files with 740 additions and 413 deletions
Download patch file Download diff file Expand all files Collapse all files

150
src/threading/formatters/JSON.cc
View file

@ -11,7 +11,7 @@
#include <math.h>
#include <stdint.h>
#include "./JSON.h"
#include "JSON.h"
using namespace threading::formatter;
@ -27,78 +27,83 @@ JSON::~JSON()
bool JSON::Describe(ODesc* desc, int num_fields, const Field* const * fields,
Value** vals) const
{
if ( surrounding_braces )
desc->AddRaw("{");
ZeekJson j = ZeekJson::object();
for ( int i = 0; i < num_fields; i++ )
{
const u_char* bytes = desc->Bytes();
int len = desc->Len();
if ( vals[i]->present )
{
ZeekJson new_entry = BuildJSON(vals[i]);
if ( new_entry.is_null() )
return false;
if ( i > 0 &&
len > 0 &&
bytes[len-1] != ',' &&
bytes[len-1] != '{' &&
bytes[len-1] != '[' &&
vals[i]->present )
desc->AddRaw(",");
if ( ! Describe(desc, vals[i], fields[i]->name) )
return false;
j[fields[i]->name] = new_entry;
}
}
if ( surrounding_braces )
desc->AddRaw("}");
desc->Add(j.dump());
return true;
}
bool JSON::Describe(ODesc* desc, Value* val, const string& name) const
{
if ( desc->IsBinary() )
{
GetThread()->Error("json formatter: binary format not supported");
return false;
}
if ( ! val->present )
return true;
if ( name.size() )
{
desc->AddRaw("\"", 1);
desc->Add(name);
desc->AddRaw("\":", 2);
}
ZeekJson j = BuildJSON(val, name);
if ( j.is_null() )
return false;
desc->Add(j.dump());
return true;
}
threading::Value* JSON::ParseValue(const string& s, const string& name, TypeTag type, TypeTag subtype) const
{
GetThread()->Error("JSON formatter does not support parsing yet.");
return nullptr;
}
ZeekJson JSON::BuildJSON(Value* val, const string& name) const
{
ZeekJson j;
switch ( val->type )
{
case TYPE_BOOL:
desc->AddRaw(val->val.int_val == 0 ? "false" : "true");
j = val->val.int_val != 0;
break;
case TYPE_INT:
desc->Add(val->val.int_val);
j = val->val.int_val;
break;
case TYPE_COUNT:
case TYPE_COUNTER:
desc->Add(val->val.uint_val);
j = val->val.uint_val;
break;
case TYPE_PORT:
desc->Add(val->val.port_val.port);
j = val->val.port_val.port;
break;
case TYPE_SUBNET:
desc->AddRaw("\"", 1);
desc->Add(Render(val->val.subnet_val));
desc->AddRaw("\"", 1);
j = Formatter::Render(val->val.subnet_val);
break;
case TYPE_ADDR:
desc->AddRaw("\"", 1);
desc->Add(Render(val->val.addr_val));
desc->AddRaw("\"", 1);
j = Formatter::Render(val->val.addr_val);
break;
case TYPE_DOUBLE:
case TYPE_INTERVAL:
desc->Add(val->val.double_val);
j = val->val.double_val;
break;
case TYPE_TIME:
@ -110,15 +115,13 @@ bool JSON::Describe(ODesc* desc, Value* val, const string& name) const
time_t the_time = time_t(floor(val->val.double_val));
struct tm t;
desc->AddRaw("\"", 1);
if ( ! gmtime_r(&the_time, &t) ||
! strftime(buffer, sizeof(buffer), "%Y-%m-%dT%H:%M:%S", &t) )
{
GetThread()->Error(GetThread()->Fmt("json formatter: failure getting time: (%lf)", val->val.double_val));
// This was a failure, doesn't really matter what gets put here
// but it should probably stand out...
desc->Add("2000-01-01T00:00:00.000000");
j = "2000-01-01T00:00:00.000000";
}
else
{
@ -129,20 +132,17 @@ bool JSON::Describe(ODesc* desc, Value* val, const string& name) const
frac += 1;
snprintf(buffer2, sizeof(buffer2), "%s.%06.0fZ", buffer, fabs(frac) * 1000000);
desc->Add(buffer2);
j = buffer2;
}
desc->AddRaw("\"", 1);
}
else if ( timestamps == TS_EPOCH )
desc->Add(val->val.double_val);
j = val->val.double_val;
else if ( timestamps == TS_MILLIS )
{
// ElasticSearch uses milliseconds for timestamps
uint64_t ts = (uint64_t) (val->val.double_val * 1000);
desc->Add(ts);
j = (uint64_t) (val->val.double_val * 1000);
}
break;
@ -153,74 +153,40 @@ bool JSON::Describe(ODesc* desc, Value* val, const string& name) const
case TYPE_FILE:
case TYPE_FUNC:
{
desc->AddRaw("\"", 1);
for ( int i = 0; i < val->val.string_val.length; ++i )
{
char c = val->val.string_val.data[i];
// 2byte Unicode escape special characters.
if ( c < 32 || c > 126 || c == '\n' || c == '"' || c == '\'' || c == '\\' || c == '&' )
{
desc->AddRaw("\\u00", 4);
char hex[2] = {'0', '0'};
bytetohex(c, hex);
desc->AddRaw(hex, 1);
desc->AddRaw(hex + 1, 1);
}
else
desc->AddRaw(&c, 1);
}
desc->AddRaw("\"", 1);
j = json_escape_utf8(string(val->val.string_val.data, val->val.string_val.length));
break;
}
case TYPE_TABLE:
{
desc->AddRaw("[", 1);
j = ZeekJson::array();
for ( int j = 0; j < val->val.set_val.size; j++ )
{
if ( j > 0 )
desc->AddRaw(",", 1);
for ( int idx = 0; idx < val->val.set_val.size; idx++ )
j.push_back(BuildJSON(val->val.set_val.vals[idx]));
Describe(desc, val->val.set_val.vals[j]);
}
desc->AddRaw("]", 1);
break;
}
case TYPE_VECTOR:
{
desc->AddRaw("[", 1);
j = ZeekJson::array();
for ( int j = 0; j < val->val.vector_val.size; j++ )
{
if ( j > 0 )
desc->AddRaw(",", 1);
Describe(desc, val->val.vector_val.vals[j]);
}
for ( int idx = 0; idx < val->val.vector_val.size; idx++ )
j.push_back(BuildJSON(val->val.vector_val.vals[idx]));
desc->AddRaw("]", 1);
break;
}
default:
return false;
break;
}
return true;
}
if ( ! name.empty() && ! j.is_null() )
{
ZeekJson j2 = ZeekJson::object();
j2[name] = j;
return j2;
}
threading::Value* JSON::ParseValue(const string& s, const string& name, TypeTag type, TypeTag subtype) const
{
GetThread()->Error("JSON formatter does not support parsing yet.");
return NULL;
}
void JSON::SurroundingBraces(bool use_braces)
{
surrounding_braces = use_braces;
return j;
}