diff --git a/CHANGES b/CHANGES index 341b649db2..2ffd5506ba 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,46 @@ +2.6-476 | 2019-06-20 13:23:22 -0700 + + * Remove opaque of ocsp_resp. (Johanna Amann, Corelight) + + Only used in one event, without any way to use the opaque for anything + else. At this point this just seems like a complication that has no + reason to be there. + + * Remove remnants of event serializer. (Johanna Amann, Corelight) + + * Reimplement serialization infrastructure for OpaqueVals. + (Robin Sommer, Corelight & Johanna Amann, Corelight) + + We need this to sender through Broker, and we also leverage it for + cloning opaques. The serialization methods now produce Broker data + instances directly, and no longer go through the binary formatter. + + Summary of the new API for types derived from OpaqueVal: + + - Add DECLARE_OPAQUE_VALUE() to the class declaration + - Add IMPLEMENT_OPAQUE_VALUE() to the class' implementation file + - Implement these two methods (which are declated by the 1st macro): + - broker::data DoSerialize() const + - bool DoUnserialize(const broker::data& data) + + This machinery should work correctly from dynamic plugins as well. + + OpaqueVal provides a default implementation of DoClone() as well that + goes through serialization. Derived classes can provide a more + efficient version if they want. + + The declaration of the "OpaqueVal" class has moved into the header + file "OpaqueVal.h", along with the new serialization infrastructure. + This is breaking existing code that relies on the location, but + because the API is changing anyways that seems fine. + + * Implement a Shallow Clone operation for types. (Johanna Amann, Corelight) + + This is needed to track name changes for the documentation. + + * Remove old serialization infrastrucutre. (Johanna Amann, Corelight) + 2.6-454 | 2019-06-19 09:39:06 -0700 * GH-393: Add slice notation for vectors (Tim Wojtulewicz, Corelight & Jon Siwek, Corelight) diff --git a/NEWS b/NEWS index 44c3734e40..197419c97d 100644 --- a/NEWS +++ b/NEWS @@ -325,6 +325,11 @@ Changed Functionality - logging - bro/logs/ +- The ``resp_ref`` argument was removed from the ``ocsp_response_bytes`` + event. ``resp_ref`` was not used by anything in the codebase and could not be + passed to any other functions for further processing. The remainder of the + ``ocsp_response_bytes`` is unchanged. + Removed Functionality --------------------- @@ -433,8 +438,18 @@ Removed Functionality and &rotate_size attributes, which were deprecated in Bro 2.6, was removed. The ``-g`` command-line option (dump-config) which relied on this functionality was also removed. +- Functionality for writing state updates for variables with the + &synchronized attribute was removed. This entails the + ``-x`` command-line option (print-state) as well as the + ``capture_state_updates`` function. + - Removed the BroControl ``update`` command, which was deprecated in Bro 2.6. +- Functionality for writing/reading binary event streams was + removed. This functionality relied on the old communication code + anc was basically untested. The ``-R`` command-line option (replay) + as well as the ``capture_events`` function were removed. + Deprecated Functionality ------------------------ diff --git a/VERSION b/VERSION index 94ee367fcd..2c31edab5c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.6-454 +2.6-476 diff --git a/doc b/doc index 9926886afc..e5b95022ff 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 9926886afcc23f61b06d1361e35cc99a8f993911 +Subproject commit e5b95022ffa68ddf4645228d123cf1ea73a55186 diff --git a/src/Attr.cc b/src/Attr.cc index e7801c12c6..079c125f9e 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -4,7 +4,6 @@ #include "Attr.h" #include "Expr.h" -#include "Serializer.h" #include "threading/SerialTypes.h" const char* attr_name(attr_tag t) @@ -509,70 +508,3 @@ bool Attributes::operator==(const Attributes& other) const return true; } -bool Attributes::Serialize(SerialInfo* info) const - { - return SerialObj::Serialize(info); - } - -Attributes* Attributes::Unserialize(UnserialInfo* info) - { - return (Attributes*) SerialObj::Unserialize(info, SER_ATTRIBUTES); - } - -IMPLEMENT_SERIAL(Attributes, SER_ATTRIBUTES); - -bool Attributes::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_ATTRIBUTES, BroObj); - - info->s->WriteOpenTag("Attributes"); - assert(type); - if ( ! (type->Serialize(info) && SERIALIZE(attrs->length())) ) - return false; - - loop_over_list((*attrs), i) - { - Attr* a = (*attrs)[i]; - - Expr* e = a->AttrExpr(); - SERIALIZE_OPTIONAL(e); - - if ( ! SERIALIZE(char(a->Tag())) ) - return false; - } - - info->s->WriteCloseTag("Attributes"); - return true; - } - -bool Attributes::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroObj); - - type = BroType::Unserialize(info); - if ( ! type ) - return false; - - int len; - if ( ! UNSERIALIZE(&len) ) - return false; - - attrs = new attr_list(len); - while ( len-- ) - { - Expr* e; - UNSERIALIZE_OPTIONAL(e, Expr::Unserialize(info)) - - char tag; - if ( ! UNSERIALIZE(&tag) ) - { - delete e; - return false; - } - - attrs->append(new Attr((attr_tag)tag, e)); - } - - return true; - } - diff --git a/src/Attr.h b/src/Attr.h index de6b07abbe..f32c11e50f 100644 --- a/src/Attr.h +++ b/src/Attr.h @@ -92,17 +92,12 @@ public: attr_list* Attrs() { return attrs; } - bool Serialize(SerialInfo* info) const; - static Attributes* Unserialize(UnserialInfo* info); - bool operator==(const Attributes& other) const; protected: Attributes() : type(), attrs(), in_record() { } void CheckAttr(Attr* attr); - DECLARE_SERIAL(Attributes); - BroType* type; attr_list* attrs; bool in_record; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f014d0a149..6c605e6ba5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -247,7 +247,6 @@ set(bro_SRCS Brofiler.cc BroString.cc CCL.cc - ChunkedIO.cc CompHash.cc Conn.cc ConvertUTF.c @@ -302,8 +301,6 @@ set(bro_SRCS SmithWaterman.cc Scope.cc SerializationFormat.cc - SerialObj.cc - Serializer.cc Sessions.cc StateAccess.cc Stats.cc diff --git a/src/ChunkedIO.cc b/src/ChunkedIO.cc deleted file mode 100644 index a3cefe3a05..0000000000 --- a/src/ChunkedIO.cc +++ /dev/null @@ -1,1357 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include "zeek-config.h" -#include "ChunkedIO.h" -#include "NetVar.h" - -ChunkedIO::ChunkedIO() : stats(), tag(), pure() - { - } - -void ChunkedIO::Stats(char* buffer, int length) - { - safe_snprintf(buffer, length, - "bytes=%luK/%luK chunks=%lu/%lu io=%lu/%lu bytes/io=%.2fK/%.2fK", - stats.bytes_read / 1024, stats.bytes_written / 1024, - stats.chunks_read, stats.chunks_written, - stats.reads, stats.writes, - stats.bytes_read / (1024.0 * stats.reads), - stats.bytes_written / (1024.0 * stats.writes)); - } - -#ifdef DEBUG_COMMUNICATION - -void ChunkedIO::AddToBuffer(uint32 len, char* data, bool is_read) - { - Chunk* copy = new Chunk; - copy->len = len; - copy->data = new char[len]; - memcpy(copy->data, data, len); - - std::list* l = is_read ? &data_read : &data_written; - l->push_back(copy); - - if ( l->size() > DEBUG_COMMUNICATION ) - { - Chunk* old = l->front(); - l->pop_front(); - delete [] old->data; - delete old; - } - } - -void ChunkedIO::AddToBuffer(Chunk* chunk, bool is_read) - { - AddToBuffer(chunk->len, chunk->data, is_read); - } - -void ChunkedIO::DumpDebugData(const char* basefnname, bool want_reads) - { - std::list* l = want_reads ? &data_read : &data_written; - - int count = 0; - - for ( std::list::iterator i = l->begin(); i != l->end(); ++i ) - { - static char buffer[128]; - snprintf(buffer, sizeof(buffer), "%s.%s.%d", basefnname, - want_reads ? "read" : "write", ++count); - buffer[sizeof(buffer) - 1] = '\0'; - - int fd = open(buffer, O_WRONLY | O_CREAT | O_TRUNC, 0600); - if ( fd < 0 ) - continue; - - ChunkedIOFd io(fd, "dump-file"); - io.Write(*i); - io.Flush(); - safe_close(fd); - } - - l->clear(); - } - -#endif - -ChunkedIOFd::ChunkedIOFd(int arg_fd, const char* arg_tag, pid_t arg_pid) - { - int flags; - - tag = arg_tag; - fd = arg_fd; - eof = 0; - last_flush = current_time(); - failed_reads = 0; - - if ( (flags = fcntl(fd, F_GETFL, 0)) < 0) - { - Log(fmt("can't obtain socket flags: %s", strerror(errno))); - exit(1); - } - - if ( fcntl(fd, F_SETFL, flags|O_NONBLOCK) < 0 ) - { - Log(fmt("can't set fd to non-blocking: %s (%d)", - strerror(errno), getpid())); - exit(1); - } - - read_buffer = new char[BUFFER_SIZE]; - read_len = 0; - read_pos = 0; - partial = 0; - write_buffer = new char[BUFFER_SIZE]; - write_len = 0; - write_pos = 0; - - pending_head = 0; - pending_tail = 0; - - pid = arg_pid; - } - -ChunkedIOFd::~ChunkedIOFd() - { - Clear(); - - delete [] read_buffer; - delete [] write_buffer; - safe_close(fd); - delete partial; - } - -bool ChunkedIOFd::Write(Chunk* chunk) - { -#ifdef DEBUG - DBG_LOG(DBG_CHUNKEDIO, "write of size %d [%s]", - chunk->len, fmt_bytes(chunk->data, min((uint32)20, chunk->len))); -#endif - -#ifdef DEBUG_COMMUNICATION - AddToBuffer(chunk, false); -#endif - - if ( chunk->len <= BUFFER_SIZE - sizeof(uint32) ) - return WriteChunk(chunk, false); - - // We have to split it up. - char* p = chunk->data; - uint32 left = chunk->len; - - while ( left ) - { - uint32 sz = min(BUFFER_SIZE - sizeof(uint32), left); - Chunk* part = new Chunk(new char[sz], sz); - - memcpy(part->data, p, part->len); - left -= part->len; - p += part->len; - - if ( ! WriteChunk(part, left != 0) ) - return false; - } - - delete chunk; - return true; - } - -bool ChunkedIOFd::WriteChunk(Chunk* chunk, bool partial) - { - assert(chunk->len <= BUFFER_SIZE - sizeof(uint32) ); - - if ( chunk->len == 0 ) - InternalError("attempt to write 0 bytes chunk"); - - if ( partial ) - chunk->len |= FLAG_PARTIAL; - - ++stats.chunks_written; - - // If it fits into the buffer, we're done (but keep care not - // to reorder chunks). - if ( ! pending_head && PutIntoWriteBuffer(chunk) ) - return true; - - // Otherwise queue it. - ++stats.pending; - ChunkQueue* q = new ChunkQueue; - q->chunk = chunk; - q->next = 0; - - if ( pending_tail ) - { - pending_tail->next = q; - pending_tail = q; - } - else - pending_head = pending_tail = q; - - write_flare.Fire(); - return Flush(); - } - - -bool ChunkedIOFd::PutIntoWriteBuffer(Chunk* chunk) - { - uint32 len = chunk->len & ~FLAG_PARTIAL; - - if ( write_len + len + (IsPure() ? 0 : sizeof(len)) > BUFFER_SIZE ) - return false; - - if ( ! IsPure() ) - { - uint32 nlen = htonl(chunk->len); - memcpy(write_buffer + write_len, &nlen, sizeof(nlen)); - write_len += sizeof(nlen); - } - - memcpy(write_buffer + write_len, chunk->data, len); - write_len += len; - - delete chunk; - write_flare.Fire(); - - if ( network_time - last_flush > 0.005 ) - FlushWriteBuffer(); - - return true; - } - -bool ChunkedIOFd::FlushWriteBuffer() - { - last_flush = network_time; - - while ( write_pos != write_len ) - { - uint32 len = write_len - write_pos; - - int written = write(fd, write_buffer + write_pos, len); - - if ( written < 0 ) - { - if ( errno == EPIPE ) - eof = true; - - if ( errno != EINTR ) - // These errnos are equal on POSIX. - return errno == EWOULDBLOCK || errno == EAGAIN; - - else - written = 0; - } - - stats.bytes_written += written; - if ( written > 0 ) - ++stats.writes; - - if ( unsigned(written) == len ) - { - write_pos = write_len = 0; - - if ( ! pending_head ) - write_flare.Extinguish(); - - return true; - } - - if ( written == 0 ) - InternalError("written==0"); - - // Short write. - write_pos += written; - } - - return true; - } - -bool ChunkedIOFd::OptionalFlush() - { - // This threshhold is quite arbitrary. -// if ( current_time() - last_flush > 0.01 ) - return Flush(); - } - -bool ChunkedIOFd::Flush() - { - // Try to write data out. - while ( pending_head ) - { - if ( ! FlushWriteBuffer() ) - return false; - - // If we couldn't write the whole buffer, we stop here - // and try again next time. - if ( write_len > 0 ) - return true; - - // Put as many pending chunks into the buffer as possible. - while ( pending_head ) - { - if ( ! PutIntoWriteBuffer(pending_head->chunk) ) - break; - - ChunkQueue* q = pending_head; - pending_head = pending_head->next; - if ( ! pending_head ) - pending_tail = 0; - - --stats.pending; - delete q; - } - } - - bool rval = FlushWriteBuffer(); - - if ( ! pending_head && write_len == 0 ) - write_flare.Extinguish(); - - return rval; - } - -uint32 ChunkedIOFd::ChunkAvailable() - { - int bytes_left = read_len - read_pos; - - if ( bytes_left < int(sizeof(uint32)) ) - return 0; - - bytes_left -= sizeof(uint32); - - // We have to copy the value here as it may not be - // aligned correctly in the data. - uint32 len; - memcpy(&len, read_buffer + read_pos, sizeof(len)); - len = ntohl(len); - - if ( uint32(bytes_left) < (len & ~FLAG_PARTIAL) ) - return 0; - - assert(len & ~FLAG_PARTIAL); - - return len; - } - -ChunkedIO::Chunk* ChunkedIOFd::ExtractChunk() - { - uint32 len = ChunkAvailable(); - uint32 real_len = len & ~FLAG_PARTIAL; - if ( ! real_len ) - return 0; - - read_pos += sizeof(uint32); - - Chunk* chunk = new Chunk(new char[real_len], len); - memcpy(chunk->data, read_buffer + read_pos, real_len); - read_pos += real_len; - - ++stats.chunks_read; - - return chunk; - } - -ChunkedIO::Chunk* ChunkedIOFd::ConcatChunks(Chunk* c1, Chunk* c2) - { - uint32 sz = c1->len + c2->len; - Chunk* c = new Chunk(new char[sz], sz); - - memcpy(c->data, c1->data, c1->len); - memcpy(c->data + c1->len, c2->data, c2->len); - - delete c1; - delete c2; - - return c; - } - -void ChunkedIO::Log(const char* str) - { - //RemoteSerializer::Log(RemoteSerializer::LogError, str); - } - -bool ChunkedIOFd::Read(Chunk** chunk, bool may_block) - { - *chunk = 0; - - // We will be called regularly. So take the opportunity - // to flush the write buffer once in a while. - OptionalFlush(); - - if ( ! ReadChunk(chunk, may_block) ) - { -#ifdef DEBUG_COMMUNICATION - AddToBuffer("", true); -#endif - if ( ! ChunkAvailable() ) - read_flare.Extinguish(); - - return false; - } - - if ( ! *chunk ) - { -#ifdef DEBUG_COMMUNICATION - AddToBuffer("", true); -#endif - read_flare.Extinguish(); - return true; - } - - if ( ChunkAvailable() ) - read_flare.Fire(); - else - read_flare.Extinguish(); - -#ifdef DEBUG - if ( *chunk ) - DBG_LOG(DBG_CHUNKEDIO, "read of size %d %s[%s]", - (*chunk)->len & ~FLAG_PARTIAL, - (*chunk)->len & FLAG_PARTIAL ? "(P) " : "", - fmt_bytes((*chunk)->data, - min((uint32)20, (*chunk)->len))); -#endif - - if ( ! ((*chunk)->len & FLAG_PARTIAL) ) - { - if ( ! partial ) - { -#ifdef DEBUG_COMMUNICATION - AddToBuffer(*chunk, true); -#endif - return true; - } - else - { - // This is the last chunk of an oversized one. - *chunk = ConcatChunks(partial, *chunk); - partial = 0; - -#ifdef DEBUG - if ( *chunk ) - DBG_LOG(DBG_CHUNKEDIO, - "built virtual chunk of size %d [%s]", - (*chunk)->len, - fmt_bytes((*chunk)->data, 20)); -#endif - -#ifdef DEBUG_COMMUNICATION - AddToBuffer(*chunk, true); -#endif - return true; - } - } - - // This chunk is the non-last part of an oversized. - (*chunk)->len &= ~FLAG_PARTIAL; - - if ( ! partial ) - // First part of oversized chunk. - partial = *chunk; - else - partial = ConcatChunks(partial, *chunk); - -#ifdef DEBUG_COMMUNICATION - AddToBuffer("", true); -#endif - - *chunk = 0; - return true; // Read following part next time. - } - -bool ChunkedIOFd::ReadChunk(Chunk** chunk, bool may_block) - { - // We will be called regularly. So take the opportunity - // to flush the write buffer once in a while. - OptionalFlush(); - - *chunk = ExtractChunk(); - if ( *chunk ) - return true; - - int bytes_left = read_len - read_pos; - - // If we have a partial chunk left, move this to the head of - // the buffer. - if ( bytes_left ) - memmove(read_buffer, read_buffer + read_pos, bytes_left); - - read_pos = 0; - read_len = bytes_left; - - if ( ! ChunkAvailable() ) - read_flare.Extinguish(); - - // If allowed, wait a bit for something to read. - if ( may_block ) - { - fd_set fd_read, fd_write, fd_except; - - FD_ZERO(&fd_read); - FD_ZERO(&fd_write); - FD_ZERO(&fd_except); - FD_SET(fd, &fd_read); - - struct timeval small_timeout; - small_timeout.tv_sec = 0; - small_timeout.tv_usec = 50; - - select(fd + 1, &fd_read, &fd_write, &fd_except, &small_timeout); - } - - // Make sure the process is still runnning - // (only checking for EPIPE after a read doesn't - // seem to be sufficient). - if ( pid && kill(pid, 0) < 0 && errno != EPERM ) - { - eof = true; - errno = EPIPE; - return false; - } - - // Try to fill the buffer. - while ( true ) - { - int len = BUFFER_SIZE - read_len; - int read = ::read(fd, read_buffer + read_len, len); - - if ( read < 0 ) - { - if ( errno != EINTR ) - { - // These errnos are equal on POSIX. - if ( errno == EWOULDBLOCK || errno == EAGAIN ) - { - // Let's see if we have a chunk now -- - // even if we time out, we may have read - // just enough in previous iterations! - *chunk = ExtractChunk(); - ++failed_reads; - return true; - } - - if ( errno == EPIPE ) - eof = true; - - return false; - } - - else - read = 0; - } - - failed_reads = 0; - - if ( read == 0 && len != 0 ) - { - *chunk = ExtractChunk(); - if ( *chunk ) - return true; - - eof = true; - return false; - } - - read_len += read; - - ++stats.reads; - stats.bytes_read += read; - - if ( read == len ) - break; - } - - // Let's see if we have a chunk now. - *chunk = ExtractChunk(); - - return true; - } - -bool ChunkedIOFd::CanRead() - { - // We will be called regularly. So take the opportunity - // to flush the write buffer once in a while. - OptionalFlush(); - - if ( ChunkAvailable() ) - return true; - - fd_set fd_read; - FD_ZERO(&fd_read); - FD_SET(fd, &fd_read); - - struct timeval no_timeout; - no_timeout.tv_sec = 0; - no_timeout.tv_usec = 0; - - return select(fd + 1, &fd_read, 0, 0, &no_timeout) > 0; - } - -bool ChunkedIOFd::CanWrite() - { - return pending_head != 0; - } - -bool ChunkedIOFd::IsIdle() - { - if ( pending_head || ChunkAvailable() ) - return false; - - if ( failed_reads > 0 ) - return true; - - return false; - } - -bool ChunkedIOFd::IsFillingUp() - { - return stats.pending > chunked_io_buffer_soft_cap; - } - -iosource::FD_Set ChunkedIOFd::ExtraReadFDs() const - { - iosource::FD_Set rval; - rval.Insert(write_flare.FD()); - rval.Insert(read_flare.FD()); - return rval; - } - -void ChunkedIOFd::Clear() - { - while ( pending_head ) - { - ChunkQueue* next = pending_head->next; - delete pending_head->chunk; - delete pending_head; - pending_head = next; - } - - pending_head = pending_tail = 0; - - if ( write_len == 0 ) - write_flare.Extinguish(); - } - -const char* ChunkedIOFd::Error() - { - static char buffer[1024]; - safe_snprintf(buffer, sizeof(buffer), "%s [%d]", strerror(errno), errno); - - return buffer; - } - -void ChunkedIOFd::Stats(char* buffer, int length) - { - int i = safe_snprintf(buffer, length, "pending=%d ", stats.pending); - ChunkedIO::Stats(buffer + i, length - i); - } - -SSL_CTX* ChunkedIOSSL::ctx; - -ChunkedIOSSL::ChunkedIOSSL(int arg_socket, bool arg_server) - { - socket = arg_socket; - last_ret = 0; - eof = false; - setup = false; - server = arg_server; - ssl = 0; - - write_state = LEN; - write_head = 0; - write_tail = 0; - - read_state = LEN; - read_chunk = 0; - read_ptr = 0; - } - -ChunkedIOSSL::~ChunkedIOSSL() - { - if ( setup ) - { - SSL_shutdown(ssl); - - // We don't care if the other side closes properly. - setup = false; - } - - if ( ssl ) - { - SSL_free(ssl); - ssl = 0; - } - - safe_close(socket); - } - - -static int pem_passwd_cb(char* buf, int size, int rwflag, void* passphrase) - { - safe_strncpy(buf, (char*) passphrase, size); - buf[size - 1] = '\0'; - return strlen(buf); - } - -bool ChunkedIOSSL::Init() - { - // If the handshake doesn't succeed immediately we will - // be called multiple times. - if ( ! ctx ) - { - SSL_load_error_strings(); - - ctx = SSL_CTX_new(SSLv23_method()); - if ( ! ctx ) - { - Log("can't create SSL context"); - return false; - } - - // We access global variables here. But as they are - // declared const and we don't modify them this should - // be fine. - const char* key = ssl_private_key->AsString()->CheckString(); - - if ( ! (key && *key && - SSL_CTX_use_certificate_chain_file(ctx, key)) ) - { - Log(fmt("can't read certificate from file %s", key)); - return false; - } - - const char* passphrase = - ssl_passphrase->AsString()->CheckString(); - - if ( passphrase && ! streq(passphrase, "") ) - { - SSL_CTX_set_default_passwd_cb(ctx, pem_passwd_cb); - SSL_CTX_set_default_passwd_cb_userdata(ctx, - (void*) passphrase); - } - - if ( ! (key && *key && - SSL_CTX_use_PrivateKey_file(ctx, key, SSL_FILETYPE_PEM)) ) - { - Log(fmt("can't read private key from file %s", key)); - return false; - } - - const char* ca = ssl_ca_certificate->AsString()->CheckString(); - if ( ! (ca && *ca && SSL_CTX_load_verify_locations(ctx, ca, 0)) ) - { - Log(fmt("can't read CA certificate from file %s", ca)); - return false; - } - - // Only use real ciphers. - if ( ! SSL_CTX_set_cipher_list(ctx, "HIGH") ) - { - Log("can't set cipher list"); - return false; - } - - // Require client certificate. - SSL_CTX_set_verify(ctx, - SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, 0); - } - - int flags; - - if ( (flags = fcntl(socket, F_GETFL, 0)) < 0) - { - Log(fmt("can't obtain socket flags: %s", strerror(errno))); - return false; - } - - if ( fcntl(socket, F_SETFL, flags|O_NONBLOCK) < 0 ) - { - Log(fmt("can't set socket to non-blocking: %s", - strerror(errno))); - return false; - } - - if ( ! ssl ) - { - ssl = SSL_new(ctx); - if ( ! ssl ) - { - Log("can't create SSL object"); - return false; - } - - BIO* bio = BIO_new_socket(socket, BIO_NOCLOSE); - BIO_set_nbio(bio, 1); - SSL_set_bio(ssl, bio, bio); - } - - int success; - if ( server ) - success = last_ret = SSL_accept(ssl); - else - success = last_ret = SSL_connect(ssl); - - if ( success > 0 ) - { // handshake done - setup = true; - return true; - } - - int error = SSL_get_error(ssl, success); - - if ( success <= 0 && - (error == SSL_ERROR_WANT_WRITE || error == SSL_ERROR_WANT_READ) ) - // Handshake not finished yet, but that's ok for now. - return true; - - // Some error. - eof = true; - return false; - } - -bool ChunkedIOSSL::Write(Chunk* chunk) - { -#ifdef DEBUG - DBG_LOG(DBG_CHUNKEDIO, "ssl write of size %d [%s]", - chunk->len, fmt_bytes(chunk->data, 20)); -#endif - - // Queue it. - ++stats.pending; - Queue* q = new Queue; - q->chunk = chunk; - q->next = 0; - - // Temporarily convert len into network byte order. - chunk->len = htonl(chunk->len); - - if ( write_tail ) - { - write_tail->next = q; - write_tail = q; - } - else - write_head = write_tail = q; - - write_flare.Fire(); - Flush(); - return true; - } - -bool ChunkedIOSSL::WriteData(char* p, uint32 len, bool* error) - { - *error = false; - - double t = current_time(); - - int written = last_ret = SSL_write(ssl, p, len); - - switch ( SSL_get_error(ssl, written) ) { - case SSL_ERROR_NONE: - // SSL guarantees us that all bytes have been written. - // That's nice. :-) - return true; - - case SSL_ERROR_WANT_READ: - case SSL_ERROR_WANT_WRITE: - // Would block. - DBG_LOG(DBG_CHUNKEDIO, - "SSL_write: SSL_ERROR_WANT_READ [%d,%d]", - written, SSL_get_error(ssl, written)); - *error = false; - return false; - - case SSL_ERROR_ZERO_RETURN: - // Regular remote connection shutdown. - DBG_LOG(DBG_CHUNKEDIO, - "SSL_write: SSL_ZERO_RETURN [%d,%d]", - written, SSL_get_error(ssl, written)); - *error = eof = true; - return false; - - case SSL_ERROR_SYSCALL: - DBG_LOG(DBG_CHUNKEDIO, - "SSL_write: SSL_SYS_CALL [%d,%d]", - written, SSL_get_error(ssl, written)); - - if ( written == 0 ) - { - // Socket connection closed. - *error = eof = true; - return false; - } - - // Fall through. - - default: - DBG_LOG(DBG_CHUNKEDIO, - "SSL_write: fatal error [%d,%d]", - written, SSL_get_error(ssl, written)); - // Fatal SSL error. - *error = true; - return false; - } - - InternalError("can't be reached"); - return false; - } - -bool ChunkedIOSSL::Flush() - { - if ( ! setup ) - { - // We may need to finish the handshake. - if ( ! Init() ) - return false; - if ( ! setup ) - return true; - } - - while ( write_head ) - { - bool error; - - Chunk* c = write_head->chunk; - - if ( write_state == LEN ) - { - if ( ! WriteData((char*)&c->len, sizeof(c->len), &error) ) - return ! error; - write_state = DATA; - - // Convert back from network byte order. - c->len = ntohl(c->len); - } - - if ( ! WriteData(c->data, c->len, &error) ) - return ! error; - - // Chunk written, throw away. - Queue* q = write_head; - write_head = write_head->next; - if ( ! write_head ) - write_tail = 0; - --stats.pending; - delete q; - - delete c; - - write_state = LEN; - } - - write_flare.Extinguish(); - return true; - } - -bool ChunkedIOSSL::ReadData(char* p, uint32 len, bool* error) - { - if ( ! read_ptr ) - read_ptr = p; - - while ( true ) - { - double t = current_time(); - - int read = last_ret = - SSL_read(ssl, read_ptr, len - (read_ptr - p)); - - switch ( SSL_get_error(ssl, read) ) { - case SSL_ERROR_NONE: - // We're fine. - read_ptr += read; - - if ( unsigned(read_ptr - p) == len ) - { - // We have read as much as requested.. - read_ptr = 0; - *error = false; - return true; - } - - break; - - case SSL_ERROR_WANT_READ: - case SSL_ERROR_WANT_WRITE: - // Would block. - DBG_LOG(DBG_CHUNKEDIO, - "SSL_read: SSL_ERROR_WANT_READ [%d,%d]", - read, SSL_get_error(ssl, read)); - *error = false; - return false; - - case SSL_ERROR_ZERO_RETURN: - // Regular remote connection shutdown. - DBG_LOG(DBG_CHUNKEDIO, - "SSL_read: SSL_ZERO_RETURN [%d,%d]", - read, SSL_get_error(ssl, read)); - *error = eof = true; - return false; - - case SSL_ERROR_SYSCALL: - DBG_LOG(DBG_CHUNKEDIO, "SSL_read: SSL_SYS_CALL [%d,%d]", - read, SSL_get_error(ssl, read)); - - if ( read == 0 ) - { - // Socket connection closed. - *error = eof = true; - return false; - } - - // Fall through. - - default: - DBG_LOG(DBG_CHUNKEDIO, - "SSL_read: fatal error [%d,%d]", - read, SSL_get_error(ssl, read)); - - // Fatal SSL error. - *error = true; - return false; - } - } - - // Can't be reached. - InternalError("can't be reached"); - return false; - } - -bool ChunkedIOSSL::Read(Chunk** chunk, bool mayblock) - { - *chunk = 0; - - if ( ! setup ) - { - // We may need to finish the handshake. - if ( ! Init() ) - return false; - if ( ! setup ) - return true; - } - - bool error; - - Flush(); - - if ( read_state == LEN ) - { - if ( ! read_chunk ) - { - read_chunk = new Chunk; - read_chunk->data = 0; - } - - if ( ! ReadData((char*)&read_chunk->len, - sizeof(read_chunk->len), - &error) ) - return ! error; - - read_state = DATA; - read_chunk->len = ntohl(read_chunk->len); - } - - if ( ! read_chunk->data ) - { - read_chunk->data = new char[read_chunk->len]; - read_chunk->free_func = Chunk::free_func_delete; - } - - if ( ! ReadData(read_chunk->data, read_chunk->len, &error) ) - return ! error; - - // Chunk fully read. Pass it on. - *chunk = read_chunk; - read_chunk = 0; - read_state = LEN; - -#ifdef DEBUG - DBG_LOG(DBG_CHUNKEDIO, "ssl read of size %d [%s]", - (*chunk)->len, fmt_bytes((*chunk)->data, 20)); -#endif - - return true; - } - -bool ChunkedIOSSL::CanRead() - { - // We will be called regularly. So take the opportunity - // to flush the write buffer. - Flush(); - - if ( SSL_pending(ssl) ) - return true; - - fd_set fd_read; - FD_ZERO(&fd_read); - FD_SET(socket, &fd_read); - - struct timeval notimeout; - notimeout.tv_sec = 0; - notimeout.tv_usec = 0; - - return select(socket + 1, &fd_read, NULL, NULL, ¬imeout) > 0; - } - -bool ChunkedIOSSL::CanWrite() - { - return write_head != 0; - } - -bool ChunkedIOSSL::IsIdle() - { - return ! (CanRead() || CanWrite()); - } - -bool ChunkedIOSSL::IsFillingUp() - { - // We don't really need this at the moment (since SSL is only used for - // peer-to-peer communication). Thus, we always return false for now. - return false; - } - -iosource::FD_Set ChunkedIOSSL::ExtraReadFDs() const - { - iosource::FD_Set rval; - rval.Insert(write_flare.FD()); - return rval; - } - -void ChunkedIOSSL::Clear() - { - while ( write_head ) - { - Queue* next = write_head->next; - delete write_head->chunk; - delete write_head; - write_head = next; - } - write_head = write_tail = 0; - write_flare.Extinguish(); - } - -const char* ChunkedIOSSL::Error() - { - const int BUFLEN = 512; - static char buffer[BUFLEN]; - - int sslcode = SSL_get_error(ssl, last_ret); - int errcode = ERR_get_error(); - - int count = safe_snprintf(buffer, BUFLEN, "[%d,%d,%d] SSL error: ", - errcode, sslcode, last_ret); - - if ( errcode ) - ERR_error_string_n(errcode, buffer + count, BUFLEN - count); - - else if ( sslcode == SSL_ERROR_SYSCALL ) - { - if ( last_ret ) - // Look at errno. - safe_snprintf(buffer + count, BUFLEN - count, - "syscall: %s", strerror(errno)); - else - // Errno is not valid in this case. - safe_strncpy(buffer + count, - "syscall: unexpected end-of-file", - BUFLEN - count); - } - else - safe_strncpy(buffer + count, "unknown error", BUFLEN - count); - - return buffer; - } - -void ChunkedIOSSL::Stats(char* buffer, int length) - { - int i = safe_snprintf(buffer, length, "pending=%ld ", stats.pending); - ChunkedIO::Stats(buffer + i, length - i); - } - -bool CompressedChunkedIO::Init() - { - zin.zalloc = 0; - zin.zfree = 0; - zin.opaque = 0; - - zout.zalloc = 0; - zout.zfree = 0; - zout.opaque = 0; - - compress = uncompress = false; - error = 0; - uncompressed_bytes_read = 0; - uncompressed_bytes_written = 0; - - return true; - } - -bool CompressedChunkedIO::Read(Chunk** chunk, bool may_block) - { - if ( ! io->Read(chunk, may_block) ) - return false; - - if ( ! uncompress ) - return true; - - if ( ! *chunk ) - return true; - - uint32 uncompressed_len = - *(uint32*)((*chunk)->data + (*chunk)->len - sizeof(uint32)); - - if ( uncompressed_len == 0 ) - { - // Not compressed. - DBG_LOG(DBG_CHUNKEDIO, "zlib read pass-through: size=%d", - (*chunk)->len); - return true; - } - - char* uncompressed = new char[uncompressed_len]; - - DBG_LOG(DBG_CHUNKEDIO, "zlib read: size=%d uncompressed=%d", - (*chunk)->len, uncompressed_len); - - zin.next_in = (Bytef*) (*chunk)->data; - zin.avail_in = (*chunk)->len - sizeof(uint32); - zin.next_out = (Bytef*) uncompressed; - zin.avail_out = uncompressed_len; - - if ( inflate(&zin, Z_SYNC_FLUSH) != Z_OK ) - { - error = zin.msg; - return false; - } - - if ( zin.avail_in > 0 ) - { - error = "compressed data longer than expected"; - return false; - } - - (*chunk)->free_func((*chunk)->data); - - uncompressed_bytes_read += uncompressed_len; - - (*chunk)->len = uncompressed_len; - (*chunk)->data = uncompressed; - (*chunk)->free_func = Chunk::free_func_delete; - - return true; - } - -bool CompressedChunkedIO::Write(Chunk* chunk) - { - if ( (! compress) || IsPure() ) - // No compression. - return io->Write(chunk); - - // We compress block-wise (rather than stream-wise) because: - // - // (1) it's significantly easier to implement due to our block-oriented - // communication model (with a stream compression, we'd need to chop - // the stream into blocks during decompression which would require - // additional buffering and copying). - // - // (2) it ensures that we do not introduce any additional latencies (a - // stream compression may decide to wait for the next chunk of data - // before writing anything out). - // - // The block-wise compression comes at the cost of a smaller compression - // factor. - // - // A compressed chunk's data looks like this: - // char[] compressed data - // uint32 uncompressed_length - // - // By including uncompressed_length, we again trade easier - // decompression for a smaller reduction factor. If uncompressed_length - // is zero, the data is *not* compressed. - - uncompressed_bytes_written += chunk->len; - uint32 original_size = chunk->len; - - char* compressed = new char[chunk->len + sizeof(uint32)]; - - if ( chunk->len < MIN_COMPRESS_SIZE ) - { - // Too small; not worth any compression. - memcpy(compressed, chunk->data, chunk->len); - *(uint32*) (compressed + chunk->len) = 0; // uncompressed_length - - chunk->free_func(chunk->data); - chunk->data = compressed; - chunk->free_func = Chunk::free_func_delete; - chunk->len += 4; - - DBG_LOG(DBG_CHUNKEDIO, "zlib write pass-through: size=%d", chunk->len); - } - else - { - zout.next_in = (Bytef*) chunk->data; - zout.avail_in = chunk->len; - zout.next_out = (Bytef*) compressed; - zout.avail_out = chunk->len; - - if ( deflate(&zout, Z_SYNC_FLUSH) != Z_OK ) - { - error = zout.msg; - return false; - } - - while ( zout.avail_out == 0 ) - { - // D'oh! Not enough space, i.e., it hasn't got smaller. - char* old = compressed; - int old_size = (char*) zout.next_out - compressed; - int new_size = old_size * 2 + sizeof(uint32); - - compressed = new char[new_size]; - memcpy(compressed, old, old_size); - delete [] old; - - zout.next_out = (Bytef*) (compressed + old_size); - zout.avail_out = old_size; // Sic! We doubled. - - if ( deflate(&zout, Z_SYNC_FLUSH) != Z_OK ) - { - error = zout.msg; - return false; - } - } - - *(uint32*) zout.next_out = original_size; // uncompressed_length - - chunk->free_func(chunk->data); - chunk->data = compressed; - chunk->free_func = Chunk::free_func_delete; - chunk->len = - ((char*) zout.next_out - compressed) + sizeof(uint32); - - DBG_LOG(DBG_CHUNKEDIO, "zlib write: size=%d compressed=%d", - original_size, chunk->len); - } - - return io->Write(chunk); - } - -void CompressedChunkedIO::Stats(char* buffer, int length) - { - const Statistics* stats = io->Stats(); - - int i = snprintf(buffer, length, "compression=%.2f/%.2f ", - uncompressed_bytes_read ? double(stats->bytes_read) / uncompressed_bytes_read : -1, - uncompressed_bytes_written ? double(stats->bytes_written) / uncompressed_bytes_written : -1 ); - - io->Stats(buffer + i, length - i); - buffer[length-1] = '\0'; - } diff --git a/src/ChunkedIO.h b/src/ChunkedIO.h deleted file mode 100644 index 24c7a489d2..0000000000 --- a/src/ChunkedIO.h +++ /dev/null @@ -1,362 +0,0 @@ -// Implements non-blocking chunk-wise I/O. - -#ifndef CHUNKEDIO_H -#define CHUNKEDIO_H - -#include "zeek-config.h" -#include "List.h" -#include "util.h" -#include "Flare.h" -#include "iosource/FD_Set.h" -#include - -#ifdef NEED_KRB5_H -# include -#endif - -class CompressedChunkedIO; - -// #define DEBUG_COMMUNICATION 10 - -// Abstract base class. -class ChunkedIO { -public: - ChunkedIO(); - virtual ~ChunkedIO() { } - - struct Chunk { - typedef void (*FreeFunc)(char*); - static void free_func_free(char* data) { free(data); } - static void free_func_delete(char* data) { delete [] data; } - - Chunk() - : data(), len(), free_func(free_func_delete) - { } - - // Takes ownership of data. - Chunk(char* arg_data, uint32 arg_len, - FreeFunc arg_ff = free_func_delete) - : data(arg_data), len(arg_len), free_func(arg_ff) - { } - - ~Chunk() - { free_func(data); } - - char* data; - uint32 len; - FreeFunc free_func; - }; - - // Initialization before any I/O operation is performed. Returns false - // on any form of error. - virtual bool Init() { return true; } - - // Tries to read the next chunk of data. If it can be read completely, - // a pointer to it is returned in 'chunk' (ownership of chunk is - // passed). If not, 'chunk' is set to nil. Returns false if any - // I/O error occurred (use Eof() to see if it's an end-of-file). - // If 'may_block' is true, we explicitly allow blocking. - virtual bool Read(Chunk** chunk, bool may_block = false) = 0; - - // Puts the chunk into the write queue and writes as much data - // as possible (takes ownership of chunk). - // Returns false on any I/O error. - virtual bool Write(Chunk* chunk) = 0; - - // Tries to write as much as currently possible. - // Returns false on any I/O error. - virtual bool Flush() = 0; - - // If an I/O error has been encountered, returns a string describing it. - virtual const char* Error() = 0; - - // Return true if there is currently at least one chunk available - // for reading. - virtual bool CanRead() = 0; - - // Return true if there is currently at least one chunk waiting to be - // written. - virtual bool CanWrite() = 0; - - // Returns true if source believes that there won't be much data soon. - virtual bool IsIdle() = 0; - - // Returns true if internal write buffers are about to fill up. - virtual bool IsFillingUp() = 0; - - // Throws away buffered data. - virtual void Clear() = 0; - - // Returns true,if end-of-file has been reached. - virtual bool Eof() = 0; - - // Returns underlying fd if available, -1 otherwise. - virtual int Fd() { return -1; } - - // Returns supplementary file descriptors that become read-ready in order - // to signal that there is some work that can be performed. - virtual iosource::FD_Set ExtraReadFDs() const - { return iosource::FD_Set(); } - - // Makes sure that no additional protocol data is written into - // the output stream. If this is activated, the output cannot - // be read again by any of these classes! - void MakePure() { pure = true; } - bool IsPure() { return pure; } - - // Writes a log message to the error_fd. - void Log(const char* str); - - struct Statistics { - Statistics() - { - bytes_read = 0; - bytes_written = 0; - chunks_read = 0; - chunks_written = 0; - reads = 0; - writes = 0; - pending = 0; - } - - unsigned long bytes_read; - unsigned long bytes_written; - unsigned long chunks_read; - unsigned long chunks_written; - unsigned long reads; // # calls which transferred > 0 bytes - unsigned long writes; - unsigned long pending; - }; - - // Returns raw statistics. - const Statistics* Stats() const { return &stats; } - - // Puts a formatted string containing statistics into buffer. - virtual void Stats(char* buffer, int length); - -#ifdef DEBUG_COMMUNICATION - void DumpDebugData(const char* basefnname, bool want_reads); -#endif - -protected: - void InternalError(const char* msg) - // We can't use the reporter here as we might be running in a - // sub-process. - { fprintf(stderr, "%s", msg); abort(); } - - Statistics stats; - const char* tag; - -#ifdef DEBUG_COMMUNICATION - void AddToBuffer(char* data, bool is_read) - { AddToBuffer(strlen(data), data, is_read); } - void AddToBuffer(uint32 len, char* data, bool is_read); - void AddToBuffer(Chunk* chunk, bool is_read); - std::list data_read; - std::list data_written; -#endif - -private: - bool pure; -}; - -// Chunked I/O using a file descriptor. -class ChunkedIOFd : public ChunkedIO { -public: - // fd is an open bidirectional file descriptor, tag is used in error - // messages, and pid gives a pid to monitor (if the process dies, we - // return EOF). - ChunkedIOFd(int fd, const char* tag, pid_t pid = 0); - ~ChunkedIOFd() override; - - bool Read(Chunk** chunk, bool may_block = false) override; - bool Write(Chunk* chunk) override; - bool Flush() override; - const char* Error() override; - bool CanRead() override; - bool CanWrite() override; - bool IsIdle() override; - bool IsFillingUp() override; - void Clear() override; - bool Eof() override { return eof; } - int Fd() override { return fd; } - iosource::FD_Set ExtraReadFDs() const override; - void Stats(char* buffer, int length) override; - -private: - - bool PutIntoWriteBuffer(Chunk* chunk); - bool FlushWriteBuffer(); - Chunk* ExtractChunk(); - - // Returns size of next chunk in buffer or 0 if none. - uint32 ChunkAvailable(); - - // Flushes if it thinks it is time to. - bool OptionalFlush(); - - // Concatenates the the data of the two chunks forming a new one. - // The old chunkds are deleted. - Chunk* ConcatChunks(Chunk* c1, Chunk* c2); - - // Reads/writes on chunk of upto BUFFER_SIZE bytes. - bool WriteChunk(Chunk* chunk, bool partial); - bool ReadChunk(Chunk** chunk, bool may_block); - - int fd; - bool eof; - double last_flush; - int failed_reads; - - // Optimally, this should match the file descriptor's - // buffer size (for sockets, it may be helpful to - // increase the send/receive buffers). - static const unsigned int BUFFER_SIZE = 1024 * 1024 * 1; - - // We 'or' this to the length of a data chunk to mark - // that it's part of a larger one. This has to be larger - // than BUFFER_SIZE. - static const uint32 FLAG_PARTIAL = 0x80000000; - - char* read_buffer; - uint32 read_len; - uint32 read_pos; - Chunk* partial; // when we read an oversized chunk, we store it here - - char* write_buffer; - uint32 write_len; - uint32 write_pos; - - struct ChunkQueue { - Chunk* chunk; - ChunkQueue* next; - }; - - // Chunks that don't fit into our write buffer. - ChunkQueue* pending_head; - ChunkQueue* pending_tail; - - pid_t pid; - bro::Flare write_flare; - bro::Flare read_flare; -}; - -// From OpenSSL. We forward-declare these here to avoid introducing a -// dependency on OpenSSL headers just for this header file. -typedef struct ssl_ctx_st SSL_CTX; -typedef struct ssl_st SSL; - -// Chunked I/O using an SSL connection. -class ChunkedIOSSL : public ChunkedIO { -public: - // Argument is an open socket and a flag indicating whether we are the - // server side of the connection. - ChunkedIOSSL(int socket, bool server); - ~ChunkedIOSSL() override; - - bool Init() override; - bool Read(Chunk** chunk, bool mayblock = false) override; - bool Write(Chunk* chunk) override; - bool Flush() override; - const char* Error() override; - bool CanRead() override; - bool CanWrite() override; - bool IsIdle() override; - bool IsFillingUp() override; - void Clear() override; - bool Eof() override { return eof; } - int Fd() override { return socket; } - iosource::FD_Set ExtraReadFDs() const override; - void Stats(char* buffer, int length) override; - -private: - - // Only returns true if all data has been read. If not, call - // it again with the same parameters as long as error is not - // set to true. - bool ReadData(char* p, uint32 len, bool* error); - // Same for writing. - bool WriteData(char* p, uint32 len, bool* error); - - int socket; - int last_ret; // last error code - bool eof; - - bool server; // are we the server? - bool setup; // has the connection been setup successfully? - - SSL* ssl; - - // Write queue. - struct Queue { - Chunk* chunk; - Queue* next; - }; - - // The chunk part we are reading/writing - enum State { LEN, DATA }; - - State write_state; - Queue* write_head; - Queue* write_tail; - - State read_state; - Chunk* read_chunk; - char* read_ptr; - - // One SSL for all connections. - static SSL_CTX* ctx; - - bro::Flare write_flare; -}; - -#include - -// Wrapper class around a another ChunkedIO which the (un-)compresses data. -class CompressedChunkedIO : public ChunkedIO { -public: - explicit CompressedChunkedIO(ChunkedIO* arg_io) // takes ownership - : io(arg_io), zin(), zout(), error(), compress(), uncompress(), - uncompressed_bytes_read(), uncompressed_bytes_written() {} - ~CompressedChunkedIO() override { delete io; } - - bool Init() override; // does *not* call arg_io->Init() - bool Read(Chunk** chunk, bool may_block = false) override; - bool Write(Chunk* chunk) override; - bool Flush() override { return io->Flush(); } - const char* Error() override { return error ? error : io->Error(); } - bool CanRead() override { return io->CanRead(); } - bool CanWrite() override { return io->CanWrite(); } - bool IsIdle() override { return io->IsIdle(); } - bool IsFillingUp() override { return io->IsFillingUp(); } - void Clear() override { return io->Clear(); } - bool Eof() override { return io->Eof(); } - - int Fd() override { return io->Fd(); } - iosource::FD_Set ExtraReadFDs() const override - { return io->ExtraReadFDs(); } - void Stats(char* buffer, int length) override; - - void EnableCompression(int level) - { deflateInit(&zout, level); compress = true; } - void EnableDecompression() - { inflateInit(&zin); uncompress = true; } - -protected: - // Only compress block with size >= this. - static const unsigned int MIN_COMPRESS_SIZE = 30; - - ChunkedIO* io; - z_stream zin; - z_stream zout; - const char* error; - - bool compress; - bool uncompress; - - // Keep some statistics. - unsigned long uncompressed_bytes_read; - unsigned long uncompressed_bytes_written; -}; - -#endif diff --git a/src/Conn.cc b/src/Conn.cc index 48a9940875..31c1b1a191 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -50,70 +50,10 @@ void ConnectionTimer::Dispatch(double t, int is_expire) reporter->InternalError("reference count inconsistency in ConnectionTimer::Dispatch"); } -IMPLEMENT_SERIAL(ConnectionTimer, SER_CONNECTION_TIMER); - -bool ConnectionTimer::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_CONNECTION_TIMER, Timer); - - // We enumerate all the possible timer functions here ... This - // has to match the list is DoUnserialize()! - char type = 0; - - if ( timer == timer_func(&Connection::DeleteTimer) ) - type = 1; - else if ( timer == timer_func(&Connection::InactivityTimer) ) - type = 2; - else if ( timer == timer_func(&Connection::StatusUpdateTimer) ) - type = 3; - else if ( timer == timer_func(&Connection::RemoveConnectionTimer) ) - type = 4; - else - reporter->InternalError("unknown function in ConnectionTimer::DoSerialize()"); - - return conn->Serialize(info) && SERIALIZE(type) && SERIALIZE(do_expire); - } - -bool ConnectionTimer::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Timer); - - conn = Connection::Unserialize(info); - if ( ! conn ) - return false; - - char type; - - if ( ! UNSERIALIZE(&type) || ! UNSERIALIZE(&do_expire) ) - return false; - - switch ( type ) { - case 1: - timer = timer_func(&Connection::DeleteTimer); - break; - case 2: - timer = timer_func(&Connection::InactivityTimer); - break; - case 3: - timer = timer_func(&Connection::StatusUpdateTimer); - break; - case 4: - timer = timer_func(&Connection::RemoveConnectionTimer); - break; - default: - info->s->Error("unknown connection timer function"); - return false; - } - - return true; - } - uint64 Connection::total_connections = 0; uint64 Connection::current_connections = 0; uint64 Connection::external_connections = 0; -IMPLEMENT_SERIAL(Connection, SER_CONNECTION); - Connection::Connection(NetSessions* s, HashKey* k, double t, const ConnID* id, uint32 flow, const Packet* pkt, const EncapsulationStack* arg_encap) @@ -900,139 +840,6 @@ void Connection::IDString(ODesc* d) const d->Add(ntohs(resp_port)); } -bool Connection::Serialize(SerialInfo* info) const - { - return SerialObj::Serialize(info); - } - -Connection* Connection::Unserialize(UnserialInfo* info) - { - return (Connection*) SerialObj::Unserialize(info, SER_CONNECTION); - } - -bool Connection::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_CONNECTION, BroObj); - - // First we write the members which are needed to - // create the HashKey. - if ( ! SERIALIZE(orig_addr) || ! SERIALIZE(resp_addr) ) - return false; - - if ( ! SERIALIZE(orig_port) || ! SERIALIZE(resp_port) ) - return false; - - if ( ! SERIALIZE(timers.length()) ) - return false; - - loop_over_list(timers, i) - if ( ! timers[i]->Serialize(info) ) - return false; - - SERIALIZE_OPTIONAL(conn_val); - - // FIXME: RuleEndpointState not yet serializable. - // FIXME: Analyzers not yet serializable. - - return - SERIALIZE(int(proto)) && - SERIALIZE(history) && - SERIALIZE(hist_seen) && - SERIALIZE(start_time) && - SERIALIZE(last_time) && - SERIALIZE(inactivity_timeout) && - SERIALIZE(suppress_event) && - SERIALIZE(login_conn != 0) && - SERIALIZE_BIT(installed_status_timer) && - SERIALIZE_BIT(timers_canceled) && - SERIALIZE_BIT(is_active) && - SERIALIZE_BIT(skip) && - SERIALIZE_BIT(weird) && - SERIALIZE_BIT(finished) && - SERIALIZE_BIT(record_packets) && - SERIALIZE_BIT(record_contents); - } - -bool Connection::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroObj); - - // Build the hash key first. Some of the recursive *::Unserialize() - // functions may need it. - ConnID id; - - if ( ! UNSERIALIZE(&orig_addr) || ! UNSERIALIZE(&resp_addr) ) - goto error; - - if ( ! UNSERIALIZE(&orig_port) || ! UNSERIALIZE(&resp_port) ) - goto error; - - id.src_addr = orig_addr; - id.dst_addr = resp_addr; - // This doesn't work for ICMP. But I guess this is not really important. - id.src_port = orig_port; - id.dst_port = resp_port; - id.is_one_way = 0; // ### incorrect for ICMP - key = BuildConnIDHashKey(id); - - int len; - if ( ! UNSERIALIZE(&len) ) - goto error; - - while ( len-- ) - { - Timer* t = Timer::Unserialize(info); - if ( ! t ) - goto error; - timers.append(t); - } - - UNSERIALIZE_OPTIONAL(conn_val, - (RecordVal*) Val::Unserialize(info, connection_type)); - - int iproto; - - if ( ! (UNSERIALIZE(&iproto) && - UNSERIALIZE(&history) && - UNSERIALIZE(&hist_seen) && - UNSERIALIZE(&start_time) && - UNSERIALIZE(&last_time) && - UNSERIALIZE(&inactivity_timeout) && - UNSERIALIZE(&suppress_event)) ) - goto error; - - proto = static_cast(iproto); - - bool has_login_conn; - if ( ! UNSERIALIZE(&has_login_conn) ) - goto error; - - login_conn = has_login_conn ? (LoginConn*) this : 0; - - UNSERIALIZE_BIT(installed_status_timer); - UNSERIALIZE_BIT(timers_canceled); - UNSERIALIZE_BIT(is_active); - UNSERIALIZE_BIT(skip); - UNSERIALIZE_BIT(weird); - UNSERIALIZE_BIT(finished); - UNSERIALIZE_BIT(record_packets); - UNSERIALIZE_BIT(record_contents); - - // Hmm... Why does each connection store a sessions ptr? - sessions = ::sessions; - - root_analyzer = 0; - primary_PIA = 0; - conn_timer_mgr = 0; - - return true; - -error: - abort(); - CancelTimers(); - return false; - } - void Connection::SetRootAnalyzer(analyzer::TransportLayerAnalyzer* analyzer, analyzer::pia::PIA* pia) { root_analyzer = analyzer; diff --git a/src/Conn.h b/src/Conn.h index fb7f5be0b4..bd5ddaae92 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -11,7 +11,6 @@ #include "Dict.h" #include "Val.h" #include "Timer.h" -#include "Serializer.h" #include "RuleMatcher.h" #include "IPAddr.h" #include "TunnelEncapsulation.h" @@ -235,11 +234,6 @@ public: // Returns true if connection has been received externally. bool IsExternal() const { return conn_timer_mgr != 0; } - bool Serialize(SerialInfo* info) const; - static Connection* Unserialize(UnserialInfo* info); - - DECLARE_SERIAL(Connection); - // Statistics. // Just a lower bound. @@ -385,8 +379,6 @@ protected: void Init(Connection* conn, timer_func timer, int do_expire); - DECLARE_SERIAL(ConnectionTimer); - Connection* conn; timer_func timer; int do_expire; diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index d92b5e0ddf..f7090f151b 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -12,8 +12,7 @@ DebugLogger debug_logger; // Same order here as in DebugStream. DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { { "serial", 0, false }, { "rules", 0, false }, - { "state", 0, false }, { "chunkedio", 0, false }, - {"string", 0, false }, + { "state", 0, false }, {"string", 0, false }, { "notifiers", 0, false }, { "main-loop", 0, false }, { "dpd", 0, false }, { "tm", 0, false }, { "logging", 0, false }, {"input", 0, false }, diff --git a/src/DebugLogger.h b/src/DebugLogger.h index db646bd0cf..2e24c7064f 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -14,10 +14,9 @@ // an entry to DebugLogger::streams in DebugLogger.cc. enum DebugStream { - DBG_SERIAL, // Serialization + DBG_SERIAL, // Serialization DBG_RULES, // Signature matching DBG_STATE, // StateAccess logging - DBG_CHUNKEDIO, // ChunkedIO logging DBG_STRING, // String code DBG_NOTIFIERS, // Notifiers (see StateAccess.h) DBG_MAINLOOP, // Main IOSource loop diff --git a/src/Event.cc b/src/Event.cc index aabbcb4ddc..2389c618d7 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -58,12 +58,6 @@ void Event::Dispatch(bool no_remote) if ( src == SOURCE_BROKER ) no_remote = true; - if ( event_serializer ) - { - SerialInfo info(event_serializer); - event_serializer->Serialize(&info, handler->Name(), &args); - } - if ( handler->ErrorHandler() ) reporter->BeginErrorHandler(); diff --git a/src/Event.h b/src/Event.h index cafe0057d6..475dc5f577 100644 --- a/src/Event.h +++ b/src/Event.h @@ -4,7 +4,6 @@ #define event_h #include "EventRegistry.h" -#include "Serializer.h" #include "analyzer/Tag.h" #include "analyzer/Analyzer.h" diff --git a/src/EventHandler.cc b/src/EventHandler.cc index 718e6d6ae8..0f06ad50ca 100644 --- a/src/EventHandler.cc +++ b/src/EventHandler.cc @@ -171,23 +171,3 @@ void EventHandler::NewEvent(val_list* vl) mgr.Dispatch(ev); } -bool EventHandler::Serialize(SerialInfo* info) const - { - return SERIALIZE(name); - } - -EventHandler* EventHandler::Unserialize(UnserialInfo* info) - { - char* name; - if ( ! UNSERIALIZE_STR(&name, 0) ) - return 0; - - EventHandler* h = event_registry->Lookup(name); - if ( ! h ) - { - h = new EventHandler(name); - event_registry->Register(h); - } - - return h; - } diff --git a/src/EventHandler.h b/src/EventHandler.h index 216badee4b..1f3e902b8f 100644 --- a/src/EventHandler.h +++ b/src/EventHandler.h @@ -11,9 +11,6 @@ class Func; class FuncType; -class Serializer; -class SerialInfo; -class UnserialInfo; class EventHandler { public: @@ -56,11 +53,6 @@ public: void SetGenerateAlways() { generate_always = true; } bool GenerateAlways() { return generate_always; } - // We don't serialize the handler(s) itself here, but - // just the reference to it. - bool Serialize(SerialInfo* info) const; - static EventHandler* Unserialize(UnserialInfo* info); - private: void NewEvent(val_list* vl); // Raise new_event() meta event. diff --git a/src/Expr.cc b/src/Expr.cc index 8f003de030..e19536359a 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -201,56 +201,6 @@ void Expr::RuntimeErrorWithCallStack(const std::string& msg) const } } -bool Expr::Serialize(SerialInfo* info) const - { - return SerialObj::Serialize(info); - } - -Expr* Expr::Unserialize(UnserialInfo* info, BroExprTag want) - { - Expr* e = (Expr*) SerialObj::Unserialize(info, SER_EXPR); - - if ( ! e ) - return 0; - - if ( want != EXPR_ANY && e->tag != want ) - { - info->s->Error("wrong expression type"); - Unref(e); - return 0; - } - - return e; - } - -bool Expr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_EXPR, BroObj); - - if ( ! (SERIALIZE(char(tag)) && SERIALIZE(paren)) ) - return false; - - SERIALIZE_OPTIONAL(type); - return true; - } - -bool Expr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroObj); - - char c; - if ( ! (UNSERIALIZE(&c) && UNSERIALIZE(&paren)) ) - return 0; - - tag = BroExprTag(c); - - BroType* t = 0; - UNSERIALIZE_OPTIONAL(t, BroType::Unserialize(info)); - SetType(t); - return true; - } - - NameExpr::NameExpr(ID* arg_id, bool const_init) : Expr(EXPR_NAME) { id = arg_id; @@ -350,55 +300,6 @@ void NameExpr::ExprDescribe(ODesc* d) const } } -IMPLEMENT_SERIAL(NameExpr, SER_NAME_EXPR); - -bool NameExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_NAME_EXPR, Expr); - - // Write out just the name of the function if requested. - if ( info->globals_as_names && id->IsGlobal() ) - return SERIALIZE('n') && SERIALIZE(id->Name()) && - SERIALIZE(in_const_init); - else - return SERIALIZE('f') && id->Serialize(info) && - SERIALIZE(in_const_init); - } - -bool NameExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Expr); - - char type; - if ( ! UNSERIALIZE(&type) ) - return false; - - if ( type == 'n' ) - { - const char* name; - if ( ! UNSERIALIZE_STR(&name, 0) ) - return false; - - id = global_scope()->Lookup(name); - if ( id ) - ::Ref(id); - else - reporter->Warning("configuration changed: unserialized unknown global name from persistent state"); - - delete [] name; - } - else - id = ID::Unserialize(info); - - if ( ! id ) - return false; - - if ( ! UNSERIALIZE(&in_const_init) ) - return false; - - return true; - } - ConstExpr::ConstExpr(Val* arg_val) : Expr(EXPR_CONST) { val = arg_val; @@ -437,22 +338,6 @@ TraversalCode ConstExpr::Traverse(TraversalCallback* cb) const HANDLE_TC_EXPR_POST(tc); } -IMPLEMENT_SERIAL(ConstExpr, SER_CONST_EXPR); - -bool ConstExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_CONST_EXPR, Expr); - return val->Serialize(info); - } - -bool ConstExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Expr); - val = Val::Unserialize(info); - return val != 0; - } - - UnaryExpr::UnaryExpr(BroExprTag arg_tag, Expr* arg_op) : Expr(arg_tag) { op = arg_op; @@ -551,21 +436,6 @@ void UnaryExpr::ExprDescribe(ODesc* d) const } } -IMPLEMENT_SERIAL(UnaryExpr, SER_UNARY_EXPR); - -bool UnaryExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_UNARY_EXPR, Expr); - return op->Serialize(info); - } - -bool UnaryExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Expr); - op = Expr::Unserialize(info); - return op != 0; - } - BinaryExpr::~BinaryExpr() { Unref(op1); @@ -1039,26 +909,6 @@ void BinaryExpr::PromoteType(TypeTag t, bool is_vector) SetType(is_vector ? new VectorType(base_type(t)) : base_type(t)); } -IMPLEMENT_SERIAL(BinaryExpr, SER_BINARY_EXPR); - -bool BinaryExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_BINARY_EXPR, Expr); - return op1->Serialize(info) && op2->Serialize(info); - } - -bool BinaryExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Expr); - - op1 = Expr::Unserialize(info); - if ( ! op1 ) - return false; - - op2 = Expr::Unserialize(info); - return op2 != 0; - } - CloneExpr::CloneExpr(Expr* arg_op) : UnaryExpr(EXPR_CLONE, arg_op) { if ( IsError() ) @@ -1089,20 +939,6 @@ Val* CloneExpr::Fold(Val* v) const return v->Clone(); } -IMPLEMENT_SERIAL(CloneExpr, SER_CLONE_EXPR); - -bool CloneExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_CLONE_EXPR, UnaryExpr); - return true; - } - -bool CloneExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - return true; - } - IncrExpr::IncrExpr(BroExprTag arg_tag, Expr* arg_op) : UnaryExpr(arg_tag, arg_op->MakeLvalue()) { @@ -1194,20 +1030,6 @@ int IncrExpr::IsPure() const return 0; } -IMPLEMENT_SERIAL(IncrExpr, SER_INCR_EXPR); - -bool IncrExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_INCR_EXPR, UnaryExpr); - return true; - } - -bool IncrExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - return true; - } - ComplementExpr::ComplementExpr(Expr* arg_op) : UnaryExpr(EXPR_COMPLEMENT, arg_op) { if ( IsError() ) @@ -1227,20 +1049,6 @@ Val* ComplementExpr::Fold(Val* v) const return val_mgr->GetCount(~ v->InternalUnsigned()); } -IMPLEMENT_SERIAL(ComplementExpr, SER_COMPLEMENT_EXPR); - -bool ComplementExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_COMPLEMENT_EXPR, UnaryExpr); - return true; - } - -bool ComplementExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - return true; - } - NotExpr::NotExpr(Expr* arg_op) : UnaryExpr(EXPR_NOT, arg_op) { if ( IsError() ) @@ -1260,20 +1068,6 @@ Val* NotExpr::Fold(Val* v) const return val_mgr->GetBool(! v->InternalInt()); } -IMPLEMENT_SERIAL(NotExpr, SER_NOT_EXPR); - -bool NotExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_NOT_EXPR, UnaryExpr); - return true; - } - -bool NotExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - return true; - } - PosExpr::PosExpr(Expr* arg_op) : UnaryExpr(EXPR_POSITIVE, arg_op) { if ( IsError() ) @@ -1310,20 +1104,6 @@ Val* PosExpr::Fold(Val* v) const return val_mgr->GetInt(v->CoerceToInt()); } -IMPLEMENT_SERIAL(PosExpr, SER_POS_EXPR); - -bool PosExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_POS_EXPR, UnaryExpr); - return true; - } - -bool PosExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - return true; - } - NegExpr::NegExpr(Expr* arg_op) : UnaryExpr(EXPR_NEGATE, arg_op) { if ( IsError() ) @@ -1360,21 +1140,6 @@ Val* NegExpr::Fold(Val* v) const return val_mgr->GetInt(- v->CoerceToInt()); } - -IMPLEMENT_SERIAL(NegExpr, SER_NEG_EXPR); - -bool NegExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_NEG_EXPR, UnaryExpr); - return true; - } - -bool NegExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - return true; - } - SizeExpr::SizeExpr(Expr* arg_op) : UnaryExpr(EXPR_SIZE, arg_op) { if ( IsError() ) @@ -1402,21 +1167,6 @@ Val* SizeExpr::Fold(Val* v) const return v->SizeVal(); } -IMPLEMENT_SERIAL(SizeExpr, SER_SIZE_EXPR); - -bool SizeExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_SIZE_EXPR, UnaryExpr); - return true; - } - -bool SizeExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - return true; - } - - AddExpr::AddExpr(Expr* arg_op1, Expr* arg_op2) : BinaryExpr(EXPR_ADD, arg_op1, arg_op2) { @@ -1464,20 +1214,6 @@ void AddExpr::Canonicize() SwapOps(); } -IMPLEMENT_SERIAL(AddExpr, SER_ADD_EXPR); - -bool AddExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_ADD_EXPR, BinaryExpr); - return true; - } - -bool AddExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BinaryExpr); - return true; - } - AddToExpr::AddToExpr(Expr* arg_op1, Expr* arg_op2) : BinaryExpr(EXPR_ADD_TO, is_vector(arg_op1) ? arg_op1 : arg_op1->MakeLvalue(), arg_op2) @@ -1559,20 +1295,6 @@ Val* AddToExpr::Eval(Frame* f) const return 0; } -IMPLEMENT_SERIAL(AddToExpr, SER_ADD_TO_EXPR); - -bool AddToExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_ADD_TO_EXPR, BinaryExpr); - return true; - } - -bool AddToExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BinaryExpr); - return true; - } - SubExpr::SubExpr(Expr* arg_op1, Expr* arg_op2) : BinaryExpr(EXPR_SUB, arg_op1, arg_op2) { @@ -1624,20 +1346,6 @@ SubExpr::SubExpr(Expr* arg_op1, Expr* arg_op2) } } -IMPLEMENT_SERIAL(SubExpr, SER_SUB_EXPR); - -bool SubExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_SUB_EXPR, BinaryExpr); - return true; - } - -bool SubExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BinaryExpr); - return true; - } - RemoveFromExpr::RemoveFromExpr(Expr* arg_op1, Expr* arg_op2) : BinaryExpr(EXPR_REMOVE_FROM, arg_op1->MakeLvalue(), arg_op2) { @@ -1682,20 +1390,6 @@ Val* RemoveFromExpr::Eval(Frame* f) const return 0; } -IMPLEMENT_SERIAL(RemoveFromExpr, SER_REMOVE_FROM_EXPR); - -bool RemoveFromExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_REMOVE_FROM_EXPR, BinaryExpr); - return true; - } - -bool RemoveFromExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BinaryExpr); - return true; - } - TimesExpr::TimesExpr(Expr* arg_op1, Expr* arg_op2) : BinaryExpr(EXPR_TIMES, arg_op1, arg_op2) { @@ -1732,20 +1426,6 @@ void TimesExpr::Canonicize() SwapOps(); } -IMPLEMENT_SERIAL(TimesExpr, SER_TIMES_EXPR); - -bool TimesExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_TIMES_EXPR, BinaryExpr); - return true; - } - -bool TimesExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BinaryExpr); - return true; - } - DivideExpr::DivideExpr(Expr* arg_op1, Expr* arg_op2) : BinaryExpr(EXPR_DIVIDE, arg_op1, arg_op2) { @@ -1810,20 +1490,6 @@ Val* DivideExpr::AddrFold(Val* v1, Val* v2) const return new SubNetVal(a, mask); } -IMPLEMENT_SERIAL(DivideExpr, SER_DIVIDE_EXPR); - -bool DivideExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_DIVIDE_EXPR, BinaryExpr); - return true; - } - -bool DivideExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BinaryExpr); - return true; - } - ModExpr::ModExpr(Expr* arg_op1, Expr* arg_op2) : BinaryExpr(EXPR_MOD, arg_op1, arg_op2) { @@ -1844,20 +1510,6 @@ ModExpr::ModExpr(Expr* arg_op1, Expr* arg_op2) ExprError("requires integral operands"); } -IMPLEMENT_SERIAL(ModExpr, SER_MOD_EXPR); - -bool ModExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_MOD_EXPR, BinaryExpr); - return true; - } - -bool ModExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BinaryExpr); - return true; - } - BoolExpr::BoolExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2) : BinaryExpr(arg_tag, arg_op1, arg_op2) { @@ -2014,20 +1666,6 @@ Val* BoolExpr::Eval(Frame* f) const return result; } -IMPLEMENT_SERIAL(BoolExpr, SER_BOOL_EXPR); - -bool BoolExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_BOOL_EXPR, BinaryExpr); - return true; - } - -bool BoolExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BinaryExpr); - return true; - } - BitExpr::BitExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2) : BinaryExpr(arg_tag, arg_op1, arg_op2) { @@ -2078,20 +1716,6 @@ BitExpr::BitExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2) ExprError("requires \"count\" or compatible \"set\" operands"); } -IMPLEMENT_SERIAL(BitExpr, SER_BIT_EXPR); - -bool BitExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_BIT_EXPR, BinaryExpr); - return true; - } - -bool BitExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BinaryExpr); - return true; - } - EqExpr::EqExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2) : BinaryExpr(arg_tag, arg_op1, arg_op2) { @@ -2193,20 +1817,6 @@ Val* EqExpr::Fold(Val* v1, Val* v2) const return BinaryExpr::Fold(v1, v2); } -IMPLEMENT_SERIAL(EqExpr, SER_EQ_EXPR); - -bool EqExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_EQ_EXPR, BinaryExpr); - return true; - } - -bool EqExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BinaryExpr); - return true; - } - RelExpr::RelExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2) : BinaryExpr(arg_tag, arg_op1, arg_op2) { @@ -2264,20 +1874,6 @@ void RelExpr::Canonicize() } } -IMPLEMENT_SERIAL(RelExpr, SER_REL_EXPR); - -bool RelExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_REL_EXPR, BinaryExpr); - return true; - } - -bool RelExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BinaryExpr); - return true; - } - CondExpr::CondExpr(Expr* arg_op1, Expr* arg_op2, Expr* arg_op3) : Expr(EXPR_COND) { @@ -2429,32 +2025,6 @@ void CondExpr::ExprDescribe(ODesc* d) const op3->Describe(d); } -IMPLEMENT_SERIAL(CondExpr, SER_COND_EXPR); - -bool CondExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_COND_EXPR, Expr); - return op1->Serialize(info) && op2->Serialize(info) - && op3->Serialize(info); - } - -bool CondExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Expr); - - op1 = Expr::Unserialize(info); - if ( ! op1 ) - return false; - - op2 = Expr::Unserialize(info); - if ( ! op2 ) - return false; - - op3 = Expr::Unserialize(info); - - return op3 != 0; - } - RefExpr::RefExpr(Expr* arg_op) : UnaryExpr(EXPR_REF, arg_op) { if ( IsError() ) @@ -2476,20 +2046,6 @@ void RefExpr::Assign(Frame* f, Val* v, Opcode opcode) op->Assign(f, v, opcode); } -IMPLEMENT_SERIAL(RefExpr, SER_REF_EXPR); - -bool RefExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_REF_EXPR, UnaryExpr); - return true; - } - -bool RefExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - return true; - } - AssignExpr::AssignExpr(Expr* arg_op1, Expr* arg_op2, int arg_is_init, Val* arg_val, attr_list* arg_attrs) : BinaryExpr(EXPR_ASSIGN, @@ -2889,22 +2445,6 @@ int AssignExpr::IsPure() const return 0; } -IMPLEMENT_SERIAL(AssignExpr, SER_ASSIGN_EXPR); - -bool AssignExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_ASSIGN_EXPR, BinaryExpr); - SERIALIZE_OPTIONAL(val); - return SERIALIZE(is_init); - } - -bool AssignExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BinaryExpr); - UNSERIALIZE_OPTIONAL(val, Val::Unserialize(info)); - return UNSERIALIZE(&is_init); - } - IndexSliceAssignExpr::IndexSliceAssignExpr(Expr* op1, Expr* op2, int is_init) : AssignExpr(op1, op2, is_init) { @@ -2929,20 +2469,6 @@ Val* IndexSliceAssignExpr::Eval(Frame* f) const return 0; } -IMPLEMENT_SERIAL(IndexSliceAssignExpr, SER_INDEX_SLICE_ASSIGN_EXPR); - -bool IndexSliceAssignExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_INDEX_SLICE_ASSIGN_EXPR, AssignExpr); - return true; - } - -bool IndexSliceAssignExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(AssignExpr); - return true; - } - IndexExpr::IndexExpr(Expr* arg_op1, ListExpr* arg_op2, bool arg_is_slice) : BinaryExpr(EXPR_INDEX, arg_op1, arg_op2), is_slice(arg_is_slice) { @@ -3334,21 +2860,6 @@ TraversalCode IndexExpr::Traverse(TraversalCallback* cb) const HANDLE_TC_EXPR_POST(tc); } - -IMPLEMENT_SERIAL(IndexExpr, SER_INDEX_EXPR); - -bool IndexExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_INDEX_EXPR, BinaryExpr); - return SERIALIZE(is_slice); - } - -bool IndexExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BinaryExpr); - return UNSERIALIZE(&is_slice); - } - FieldExpr::FieldExpr(Expr* arg_op, const char* arg_field_name) : UnaryExpr(EXPR_FIELD, arg_op) { @@ -3446,29 +2957,6 @@ void FieldExpr::ExprDescribe(ODesc* d) const d->Add(field); } -IMPLEMENT_SERIAL(FieldExpr, SER_FIELD_EXPR); - -bool FieldExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_FIELD_EXPR, UnaryExpr); - - if ( ! (SERIALIZE(field_name) && SERIALIZE(field) ) ) - return false; - - return td->Serialize(info); - } - -bool FieldExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - - if ( ! (UNSERIALIZE_STR(&field_name, 0) && UNSERIALIZE(&field) ) ) - return false; - - td = TypeDecl::Unserialize(info); - return td != 0; - } - HasFieldExpr::HasFieldExpr(Expr* arg_op, const char* arg_field_name) : UnaryExpr(EXPR_HAS_FIELD, arg_op) { @@ -3531,24 +3019,6 @@ void HasFieldExpr::ExprDescribe(ODesc* d) const d->Add(field); } -IMPLEMENT_SERIAL(HasFieldExpr, SER_HAS_FIELD_EXPR); - -bool HasFieldExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_HAS_FIELD_EXPR, UnaryExpr); - - // Serialize former "bool is_attr" member first for backwards compatibility. - return SERIALIZE(false) && SERIALIZE(field_name) && SERIALIZE(field); - } - -bool HasFieldExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - // Unserialize former "bool is_attr" member for backwards compatibility. - bool not_used; - return UNSERIALIZE(¬_used) && UNSERIALIZE_STR(&field_name, 0) && UNSERIALIZE(&field); - } - RecordConstructorExpr::RecordConstructorExpr(ListExpr* constructor_list) : UnaryExpr(EXPR_RECORD_CONSTRUCTOR, constructor_list) { @@ -3628,20 +3098,6 @@ void RecordConstructorExpr::ExprDescribe(ODesc* d) const d->Add("]"); } -IMPLEMENT_SERIAL(RecordConstructorExpr, SER_RECORD_CONSTRUCTOR_EXPR); - -bool RecordConstructorExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_RECORD_CONSTRUCTOR_EXPR, UnaryExpr); - return true; - } - -bool RecordConstructorExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - return true; - } - TableConstructorExpr::TableConstructorExpr(ListExpr* constructor_list, attr_list* arg_attrs, BroType* arg_type) : UnaryExpr(EXPR_TABLE_CONSTRUCTOR, constructor_list) @@ -3752,22 +3208,6 @@ void TableConstructorExpr::ExprDescribe(ODesc* d) const d->Add(")"); } -IMPLEMENT_SERIAL(TableConstructorExpr, SER_TABLE_CONSTRUCTOR_EXPR); - -bool TableConstructorExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_TABLE_CONSTRUCTOR_EXPR, UnaryExpr); - SERIALIZE_OPTIONAL(attrs); - return true; - } - -bool TableConstructorExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - UNSERIALIZE_OPTIONAL(attrs, Attributes::Unserialize(info)); - return true; - } - SetConstructorExpr::SetConstructorExpr(ListExpr* constructor_list, attr_list* arg_attrs, BroType* arg_type) : UnaryExpr(EXPR_SET_CONSTRUCTOR, constructor_list) @@ -3888,22 +3328,6 @@ void SetConstructorExpr::ExprDescribe(ODesc* d) const d->Add(")"); } -IMPLEMENT_SERIAL(SetConstructorExpr, SER_SET_CONSTRUCTOR_EXPR); - -bool SetConstructorExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_SET_CONSTRUCTOR_EXPR, UnaryExpr); - SERIALIZE_OPTIONAL(attrs); - return true; - } - -bool SetConstructorExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - UNSERIALIZE_OPTIONAL(attrs, Attributes::Unserialize(info)); - return true; - } - VectorConstructorExpr::VectorConstructorExpr(ListExpr* constructor_list, BroType* arg_type) : UnaryExpr(EXPR_VECTOR_CONSTRUCTOR, constructor_list) @@ -4007,20 +3431,6 @@ void VectorConstructorExpr::ExprDescribe(ODesc* d) const d->Add(")"); } -IMPLEMENT_SERIAL(VectorConstructorExpr, SER_VECTOR_CONSTRUCTOR_EXPR); - -bool VectorConstructorExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_VECTOR_CONSTRUCTOR_EXPR, UnaryExpr); - return true; - } - -bool VectorConstructorExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - return true; - } - FieldAssignExpr::FieldAssignExpr(const char* arg_field_name, Expr* value) : UnaryExpr(EXPR_FIELD_ASSIGN, value), field_name(arg_field_name) { @@ -4069,20 +3479,6 @@ void FieldAssignExpr::ExprDescribe(ODesc* d) const op->Describe(d); } -IMPLEMENT_SERIAL(FieldAssignExpr, SER_FIELD_ASSIGN_EXPR); - -bool FieldAssignExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_FIELD_ASSIGN_EXPR, UnaryExpr); - return true; - } - -bool FieldAssignExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - return true; - } - ArithCoerceExpr::ArithCoerceExpr(Expr* arg_op, TypeTag t) : UnaryExpr(EXPR_ARITH_COERCE, arg_op) { @@ -4160,21 +3556,6 @@ Val* ArithCoerceExpr::Fold(Val* v) const return result; } -IMPLEMENT_SERIAL(ArithCoerceExpr, SER_ARITH_COERCE_EXPR); - -bool ArithCoerceExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_ARITH_COERCE_EXPR, UnaryExpr); - return true; - } - -bool ArithCoerceExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - return true; - } - - RecordCoerceExpr::RecordCoerceExpr(Expr* op, RecordType* r) : UnaryExpr(EXPR_RECORD_COERCE, op) { @@ -4413,39 +3794,6 @@ Val* RecordCoerceExpr::Fold(Val* v) const return val; } -IMPLEMENT_SERIAL(RecordCoerceExpr, SER_RECORD_COERCE_EXPR); - -bool RecordCoerceExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_RECORD_COERCE_EXPR, UnaryExpr); - - if ( ! SERIALIZE(map_size) ) - return false; - - for ( int i = 0; i < map_size; ++i ) - if ( ! SERIALIZE(map[i]) ) - return false; - - return true; - } - -bool RecordCoerceExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - - if ( ! UNSERIALIZE(&map_size) ) - return false; - - map = new int[map_size]; - - for ( int i = 0; i < map_size; ++i ) - if ( ! UNSERIALIZE(&map[i]) ) - return false; - - return true; - } - - TableCoerceExpr::TableCoerceExpr(Expr* op, TableType* r) : UnaryExpr(EXPR_TABLE_COERCE, op) { @@ -4476,20 +3824,6 @@ Val* TableCoerceExpr::Fold(Val* v) const return new TableVal(Type()->AsTableType(), tv->Attrs()); } -IMPLEMENT_SERIAL(TableCoerceExpr, SER_TABLE_COERCE_EXPR); - -bool TableCoerceExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_TABLE_COERCE_EXPR, UnaryExpr); - return true; - } - -bool TableCoerceExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - return true; - } - VectorCoerceExpr::VectorCoerceExpr(Expr* op, VectorType* v) : UnaryExpr(EXPR_VECTOR_COERCE, op) { @@ -4520,20 +3854,6 @@ Val* VectorCoerceExpr::Fold(Val* v) const return new VectorVal(Type()->Ref()->AsVectorType()); } -IMPLEMENT_SERIAL(VectorCoerceExpr, SER_VECTOR_COERCE_EXPR); - -bool VectorCoerceExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_VECTOR_COERCE_EXPR, UnaryExpr); - return true; - } - -bool VectorCoerceExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - return true; - } - FlattenExpr::FlattenExpr(Expr* arg_op) : UnaryExpr(EXPR_FLATTEN, arg_op) { @@ -4582,20 +3902,6 @@ Val* FlattenExpr::Fold(Val* v) const return l; } -IMPLEMENT_SERIAL(FlattenExpr, SER_FLATTEN_EXPR); - -bool FlattenExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_FLATTEN_EXPR, UnaryExpr); - return SERIALIZE(num_fields); - } - -bool FlattenExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - return UNSERIALIZE(&num_fields); - } - ScheduleTimer::ScheduleTimer(EventHandlerPtr arg_event, val_list* arg_args, double t, TimerMgr* arg_tmgr) : Timer(t, TIMER_SCHEDULE), @@ -4707,26 +4013,6 @@ void ScheduleExpr::ExprDescribe(ODesc* d) const event->Describe(d); } -IMPLEMENT_SERIAL(ScheduleExpr, SER_SCHEDULE_EXPR); - -bool ScheduleExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_SCHEDULE_EXPR, Expr); - return when->Serialize(info) && event->Serialize(info); - } - -bool ScheduleExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Expr); - - when = Expr::Unserialize(info); - if ( ! when ) - return false; - - event = (EventExpr*) Expr::Unserialize(info, EXPR_EVENT); - return event != 0; - } - InExpr::InExpr(Expr* arg_op1, Expr* arg_op2) : BinaryExpr(EXPR_IN, arg_op1, arg_op2) { @@ -4843,20 +4129,6 @@ Val* InExpr::Fold(Val* v1, Val* v2) const return val_mgr->GetBool(0); } -IMPLEMENT_SERIAL(InExpr, SER_IN_EXPR); - -bool InExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_IN_EXPR, BinaryExpr); - return true; - } - -bool InExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BinaryExpr); - return true; - } - CallExpr::CallExpr(Expr* arg_func, ListExpr* arg_args, bool in_hook) : Expr(EXPR_CALL) { @@ -5054,26 +4326,6 @@ void CallExpr::ExprDescribe(ODesc* d) const args->Describe(d); } -IMPLEMENT_SERIAL(CallExpr, SER_CALL_EXPR); - -bool CallExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_CALL_EXPR, Expr); - return func->Serialize(info) && args->Serialize(info); - } - -bool CallExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Expr); - - func = Expr::Unserialize(info); - if ( ! func ) - return false; - - args = (ListExpr*) Expr::Unserialize(info, EXPR_LIST); - return args != 0; - } - EventExpr::EventExpr(const char* arg_name, ListExpr* arg_args) : Expr(EXPR_EVENT) { @@ -5159,35 +4411,6 @@ void EventExpr::ExprDescribe(ODesc* d) const args->Describe(d); } -IMPLEMENT_SERIAL(EventExpr, SER_EVENT_EXPR); - -bool EventExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_EVENT_EXPR, Expr); - - if ( ! handler->Serialize(info) ) - return false; - - return SERIALIZE(name) && args->Serialize(info); - } - -bool EventExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Expr); - - EventHandler* h = EventHandler::Unserialize(info); - if ( ! h ) - return false; - - handler = h; - - if ( ! UNSERIALIZE(&name) ) - return false; - - args = (ListExpr*) Expr::Unserialize(info, EXPR_LIST); - return args; - } - ListExpr::ListExpr() : Expr(EXPR_LIST) { SetType(new TypeList()); @@ -5558,42 +4781,6 @@ TraversalCode ListExpr::Traverse(TraversalCallback* cb) const HANDLE_TC_EXPR_POST(tc); } -IMPLEMENT_SERIAL(ListExpr, SER_LIST_EXPR); - -bool ListExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_LIST_EXPR, Expr); - - if ( ! SERIALIZE(exprs.length()) ) - return false; - - loop_over_list(exprs, i) - if ( ! exprs[i]->Serialize(info) ) - return false; - - return true; - } - -bool ListExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Expr); - - int len; - if ( ! UNSERIALIZE(&len) ) - return false; - - while ( len-- ) - { - Expr* e = Expr::Unserialize(info); - if ( ! e ) - return false; - - exprs.append(e); - } - - return true; - } - RecordAssignExpr::RecordAssignExpr(Expr* record, Expr* init_list, int is_init) { const expr_list& inits = init_list->AsListExpr()->Exprs(); @@ -5655,20 +4842,6 @@ RecordAssignExpr::RecordAssignExpr(Expr* record, Expr* init_list, int is_init) } } -IMPLEMENT_SERIAL(RecordAssignExpr, SER_RECORD_ASSIGN_EXPR); - -bool RecordAssignExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_RECORD_ASSIGN_EXPR, ListExpr); - return true; - } - -bool RecordAssignExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(ListExpr); - return true; - } - CastExpr::CastExpr(Expr* arg_op, BroType* t) : UnaryExpr(EXPR_CAST, arg_op) { auto stype = Op()->Type(); @@ -5721,20 +4894,6 @@ void CastExpr::ExprDescribe(ODesc* d) const Type()->Describe(d); } -IMPLEMENT_SERIAL(CastExpr, SER_CAST_EXPR); - -bool CastExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_CAST_EXPR, UnaryExpr); - return true; - } - -bool CastExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - return true; - } - IsExpr::IsExpr(Expr* arg_op, BroType* arg_t) : UnaryExpr(EXPR_IS, arg_op) { t = arg_t; @@ -5766,20 +4925,6 @@ void IsExpr::ExprDescribe(ODesc* d) const t->Describe(d); } -IMPLEMENT_SERIAL(IsExpr, SER_IS_EXPR_ /* sic */); - -bool IsExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_IS_EXPR_, UnaryExpr); - return true; - } - -bool IsExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(UnaryExpr); - return true; - } - Expr* get_assign_expr(Expr* op1, Expr* op2, int is_init) { if ( op1->Type()->Tag() == TYPE_RECORD && diff --git a/src/Expr.h b/src/Expr.h index 111cc20e60..533e107e16 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -203,9 +203,6 @@ public: void Describe(ODesc* d) const override; - bool Serialize(SerialInfo* info) const; - static Expr* Unserialize(UnserialInfo* info, BroExprTag want = EXPR_ANY); - virtual TraversalCode Traverse(TraversalCallback* cb) const = 0; protected: @@ -228,8 +225,6 @@ protected: void RuntimeErrorWithCallStack(const std::string& msg) const; - DECLARE_ABSTRACT_SERIAL(Expr); - BroExprTag tag; BroType* type; @@ -256,8 +251,6 @@ protected: void ExprDescribe(ODesc* d) const override; - DECLARE_SERIAL(NameExpr); - ID* id; bool in_const_init; }; @@ -278,8 +271,6 @@ protected: ConstExpr() { val = 0; } void ExprDescribe(ODesc* d) const override; - DECLARE_SERIAL(ConstExpr); - Val* val; }; @@ -308,8 +299,6 @@ protected: // Returns the expression folded using the given constant. virtual Val* Fold(Val* v) const; - DECLARE_SERIAL(UnaryExpr); - Expr* op; }; @@ -371,8 +360,6 @@ protected: void ExprDescribe(ODesc* d) const override; - DECLARE_SERIAL(BinaryExpr); - Expr* op1; Expr* op2; }; @@ -387,8 +374,6 @@ protected: CloneExpr() { } Val* Fold(Val* v) const override; - - DECLARE_SERIAL(CloneExpr); }; class IncrExpr : public UnaryExpr { @@ -402,8 +387,6 @@ public: protected: friend class Expr; IncrExpr() { } - - DECLARE_SERIAL(IncrExpr); }; class ComplementExpr : public UnaryExpr { @@ -415,8 +398,6 @@ protected: ComplementExpr() { } Val* Fold(Val* v) const override; - - DECLARE_SERIAL(ComplementExpr); }; class NotExpr : public UnaryExpr { @@ -428,8 +409,6 @@ protected: NotExpr() { } Val* Fold(Val* v) const override; - - DECLARE_SERIAL(NotExpr); }; class PosExpr : public UnaryExpr { @@ -441,8 +420,6 @@ protected: PosExpr() { } Val* Fold(Val* v) const override; - - DECLARE_SERIAL(PosExpr); }; class NegExpr : public UnaryExpr { @@ -454,8 +431,6 @@ protected: NegExpr() { } Val* Fold(Val* v) const override; - - DECLARE_SERIAL(NegExpr); }; class SizeExpr : public UnaryExpr { @@ -468,7 +443,6 @@ protected: SizeExpr() { } Val* Fold(Val* v) const override; - DECLARE_SERIAL(SizeExpr); }; class AddExpr : public BinaryExpr { @@ -479,9 +453,6 @@ public: protected: friend class Expr; AddExpr() { } - - DECLARE_SERIAL(AddExpr); - }; class AddToExpr : public BinaryExpr { @@ -492,8 +463,6 @@ public: protected: friend class Expr; AddToExpr() { } - - DECLARE_SERIAL(AddToExpr); }; class RemoveFromExpr : public BinaryExpr { @@ -504,8 +473,6 @@ public: protected: friend class Expr; RemoveFromExpr() { } - - DECLARE_SERIAL(RemoveFromExpr); }; class SubExpr : public BinaryExpr { @@ -515,9 +482,6 @@ public: protected: friend class Expr; SubExpr() { } - - DECLARE_SERIAL(SubExpr); - }; class TimesExpr : public BinaryExpr { @@ -528,9 +492,6 @@ public: protected: friend class Expr; TimesExpr() { } - - DECLARE_SERIAL(TimesExpr); - }; class DivideExpr : public BinaryExpr { @@ -542,9 +503,6 @@ protected: DivideExpr() { } Val* AddrFold(Val* v1, Val* v2) const override; - - DECLARE_SERIAL(DivideExpr); - }; class ModExpr : public BinaryExpr { @@ -554,8 +512,6 @@ public: protected: friend class Expr; ModExpr() { } - - DECLARE_SERIAL(ModExpr); }; class BoolExpr : public BinaryExpr { @@ -568,8 +524,6 @@ public: protected: friend class Expr; BoolExpr() { } - - DECLARE_SERIAL(BoolExpr); }; class BitExpr : public BinaryExpr { @@ -579,8 +533,6 @@ public: protected: friend class Expr; BitExpr() { } - - DECLARE_SERIAL(BitExpr); }; class EqExpr : public BinaryExpr { @@ -593,8 +545,6 @@ protected: EqExpr() { } Val* Fold(Val* v1, Val* v2) const override; - - DECLARE_SERIAL(EqExpr); }; class RelExpr : public BinaryExpr { @@ -605,8 +555,6 @@ public: protected: friend class Expr; RelExpr() { } - - DECLARE_SERIAL(RelExpr); }; class CondExpr : public Expr { @@ -629,8 +577,6 @@ protected: void ExprDescribe(ODesc* d) const override; - DECLARE_SERIAL(CondExpr); - Expr* op1; Expr* op2; Expr* op3; @@ -646,8 +592,6 @@ public: protected: friend class Expr; RefExpr() { } - - DECLARE_SERIAL(RefExpr); }; class AssignExpr : public BinaryExpr { @@ -671,8 +615,6 @@ protected: bool TypeCheck(attr_list* attrs = 0); bool TypeCheckArithmetics(TypeTag bt1, TypeTag bt2); - DECLARE_SERIAL(AssignExpr); - int is_init; Val* val; // optional }; @@ -685,8 +627,6 @@ public: protected: friend class Expr; IndexSliceAssignExpr() {} - - DECLARE_SERIAL(IndexSliceAssignExpr); }; class IndexExpr : public BinaryExpr { @@ -718,8 +658,6 @@ protected: void ExprDescribe(ODesc* d) const override; - DECLARE_SERIAL(IndexExpr); - bool is_slice; }; @@ -746,8 +684,6 @@ protected: void ExprDescribe(ODesc* d) const override; - DECLARE_SERIAL(FieldExpr); - const char* field_name; const TypeDecl* td; int field; // -1 = attributes @@ -770,8 +706,6 @@ protected: void ExprDescribe(ODesc* d) const override; - DECLARE_SERIAL(HasFieldExpr); - const char* field_name; int field; }; @@ -789,8 +723,6 @@ protected: Val* Fold(Val* v) const override; void ExprDescribe(ODesc* d) const override; - - DECLARE_SERIAL(RecordConstructorExpr); }; class TableConstructorExpr : public UnaryExpr { @@ -811,8 +743,6 @@ protected: void ExprDescribe(ODesc* d) const override; - DECLARE_SERIAL(TableConstructorExpr); - Attributes* attrs; }; @@ -834,8 +764,6 @@ protected: void ExprDescribe(ODesc* d) const override; - DECLARE_SERIAL(SetConstructorExpr); - Attributes* attrs; }; @@ -852,8 +780,6 @@ protected: Val* InitVal(const BroType* t, Val* aggr) const override; void ExprDescribe(ODesc* d) const override; - - DECLARE_SERIAL(VectorConstructorExpr); }; class FieldAssignExpr : public UnaryExpr { @@ -871,8 +797,6 @@ protected: void ExprDescribe(ODesc* d) const override; - DECLARE_SERIAL(FieldAssignExpr); - string field_name; }; @@ -886,8 +810,6 @@ protected: Val* FoldSingleVal(Val* v, InternalTypeTag t) const; Val* Fold(Val* v) const override; - - DECLARE_SERIAL(ArithCoerceExpr); }; class RecordCoerceExpr : public UnaryExpr { @@ -902,8 +824,6 @@ protected: Val* InitVal(const BroType* t, Val* aggr) const override; Val* Fold(Val* v) const override; - DECLARE_SERIAL(RecordCoerceExpr); - // For each super-record slot, gives subrecord slot with which to // fill it. int* map; @@ -920,8 +840,6 @@ protected: TableCoerceExpr() { } Val* Fold(Val* v) const override; - - DECLARE_SERIAL(TableCoerceExpr); }; class VectorCoerceExpr : public UnaryExpr { @@ -934,8 +852,6 @@ protected: VectorCoerceExpr() { } Val* Fold(Val* v) const override; - - DECLARE_SERIAL(VectorCoerceExpr); }; // An internal operator for flattening array indices that are records @@ -950,8 +866,6 @@ protected: Val* Fold(Val* v) const override; - DECLARE_SERIAL(FlattenExpr); - int num_fields; }; @@ -991,8 +905,6 @@ protected: void ExprDescribe(ODesc* d) const override; - DECLARE_SERIAL(ScheduleExpr); - Expr* when; EventExpr* event; }; @@ -1007,8 +919,6 @@ protected: Val* Fold(Val* v1, Val* v2) const override; - DECLARE_SERIAL(InExpr); - }; class CallExpr : public Expr { @@ -1031,8 +941,6 @@ protected: void ExprDescribe(ODesc* d) const override; - DECLARE_SERIAL(CallExpr); - Expr* func; ListExpr* args; }; @@ -1056,8 +964,6 @@ protected: void ExprDescribe(ODesc* d) const override; - DECLARE_SERIAL(EventExpr); - string name; EventHandlerPtr handler; ListExpr* args; @@ -1094,8 +1000,6 @@ protected: void ExprDescribe(ODesc* d) const override; - DECLARE_SERIAL(ListExpr); - expr_list exprs; }; @@ -1109,8 +1013,6 @@ public: protected: friend class Expr; RecordAssignExpr() { } - - DECLARE_SERIAL(RecordAssignExpr); }; class CastExpr : public UnaryExpr { @@ -1123,8 +1025,6 @@ protected: Val* Eval(Frame* f) const override; void ExprDescribe(ODesc* d) const override; - - DECLARE_SERIAL(CastExpr); }; class IsExpr : public UnaryExpr { @@ -1138,7 +1038,6 @@ protected: Val* Fold(Val* v) const override; void ExprDescribe(ODesc* d) const override; - DECLARE_SERIAL(IsExpr); private: BroType* t; diff --git a/src/File.cc b/src/File.cc index 861ae4dab0..8ac229b9fc 100644 --- a/src/File.cc +++ b/src/File.cc @@ -25,7 +25,6 @@ #include "Expr.h" #include "NetVar.h" #include "Net.h" -#include "Serializer.h" #include "Event.h" #include "Reporter.h" @@ -518,11 +517,6 @@ double BroFile::Size() return s.st_size; } -bool BroFile::Serialize(SerialInfo* info) const - { - return SerialObj::Serialize(info); - } - BroFile* BroFile::GetFile(const char* name) { for ( BroFile* f = head; f; f = f->next ) @@ -534,135 +528,3 @@ BroFile* BroFile::GetFile(const char* name) return new BroFile(name, "w", 0); } -BroFile* BroFile::Unserialize(UnserialInfo* info) - { - BroFile* file = (BroFile*) SerialObj::Unserialize(info, SER_BRO_FILE); - - if ( ! file ) - return 0; - - if ( file->is_open ) - return file; - - // If there is already an object for this file, return it. - if ( file->name ) - { - for ( BroFile* f = head; f; f = f->next ) - { - if ( f->name && streq(file->name, f->name) ) - { - Unref(file); - Ref(f); - return f; - } - } - } - - // Otherwise, open, but don't clobber. - if ( ! file->Open(0, "a") ) - { - info->s->Error(fmt("cannot open %s: %s", - file->name, strerror(errno))); - return 0; - } - - // Here comes a hack. This method will return a pointer to a newly - // instantiated file object. As soon as this pointer is Unref'ed, the - // file will be closed. That means that when we unserialize the same - // file next time, we will re-open it and thereby delete the first one, - // i.e., we will be keeping to delete what we've written just before. - // - // To avoid this loop, we do an extra Ref here, i.e., this file will - // *never* be closed anymore (as long the file cache does not overflow). - Ref(file); - - file->SetBuf(file->buffered); - - return file; - } - -IMPLEMENT_SERIAL(BroFile, SER_BRO_FILE); - -bool BroFile::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_BRO_FILE, BroObj); - - const char* s = name; - - if ( ! okay_to_manage ) - { - // We can handle stdin/stdout/stderr but no others. - if ( f == stdin ) - s = "/dev/stdin"; - else if ( f == stdout ) - s = "/dev/stdout"; - else if ( f == stderr ) - s = "/dev/stderr"; - else - { - // We don't manage the file, and therefore don't - // really know how to pass it on to the other side. - // However, in order to not abort communication - // when this happens, we still send the name if we - // have one; or if we don't, we create a special - // "dont-have-a-file" file to be created on the - // receiver side. - if ( ! s ) - s = "unmanaged-bro-output-file.log"; - } - } - - if ( ! (SERIALIZE(s) && SERIALIZE(buffered)) ) - return false; - - SERIALIZE_OPTIONAL_STR(access); - - if ( ! t->Serialize(info) ) - return false; - - SERIALIZE_OPTIONAL(attrs); - return true; - } - -bool BroFile::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroObj); - - if ( ! (UNSERIALIZE_STR(&name, 0) && UNSERIALIZE(&buffered)) ) - return false; - - UNSERIALIZE_OPTIONAL_STR(access); - - t = BroType::Unserialize(info); - if ( ! t ) - return false; - - UNSERIALIZE_OPTIONAL(attrs, Attributes::Unserialize(info)); - - // Parse attributes. - SetAttrs(attrs); - // SetAttrs() has ref'ed attrs again. - Unref(attrs); - - // Bind stdin/stdout/stderr. - FILE* file = 0; - is_open = false; - f = 0; - - if ( streq(name, "/dev/stdin") ) - file = stdin; - else if ( streq(name, "/dev/stdout") ) - file = stdout; - else if ( streq(name, "/dev/stderr") ) - file = stderr; - - if ( file ) - { - delete [] name; - name = 0; - f = file; - is_open = true; - } - - return true; - } diff --git a/src/File.h b/src/File.h index b0ad571435..48689b4617 100644 --- a/src/File.h +++ b/src/File.h @@ -66,9 +66,6 @@ public: void EnableRawOutput() { raw_output = true; } bool IsRawOutput() const { return raw_output; } - bool Serialize(SerialInfo* info) const; - static BroFile* Unserialize(UnserialInfo* info); - protected: BroFile() { Init(); } void Init(); @@ -100,8 +97,6 @@ protected: // Raises a file_opened event. void RaiseOpenEvent(); - DECLARE_SERIAL(BroFile); - FILE* f; BroType* t; char* name; diff --git a/src/Func.cc b/src/Func.cc index dc96070980..ccd9793291 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -41,7 +41,6 @@ #include "analyzer/protocol/login/Login.h" #include "Sessions.h" #include "RE.h" -#include "Serializer.h" #include "Event.h" #include "Traverse.h" #include "Reporter.h" @@ -127,110 +126,6 @@ void Func::AddBody(Stmt* /* new_body */, id_list* /* new_inits */, Internal("Func::AddBody called"); } -bool Func::Serialize(SerialInfo* info) const - { - return SerialObj::Serialize(info); - } - -Func* Func::Unserialize(UnserialInfo* info) - { - Func* f = (Func*) SerialObj::Unserialize(info, SER_FUNC); - - // For builtins, we return a reference to the (hopefully) already - // existing function. - if ( f && f->kind == BUILTIN_FUNC ) - { - const char* name = ((BuiltinFunc*) f)->Name(); - ID* id = global_scope()->Lookup(name); - if ( ! id ) - { - info->s->Error(fmt("can't find built-in %s", name)); - return 0; - } - - if ( ! (id->HasVal() && id->ID_Val()->Type()->Tag() == TYPE_FUNC) ) - { - info->s->Error(fmt("ID %s is not a built-in", name)); - return 0; - } - - Unref(f); - f = id->ID_Val()->AsFunc(); - Ref(f); - } - - return f; - } - -bool Func::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_FUNC, BroObj); - - if ( ! SERIALIZE(int(bodies.size())) ) - return false; - - for ( unsigned int i = 0; i < bodies.size(); ++i ) - { - if ( ! bodies[i].stmts->Serialize(info) ) - return false; - if ( ! SERIALIZE(bodies[i].priority) ) - return false; - } - - if ( ! SERIALIZE(char(kind) ) ) - return false; - - if ( ! type->Serialize(info) ) - return false; - - if ( ! SERIALIZE(Name()) ) - return false; - - // We don't serialize scope as only global functions are considered here - // anyway. - return true; - } - -bool Func::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroObj); - - int len; - if ( ! UNSERIALIZE(&len) ) - return false; - - while ( len-- ) - { - Body b; - b.stmts = Stmt::Unserialize(info); - if ( ! b.stmts ) - return false; - - if ( ! UNSERIALIZE(&b.priority) ) - return false; - - bodies.push_back(b); - } - - char c; - if ( ! UNSERIALIZE(&c) ) - return false; - - kind = (Kind) c; - - type = BroType::Unserialize(info); - if ( ! type ) - return false; - - const char* n; - if ( ! UNSERIALIZE_STR(&n, 0) ) - return false; - - name = n; - delete [] n; - - return true; - } void Func::DescribeDebug(ODesc* d, const val_list* args) const { @@ -584,21 +479,6 @@ Stmt* BroFunc::AddInits(Stmt* body, id_list* inits) return stmt_series; } -IMPLEMENT_SERIAL(BroFunc, SER_BRO_FUNC); - -bool BroFunc::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_BRO_FUNC, Func); - return SERIALIZE(frame_size); - } - -bool BroFunc::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Func); - - return UNSERIALIZE(&frame_size); - } - BuiltinFunc::BuiltinFunc(built_in_func arg_func, const char* arg_name, int arg_is_pure) : Func(BUILTIN_FUNC) @@ -681,20 +561,6 @@ void BuiltinFunc::Describe(ODesc* d) const d->AddCount(is_pure); } -IMPLEMENT_SERIAL(BuiltinFunc, SER_BUILTIN_FUNC); - -bool BuiltinFunc::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_BUILTIN_FUNC, Func); - return true; - } - -bool BuiltinFunc::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Func); - return true; - } - void builtin_error(const char* msg, BroObj* arg) { auto emit = [=](const CallExpr* ce) diff --git a/src/Func.h b/src/Func.h index 48e0c2e8b8..765d1ec499 100644 --- a/src/Func.h +++ b/src/Func.h @@ -59,10 +59,6 @@ public: void Describe(ODesc* d) const override = 0; virtual void DescribeDebug(ODesc* d, const val_list* args) const; - // This (un-)serializes only a single body (as given in SerialInfo). - bool Serialize(SerialInfo* info) const; - static Func* Unserialize(UnserialInfo* info); - virtual TraversalCode Traverse(TraversalCallback* cb) const; uint32 GetUniqueFuncID() const { return unique_id; } @@ -75,8 +71,6 @@ protected: // Helper function for handling result of plugin hook. std::pair HandlePluginResult(std::pair plugin_result, val_list* args, function_flavor flavor) const; - DECLARE_ABSTRACT_SERIAL(Func); - vector bodies; Scope* scope; Kind kind; @@ -106,8 +100,6 @@ protected: BroFunc() : Func(BRO_FUNC) {} Stmt* AddInits(Stmt* body, id_list* inits); - DECLARE_SERIAL(BroFunc); - int frame_size; }; @@ -127,8 +119,6 @@ public: protected: BuiltinFunc() { func = 0; is_pure = 0; } - DECLARE_SERIAL(BuiltinFunc); - built_in_func func; int is_pure; }; diff --git a/src/ID.cc b/src/ID.cc index 89a320a5e0..12677bec75 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -9,7 +9,6 @@ #include "Func.h" #include "Scope.h" #include "File.h" -#include "Serializer.h" #include "Scope.h" #include "Traverse.h" #include "zeekygen/Manager.h" @@ -283,11 +282,6 @@ void ID::EvalFunc(Expr* ef, Expr* ev) Unref(ce); } -bool ID::Serialize(SerialInfo* info) const - { - return (ID*) SerialObj::Serialize(info); - } - #if 0 void ID::CopyFrom(const ID* id) { @@ -320,205 +314,6 @@ void ID::CopyFrom(const ID* id) #endif #endif -ID* ID::Unserialize(UnserialInfo* info) - { - ID* id = (ID*) SerialObj::Unserialize(info, SER_ID); - if ( ! id ) - return 0; - - if ( ! id->IsGlobal() ) - return id; - - // Globals. - ID* current = global_scope()->Lookup(id->name); - - if ( ! current ) - { - if ( ! info->install_globals ) - { - info->s->Error("undefined"); - Unref(id); - return 0; - } - - Ref(id); - global_scope()->Insert(id->Name(), id); -#ifdef USE_PERFTOOLS_DEBUG - heap_checker->IgnoreObject(id); -#endif - } - - else - { - switch ( info->id_policy ) { - - case UnserialInfo::Keep: - Unref(id); - Ref(current); - id = current; - break; - - case UnserialInfo::Replace: - Unref(current); - Ref(id); - global_scope()->Insert(id->Name(), id); - break; - - case UnserialInfo::CopyNewToCurrent: - if ( ! same_type(current->type, id->type) ) - { - info->s->Error("type mismatch"); - Unref(id); - return 0; - } - - if ( ! current->weak_ref ) - Unref(current->val); - - current->val = id->val; - current->weak_ref = id->weak_ref; - if ( current->val && ! current->weak_ref ) - Ref(current->val); - -#ifdef DEBUG - current->UpdateValID(); -#endif - - Unref(id); - Ref(current); - id = current; - - break; - - case UnserialInfo::CopyCurrentToNew: - if ( ! same_type(current->type, id->type) ) - { - info->s->Error("type mismatch"); - return 0; - } - if ( ! id->weak_ref ) - Unref(id->val); - id->val = current->val; - id->weak_ref = current->weak_ref; - if ( id->val && ! id->weak_ref ) - Ref(id->val); - -#ifdef DEBUG - id->UpdateValID(); -#endif - - Unref(current); - Ref(id); - global_scope()->Insert(id->Name(), id); - break; - - case UnserialInfo::InstantiateNew: - // Do nothing. - break; - - default: - reporter->InternalError("unknown type for UnserialInfo::id_policy"); - } - } - - return id; - - } - -IMPLEMENT_SERIAL(ID, SER_ID); - -bool ID::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE_WITH_SUSPEND(SER_ID, BroObj); - - if ( info->cont.NewInstance() ) - { - DisableSuspend suspend(info); - - info->s->WriteOpenTag("ID"); - - if ( ! (SERIALIZE(name) && - SERIALIZE(char(scope)) && - SERIALIZE(is_export) && - SERIALIZE(is_const) && - SERIALIZE(is_enum_const) && - SERIALIZE(is_type) && - SERIALIZE(offset) && - SERIALIZE(infer_return_type) && - SERIALIZE(weak_ref) && - type->Serialize(info)) ) - return false; - - SERIALIZE_OPTIONAL(attrs); - } - - SERIALIZE_OPTIONAL(val); - - return true; - } - -bool ID::DoUnserialize(UnserialInfo* info) - { - bool installed_tmp = false; - - DO_UNSERIALIZE(BroObj); - - char id_scope; - - if ( ! (UNSERIALIZE_STR(&name, 0) && - UNSERIALIZE(&id_scope) && - UNSERIALIZE(&is_export) && - UNSERIALIZE(&is_const) && - UNSERIALIZE(&is_enum_const) && - UNSERIALIZE(&is_type) && - UNSERIALIZE(&offset) && - UNSERIALIZE(&infer_return_type) && - UNSERIALIZE(&weak_ref) - ) ) - return false; - - scope = IDScope(id_scope); - - info->s->SetErrorDescr(fmt("unserializing ID %s", name)); - - type = BroType::Unserialize(info); - if ( ! type ) - return false; - - UNSERIALIZE_OPTIONAL(attrs, Attributes::Unserialize(info)); - - // If it's a global function not currently known, - // we temporarily install it in global scope. - // This is necessary for recursive functions. - if ( IsGlobal() && Type()->Tag() == TYPE_FUNC ) - { - ID* current = global_scope()->Lookup(name); - if ( ! current ) - { - installed_tmp = true; - global_scope()->Insert(Name(), this); - } - } - - UNSERIALIZE_OPTIONAL(val, Val::Unserialize(info)); -#ifdef DEBUG - UpdateValID(); -#endif - - if ( weak_ref ) - { - // At this point at least the serialization cache will hold a - // reference so this will not delete the val. - assert(val->RefCnt() > 1); - Unref(val); - } - - if ( installed_tmp && ! global_scope()->Remove(name) ) - reporter->InternalWarning("missing tmp ID in %s unserialization", name); - - return true; - } - TraversalCode ID::Traverse(TraversalCallback* cb) const { TraversalCode tc = cb->PreID(this); diff --git a/src/ID.h b/src/ID.h index 18754584df..bb9e11ca06 100644 --- a/src/ID.h +++ b/src/ID.h @@ -10,7 +10,6 @@ #include class Val; -class SerialInfo; class Func; typedef enum { INIT_NONE, INIT_FULL, INIT_EXTRA, INIT_REMOVE, } init_class; @@ -98,9 +97,6 @@ public: void DescribeReST(ODesc* d, bool roles_only = false) const; void DescribeReSTShort(ODesc* d) const; - bool Serialize(SerialInfo* info) const; - static ID* Unserialize(UnserialInfo* info); - bool DoInferReturnType() const { return infer_return_type; } void SetInferReturnType(bool infer) @@ -124,8 +120,6 @@ protected: void UpdateValID(); #endif - DECLARE_SERIAL(ID); - const char* name; IDScope scope; bool is_export; diff --git a/src/Net.cc b/src/Net.cc index 96aaeddc69..0b1a5346e6 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -27,7 +27,6 @@ #include "Reporter.h" #include "Net.h" #include "Anon.h" -#include "Serializer.h" #include "PacketDumper.h" #include "iosource/Manager.h" #include "iosource/PktSrc.h" diff --git a/src/Obj.cc b/src/Obj.cc index fdf2b7ef99..c2e47f50b8 100644 --- a/src/Obj.cc +++ b/src/Obj.cc @@ -5,7 +5,6 @@ #include #include "Obj.h" -#include "Serializer.h" #include "Func.h" #include "File.h" #include "plugin/Manager.h" @@ -14,47 +13,6 @@ Location no_location("", 0, 0, 0, 0); Location start_location("", 0, 0, 0, 0); Location end_location("", 0, 0, 0, 0); -bool Location::Serialize(SerialInfo* info) const - { - return SerialObj::Serialize(info); - } - -Location* Location::Unserialize(UnserialInfo* info) - { - return (Location*) SerialObj::Unserialize(info, SER_LOCATION); - } - -IMPLEMENT_SERIAL(Location, SER_LOCATION); - -bool Location::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_LOCATION, SerialObj); - info->s->WriteOpenTag("Location"); - - if ( ! (SERIALIZE(filename) && - SERIALIZE(first_line) && - SERIALIZE(last_line) && - SERIALIZE(first_column) && - SERIALIZE(last_column)) ) - return false; - - info->s->WriteCloseTag("Location"); - return true; - } - -bool Location::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(SerialObj); - - delete_data = true; - - return UNSERIALIZE_STR(&filename, 0) - && UNSERIALIZE(&first_line) - && UNSERIALIZE(&last_line) - && UNSERIALIZE(&first_column) - && UNSERIALIZE(&last_column); - } - void Location::Describe(ODesc* d) const { if ( filename ) @@ -230,29 +188,6 @@ void BroObj::PinPoint(ODesc* d, const BroObj* obj2, int pinpoint_only) const d->Add(")"); } -bool BroObj::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_BRO_OBJ, SerialObj); - - info->s->WriteOpenTag("Object"); - - Location* loc = info->include_locations ? location : 0; - SERIALIZE_OPTIONAL(loc); - info->s->WriteCloseTag("Object"); - - return true; - } - -bool BroObj::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(SerialObj); - - delete location; - - UNSERIALIZE_OPTIONAL(location, Location::Unserialize(info)); - return true; - } - void print(const BroObj* obj) { static BroFile fstderr(stderr); diff --git a/src/Obj.h b/src/Obj.h index 21730ff367..78f8d018d3 100644 --- a/src/Obj.h +++ b/src/Obj.h @@ -7,12 +7,8 @@ #include "input.h" #include "Desc.h" -#include "SerialObj.h" -class Serializer; -class SerialInfo; - -class Location : SerialObj { +class Location { public: Location(const char* fname, int line_f, int line_l, int col_f, int col_l) { @@ -36,7 +32,7 @@ public: text = 0; } - ~Location() override + virtual ~Location() { if ( delete_data ) delete [] filename; @@ -44,9 +40,6 @@ public: void Describe(ODesc* d) const; - bool Serialize(SerialInfo* info) const; - static Location* Unserialize(UnserialInfo* info); - bool operator==(const Location& l) const; bool operator!=(const Location& l) const { return ! (*this == l); } @@ -59,8 +52,6 @@ public: // Timestamp and text for compatibility with Bison's default yyltype. int timestamp; char* text; -protected: - DECLARE_SERIAL(Location); }; #define YYLTYPE yyltype @@ -86,7 +77,7 @@ inline void set_location(const Location start, const Location end) end_location = end; } -class BroObj : public SerialObj { +class BroObj { public: BroObj() { @@ -112,7 +103,7 @@ public: SetLocationInfo(&start_location, &end_location); } - ~BroObj() override; + virtual ~BroObj(); // Report user warnings/errors. If obj2 is given, then it's // included in the message, though if pinpoint_only is non-zero, @@ -168,10 +159,6 @@ public: bool in_ser_cache; protected: - friend class SerializationCache; - - DECLARE_ABSTRACT_SERIAL(BroObj); - Location* location; // all that matters in real estate private: diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index bf91c3f40d..4d7c66b431 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -3,10 +3,151 @@ #include "OpaqueVal.h" #include "NetVar.h" #include "Reporter.h" -#include "Serializer.h" #include "probabilistic/BloomFilter.h" #include "probabilistic/CardinalityCounter.h" +// Helper to retrieve a broker value out of a broker::vector at a specified +// index, and casted to the expected destination type. +template +inline bool get_vector_idx(const V& v, unsigned int i, D* dst) + { + if ( i >= v.size() ) + return false; + + auto x = caf::get_if(&v[i]); + if ( ! x ) + return false; + + *dst = static_cast(*x); + return true; + } + +OpaqueMgr* OpaqueMgr::mgr() + { + static OpaqueMgr mgr; + return &mgr; + } + +OpaqueVal::OpaqueVal(OpaqueType* t) : Val(t) + { + } + +OpaqueVal::~OpaqueVal() + { + } + +const std::string& OpaqueMgr::TypeID(const OpaqueVal* v) const + { + auto x = _types.find(v->OpaqueName()); + + if ( x == _types.end() ) + reporter->InternalError("OpaqueMgr::TypeID: opaque type %s not registered", + v->OpaqueName()); + + return x->first; + } + +OpaqueVal* OpaqueMgr::Instantiate(const std::string& id) const + { + auto x = _types.find(id); + return x != _types.end() ? (*x->second)() : nullptr; + } + +broker::expected OpaqueVal::Serialize() const + { + auto type = OpaqueMgr::mgr()->TypeID(this); + + auto d = DoSerialize(); + if ( ! d ) + return d.error(); + + return {broker::vector{std::move(type), std::move(*d)}}; + } + +OpaqueVal* OpaqueVal::Unserialize(const broker::data& data) + { + auto v = caf::get_if(&data); + + if ( ! (v && v->size() == 2) ) + return nullptr; + + auto type = caf::get_if(&(*v)[0]); + if ( ! type ) + return nullptr; + + auto val = OpaqueMgr::mgr()->Instantiate(*type); + if ( ! val ) + return nullptr; + + if ( ! val->DoUnserialize((*v)[1]) ) + { + Unref(val); + return nullptr; + } + + return val; + } + +broker::expected OpaqueVal::SerializeType(BroType* t) + { + if ( t->InternalType() == TYPE_INTERNAL_ERROR ) + return broker::ec::invalid_data; + + if ( t->InternalType() == TYPE_INTERNAL_OTHER ) + { + // Serialize by name. + assert(t->GetName().size()); + return {broker::vector{true, t->GetName()}}; + } + + // A base type. + return {broker::vector{false, static_cast(t->Tag())}}; + } + +BroType* OpaqueVal::UnserializeType(const broker::data& data) + { + auto v = caf::get_if(&data); + if ( ! (v && v->size() == 2) ) + return nullptr; + + auto by_name = caf::get_if(&(*v)[0]); + if ( ! by_name ) + return nullptr; + + if ( *by_name ) + { + auto name = caf::get_if(&(*v)[1]); + if ( ! name ) + return nullptr; + + ID* id = global_scope()->Lookup(name->c_str()); + if ( ! id ) + return nullptr; + + BroType* t = id->AsType(); + if ( ! t ) + return nullptr; + + return t->Ref(); + } + + auto tag = caf::get_if(&(*v)[1]); + if ( ! tag ) + return nullptr; + + return base_type(static_cast(*tag)); + } + +Val* OpaqueVal::DoClone(CloneState* state) + { + auto d = OpaqueVal::Serialize(); + if ( ! d ) + return nullptr; + + auto rval = OpaqueVal::Unserialize(std::move(*d)); + return state->NewClone(this, rval); + } + bool HashVal::IsValid() const { return valid; @@ -63,20 +204,6 @@ HashVal::HashVal(OpaqueType* t) : OpaqueVal(t) valid = false; } -IMPLEMENT_SERIAL(HashVal, SER_HASH_VAL); - -bool HashVal::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_HASH_VAL, OpaqueVal); - return SERIALIZE(valid); - } - -bool HashVal::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(OpaqueVal); - return UNSERIALIZE(&valid); - } - MD5Val::MD5Val() : HashVal(md5_type) { } @@ -160,64 +287,72 @@ StringVal* MD5Val::DoGet() return new StringVal(md5_digest_print(digest)); } -IMPLEMENT_SERIAL(MD5Val, SER_MD5_VAL); +IMPLEMENT_OPAQUE_VALUE(MD5Val) -bool MD5Val::DoSerialize(SerialInfo* info) const +broker::expected MD5Val::DoSerialize() const { - DO_SERIALIZE(SER_MD5_VAL, HashVal); - if ( ! IsValid() ) - return true; + return {broker::vector{false}}; MD5_CTX* md = (MD5_CTX*) EVP_MD_CTX_md_data(ctx); - if ( ! (SERIALIZE(md->A) && - SERIALIZE(md->B) && - SERIALIZE(md->C) && - SERIALIZE(md->D) && - SERIALIZE(md->Nl) && - SERIALIZE(md->Nh)) ) - return false; + broker::vector d = { + true, + static_cast(md->A), + static_cast(md->B), + static_cast(md->C), + static_cast(md->D), + static_cast(md->Nl), + static_cast(md->Nh), + static_cast(md->num) + }; for ( int i = 0; i < MD5_LBLOCK; ++i ) - { - if ( ! SERIALIZE(md->data[i]) ) - return false; - } + d.emplace_back(static_cast(md->data[i])); - if ( ! SERIALIZE(md->num) ) - return false; - - return true; + return {std::move(d)}; } -bool MD5Val::DoUnserialize(UnserialInfo* info) +bool MD5Val::DoUnserialize(const broker::data& data) { - DO_UNSERIALIZE(HashVal); + auto d = caf::get_if(&data); + if ( ! d ) + return false; - if ( ! IsValid() ) + auto valid = caf::get_if(&(*d)[0]); + if ( ! valid ) + return false; + + if ( ! *valid ) + { + assert(! IsValid()); // default set by ctor return true; + } - ctx = hash_init(Hash_MD5); + Init(); MD5_CTX* md = (MD5_CTX*) EVP_MD_CTX_md_data(ctx); - if ( ! (UNSERIALIZE(&md->A) && - UNSERIALIZE(&md->B) && - UNSERIALIZE(&md->C) && - UNSERIALIZE(&md->D) && - UNSERIALIZE(&md->Nl) && - UNSERIALIZE(&md->Nh)) ) + if ( ! get_vector_idx(*d, 1, &md->A) ) + return false; + if ( ! get_vector_idx(*d, 2, &md->B) ) + return false; + if ( ! get_vector_idx(*d, 3, &md->C) ) + return false; + if ( ! get_vector_idx(*d, 4, &md->D) ) + return false; + if ( ! get_vector_idx(*d, 5, &md->Nl) ) + return false; + if ( ! get_vector_idx(*d, 6, &md->Nh) ) + return false; + if ( ! get_vector_idx(*d, 7, &md->num) ) return false; for ( int i = 0; i < MD5_LBLOCK; ++i ) { - if ( ! UNSERIALIZE(&md->data[i]) ) + if ( ! get_vector_idx(*d, 8 + i, &md->data[i]) ) return false; } - if ( ! UNSERIALIZE(&md->num) ) - return false; - return true; } @@ -293,66 +428,75 @@ StringVal* SHA1Val::DoGet() return new StringVal(sha1_digest_print(digest)); } -IMPLEMENT_SERIAL(SHA1Val, SER_SHA1_VAL); +IMPLEMENT_OPAQUE_VALUE(SHA1Val) -bool SHA1Val::DoSerialize(SerialInfo* info) const +broker::expected SHA1Val::DoSerialize() const { - DO_SERIALIZE(SER_SHA1_VAL, HashVal); - if ( ! IsValid() ) - return true; + return {broker::vector{false}}; SHA_CTX* md = (SHA_CTX*) EVP_MD_CTX_md_data(ctx); - if ( ! (SERIALIZE(md->h0) && - SERIALIZE(md->h1) && - SERIALIZE(md->h2) && - SERIALIZE(md->h3) && - SERIALIZE(md->h4) && - SERIALIZE(md->Nl) && - SERIALIZE(md->Nh)) ) - return false; + broker::vector d = { + true, + static_cast(md->h0), + static_cast(md->h1), + static_cast(md->h2), + static_cast(md->h3), + static_cast(md->h4), + static_cast(md->Nl), + static_cast(md->Nh), + static_cast(md->num) + }; for ( int i = 0; i < SHA_LBLOCK; ++i ) - { - if ( ! SERIALIZE(md->data[i]) ) - return false; - } + d.emplace_back(static_cast(md->data[i])); - if ( ! SERIALIZE(md->num) ) - return false; - - return true; + return {std::move(d)}; } -bool SHA1Val::DoUnserialize(UnserialInfo* info) +bool SHA1Val::DoUnserialize(const broker::data& data) { - DO_UNSERIALIZE(HashVal); + auto d = caf::get_if(&data); + if ( ! d ) + return false; - if ( ! IsValid() ) + auto valid = caf::get_if(&(*d)[0]); + if ( ! valid ) + return false; + + if ( ! *valid ) + { + assert(! IsValid()); // default set by ctor return true; + } - ctx = hash_init(Hash_SHA1); + Init(); SHA_CTX* md = (SHA_CTX*) EVP_MD_CTX_md_data(ctx); - if ( ! (UNSERIALIZE(&md->h0) && - UNSERIALIZE(&md->h1) && - UNSERIALIZE(&md->h2) && - UNSERIALIZE(&md->h3) && - UNSERIALIZE(&md->h4) && - UNSERIALIZE(&md->Nl) && - UNSERIALIZE(&md->Nh)) ) + if ( ! get_vector_idx(*d, 1, &md->h0) ) + return false; + if ( ! get_vector_idx(*d, 2, &md->h1) ) + return false; + if ( ! get_vector_idx(*d, 3, &md->h2) ) + return false; + if ( ! get_vector_idx(*d, 4, &md->h3) ) + return false; + if ( ! get_vector_idx(*d, 5, &md->h4) ) + return false; + if ( ! get_vector_idx(*d, 6, &md->Nl) ) + return false; + if ( ! get_vector_idx(*d, 7, &md->Nh) ) + return false; + if ( ! get_vector_idx(*d, 8, &md->num) ) return false; for ( int i = 0; i < SHA_LBLOCK; ++i ) { - if ( ! UNSERIALIZE(&md->data[i]) ) + if ( ! get_vector_idx(*d, 9 + i, &md->data[i]) ) return false; } - if ( ! UNSERIALIZE(&md->num) ) - return false; - return true; } @@ -428,71 +572,72 @@ StringVal* SHA256Val::DoGet() return new StringVal(sha256_digest_print(digest)); } -IMPLEMENT_SERIAL(SHA256Val, SER_SHA256_VAL); +IMPLEMENT_OPAQUE_VALUE(SHA256Val) -bool SHA256Val::DoSerialize(SerialInfo* info) const +broker::expected SHA256Val::DoSerialize() const { - DO_SERIALIZE(SER_SHA256_VAL, HashVal); - if ( ! IsValid() ) - return true; + return {broker::vector{false}}; SHA256_CTX* md = (SHA256_CTX*) EVP_MD_CTX_md_data(ctx); - for ( int i = 0; i < 8; ++i ) - { - if ( ! SERIALIZE(md->h[i]) ) - return false; - } + broker::vector d = { + true, + static_cast(md->Nl), + static_cast(md->Nh), + static_cast(md->num), + static_cast(md->md_len) + }; - if ( ! (SERIALIZE(md->Nl) && - SERIALIZE(md->Nh)) ) - return false; + for ( int i = 0; i < 8; ++i ) + d.emplace_back(static_cast(md->h[i])); for ( int i = 0; i < SHA_LBLOCK; ++i ) - { - if ( ! SERIALIZE(md->data[i]) ) - return false; - } + d.emplace_back(static_cast(md->data[i])); - if ( ! (SERIALIZE(md->num) && - SERIALIZE(md->md_len)) ) - return false; - - return true; + return {std::move(d)}; } -bool SHA256Val::DoUnserialize(UnserialInfo* info) +bool SHA256Val::DoUnserialize(const broker::data& data) { - DO_UNSERIALIZE(HashVal); + auto d = caf::get_if(&data); + if ( ! d ) + return false; - if ( ! IsValid() ) + auto valid = caf::get_if(&(*d)[0]); + if ( ! valid ) + return false; + + if ( ! *valid ) + { + assert(! IsValid()); // default set by ctor return true; + } - ctx = hash_init(Hash_SHA256); + Init(); SHA256_CTX* md = (SHA256_CTX*) EVP_MD_CTX_md_data(ctx); + if ( ! get_vector_idx(*d, 1, &md->Nl) ) + return false; + if ( ! get_vector_idx(*d, 2, &md->Nh) ) + return false; + if ( ! get_vector_idx(*d, 3, &md->num) ) + return false; + if ( ! get_vector_idx(*d, 4, &md->md_len) ) + return false; + for ( int i = 0; i < 8; ++i ) { - if ( ! UNSERIALIZE(&md->h[i]) ) + if ( ! get_vector_idx(*d, 5 + i, &md->h[i]) ) return false; } - if ( ! (UNSERIALIZE(&md->Nl) && - UNSERIALIZE(&md->Nh)) ) - return false; - for ( int i = 0; i < SHA_LBLOCK; ++i ) { - if ( ! UNSERIALIZE(&md->data[i]) ) + if ( ! get_vector_idx(*d, 13 + i, &md->data[i]) ) return false; } - - if ( ! (UNSERIALIZE(&md->num) && - UNSERIALIZE(&md->md_len)) ) - return false; - return true; } @@ -500,26 +645,6 @@ EntropyVal::EntropyVal() : OpaqueVal(entropy_type) { } -Val* EntropyVal::DoClone(CloneState* state) - { - SerializationFormat* form = new BinarySerializationFormat(); - form->StartWrite(); - CloneSerializer ss(form); - SerialInfo sinfo(&ss); - sinfo.cache = false; - sinfo.include_locations = false; - if ( ! this->Serialize(&sinfo) ) - return nullptr; - char* data; - uint32 len = form->EndWrite(&data); - form->StartRead(data, len); - UnserialInfo uinfo(&ss); - uinfo.cache = false; - Val* clone = Unserialize(&uinfo, type); - free(data); - return state->NewClone(this, clone); - } - bool EntropyVal::Feed(const void* data, size_t size) { state.add(data, size); @@ -533,79 +658,86 @@ bool EntropyVal::Get(double *r_ent, double *r_chisq, double *r_mean, return true; } -IMPLEMENT_SERIAL(EntropyVal, SER_ENTROPY_VAL); +IMPLEMENT_OPAQUE_VALUE(EntropyVal) -bool EntropyVal::DoSerialize(SerialInfo* info) const +broker::expected EntropyVal::DoSerialize() const { - DO_SERIALIZE(SER_ENTROPY_VAL, OpaqueVal); + broker::vector d = + { + static_cast(state.totalc), + static_cast(state.mp), + static_cast(state.sccfirst), + static_cast(state.inmont), + static_cast(state.mcount), + static_cast(state.cexp), + static_cast(state.montex), + static_cast(state.montey), + static_cast(state.montepi), + static_cast(state.sccu0), + static_cast(state.scclast), + static_cast(state.scct1), + static_cast(state.scct2), + static_cast(state.scct3), + }; + + d.reserve(256 + 3 + RT_MONTEN + 11); for ( int i = 0; i < 256; ++i ) - { - if ( ! SERIALIZE(state.ccount[i]) ) - return false; - } + d.emplace_back(static_cast(state.ccount[i])); - if ( ! (SERIALIZE(state.totalc) && - SERIALIZE(state.mp) && - SERIALIZE(state.sccfirst)) ) - return false; + for ( int i = 0; i < RT_MONTEN; ++i ) + d.emplace_back(static_cast(state.monte[i])); - for ( int i = 0; i < RT_MONTEN; ++i ) - { - if ( ! SERIALIZE(state.monte[i]) ) - return false; - } - - if ( ! (SERIALIZE(state.inmont) && - SERIALIZE(state.mcount) && - SERIALIZE(state.cexp) && - SERIALIZE(state.montex) && - SERIALIZE(state.montey) && - SERIALIZE(state.montepi) && - SERIALIZE(state.sccu0) && - SERIALIZE(state.scclast) && - SERIALIZE(state.scct1) && - SERIALIZE(state.scct2) && - SERIALIZE(state.scct3)) ) - return false; - - return true; + return {std::move(d)}; } -bool EntropyVal::DoUnserialize(UnserialInfo* info) +bool EntropyVal::DoUnserialize(const broker::data& data) { - DO_UNSERIALIZE(OpaqueVal); + auto d = caf::get_if(&data); + if ( ! d ) + return false; + + if ( ! get_vector_idx(*d, 0, &state.totalc) ) + return false; + if ( ! get_vector_idx(*d, 1, &state.mp) ) + return false; + if ( ! get_vector_idx(*d, 2, &state.sccfirst) ) + return false; + if ( ! get_vector_idx(*d, 3, &state.inmont) ) + return false; + if ( ! get_vector_idx(*d, 4, &state.mcount) ) + return false; + if ( ! get_vector_idx(*d, 5, &state.cexp) ) + return false; + if ( ! get_vector_idx(*d, 6, &state.montex) ) + return false; + if ( ! get_vector_idx(*d, 7, &state.montey) ) + return false; + if ( ! get_vector_idx(*d, 8, &state.montepi) ) + return false; + if ( ! get_vector_idx(*d, 9, &state.sccu0) ) + return false; + if ( ! get_vector_idx(*d, 10, &state.scclast) ) + return false; + if ( ! get_vector_idx(*d, 11, &state.scct1) ) + return false; + if ( ! get_vector_idx(*d, 12, &state.scct2) ) + return false; + if ( ! get_vector_idx(*d, 13, &state.scct3) ) + return false; for ( int i = 0; i < 256; ++i ) { - if ( ! UNSERIALIZE(&state.ccount[i]) ) + if ( ! get_vector_idx(*d, 14 + i, &state.ccount[i]) ) return false; } - if ( ! (UNSERIALIZE(&state.totalc) && - UNSERIALIZE(&state.mp) && - UNSERIALIZE(&state.sccfirst)) ) - return false; - for ( int i = 0; i < RT_MONTEN; ++i ) { - if ( ! UNSERIALIZE(&state.monte[i]) ) + if ( ! get_vector_idx(*d, 14 + 256 + i, &state.monte[i]) ) return false; } - if ( ! (UNSERIALIZE(&state.inmont) && - UNSERIALIZE(&state.mcount) && - UNSERIALIZE(&state.cexp) && - UNSERIALIZE(&state.montex) && - UNSERIALIZE(&state.montey) && - UNSERIALIZE(&state.montepi) && - UNSERIALIZE(&state.sccu0) && - UNSERIALIZE(&state.scclast) && - UNSERIALIZE(&state.scct1) && - UNSERIALIZE(&state.scct2) && - UNSERIALIZE(&state.scct3)) ) - return false; - return true; } @@ -739,42 +871,52 @@ BloomFilterVal::~BloomFilterVal() delete bloom_filter; } -IMPLEMENT_SERIAL(BloomFilterVal, SER_BLOOMFILTER_VAL); +IMPLEMENT_OPAQUE_VALUE(BloomFilterVal) -bool BloomFilterVal::DoSerialize(SerialInfo* info) const +broker::expected BloomFilterVal::DoSerialize() const { - DO_SERIALIZE(SER_BLOOMFILTER_VAL, OpaqueVal); + broker::vector d; - bool is_typed = (type != 0); + if ( type ) + { + auto t = SerializeType(type); + if ( ! t ) + return broker::ec::invalid_data; - if ( ! SERIALIZE(is_typed) ) - return false; + d.emplace_back(std::move(*t)); + } + else + d.emplace_back(broker::none()); - if ( is_typed && ! type->Serialize(info) ) - return false; + auto bf = bloom_filter->Serialize(); + if ( ! bf ) + return broker::ec::invalid_data; // Cannot serialize; - return bloom_filter->Serialize(info); + d.emplace_back(*bf); + return {std::move(d)}; } -bool BloomFilterVal::DoUnserialize(UnserialInfo* info) +bool BloomFilterVal::DoUnserialize(const broker::data& data) { - DO_UNSERIALIZE(OpaqueVal); + auto v = caf::get_if(&data); - bool is_typed; - if ( ! UNSERIALIZE(&is_typed) ) + if ( ! (v && v->size() == 2) ) return false; - if ( is_typed ) + auto no_type = caf::get_if(&(*v)[0]); + if ( ! no_type ) { - BroType* t = BroType::Unserialize(info); - if ( ! Typify(t) ) + BroType* t = UnserializeType((*v)[0]); + if ( ! (t && Typify(t)) ) return false; - - Unref(t); } - bloom_filter = probabilistic::BloomFilter::Unserialize(info); - return bloom_filter != 0; + auto bf = probabilistic::BloomFilter::Unserialize((*v)[1]); + if ( ! bf ) + return false; + + bloom_filter = bf.release(); + return true; } CardinalityVal::CardinalityVal() : OpaqueVal(cardinality_type) @@ -805,44 +947,6 @@ Val* CardinalityVal::DoClone(CloneState* state) new CardinalityVal(new probabilistic::CardinalityCounter(*c))); } -IMPLEMENT_SERIAL(CardinalityVal, SER_CARDINALITY_VAL); - -bool CardinalityVal::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_CARDINALITY_VAL, OpaqueVal); - - bool valid = true; - bool is_typed = (type != 0); - - valid &= SERIALIZE(is_typed); - - if ( is_typed ) - valid &= type->Serialize(info); - - return c->Serialize(info); - } - -bool CardinalityVal::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(OpaqueVal); - - bool is_typed; - if ( ! UNSERIALIZE(&is_typed) ) - return false; - - if ( is_typed ) - { - BroType* t = BroType::Unserialize(info); - if ( ! Typify(t) ) - return false; - - Unref(t); - } - - c = probabilistic::CardinalityCounter::Unserialize(info); - return c != 0; - } - bool CardinalityVal::Typify(BroType* arg_type) { if ( type ) @@ -870,3 +974,51 @@ void CardinalityVal::Add(const Val* val) c->AddElement(key->Hash()); delete key; } + +IMPLEMENT_OPAQUE_VALUE(CardinalityVal) + +broker::expected CardinalityVal::DoSerialize() const + { + broker::vector d; + + if ( type ) + { + auto t = SerializeType(type); + if ( ! t ) + return broker::ec::invalid_data; + + d.emplace_back(std::move(*t)); + } + else + d.emplace_back(broker::none()); + + auto cs = c->Serialize(); + if ( ! cs ) + return broker::ec::invalid_data; + + d.emplace_back(*cs); + return {std::move(d)}; + } + +bool CardinalityVal::DoUnserialize(const broker::data& data) + { + auto v = caf::get_if(&data); + + if ( ! (v && v->size() == 2) ) + return false; + + auto no_type = caf::get_if(&(*v)[0]); + if ( ! no_type ) + { + BroType* t = UnserializeType((*v)[0]); + if ( ! (t && Typify(t)) ) + return false; + } + + auto cu = probabilistic::CardinalityCounter::Unserialize((*v)[1]); + if ( ! cu ) + return false; + + c = cu.release(); + return true; + } diff --git a/src/OpaqueVal.h b/src/OpaqueVal.h index 5fe0823436..17ee78e2fd 100644 --- a/src/OpaqueVal.h +++ b/src/OpaqueVal.h @@ -3,12 +3,151 @@ #ifndef OPAQUEVAL_H #define OPAQUEVAL_H -#include - #include "RandTest.h" #include "Val.h" #include "digest.h" +class OpaqueVal; + +/** + * Singleton that registers all available all available types of opaque + * values. This faciliates their serialization into Broker values. + */ +class OpaqueMgr { +public: + using Factory = OpaqueVal* (); + + /** + * Return's a unique ID for the type of an opaque value. + * @param v opaque value to return type for; its class must have been + * registered with the manager, otherwise this method will abort + * execution. + * + * @return type ID, which can used with *Instantiate()* to create a + * new instance of the same type. + */ + const std::string& TypeID(const OpaqueVal* v) const; + + /** + * Instantiates a new opaque value of a specific opaque type. + * + * @param id unique type ID for the class to instantiate; this will + * normally have been returned earlier by *TypeID()*. + * + * @return A freshly instantiated value of the OpaqueVal-derived + * classes that *id* specifies, with reference count at +1. If *id* + * is unknown, this will return null. + * + */ + OpaqueVal* Instantiate(const std::string& id) const; + + /** Returns the global manager singleton object. */ + static OpaqueMgr* mgr(); + + /** + * Internal helper class to register an OpaqueVal-derived classes + * with the manager. + */ + template + class Register { + public: + Register(const char* id) + { OpaqueMgr::mgr()->_types.emplace(id, &T::OpaqueInstantiate); } + }; + +private: + std::unordered_map _types; +}; + +/** Macro to insert into an OpaqueVal-derived class's declaration. */ +#define DECLARE_OPAQUE_VALUE(T) \ + friend class OpaqueMgr::Register; \ + broker::expected DoSerialize() const override; \ + bool DoUnserialize(const broker::data& data) override; \ + const char* OpaqueName() const override { return #T; } \ + static OpaqueVal* OpaqueInstantiate() { return new T(); } + +#define __OPAQUE_MERGE(a, b) a ## b +#define __OPAQUE_ID(x) __OPAQUE_MERGE(_opaque, x) + +/** Macro to insert into an OpaqueVal-derived class's implementation file. */ +#define IMPLEMENT_OPAQUE_VALUE(T) static OpaqueMgr::Register __OPAQUE_ID(__LINE__)(#T); + +/** + * Base class for all opaque values. Opaque values are types that are managed + * completely internally, with no further script-level operators provided + * (other than bif functions). See OpaqueVal.h for derived classes. + */ +class OpaqueVal : public Val { +public: + explicit OpaqueVal(OpaqueType* t); + ~OpaqueVal() override; + + /** + * Serializes the value into a Broker representation. + * + * @return the broker representation, or an error if serialization + * isn't supported or failed. + */ + broker::expected Serialize() const; + + /** + * Reinstantiates a value from its serialized Broker representation. + * + * @param data Broker representation as returned by *Serialize()*. + * @return unserialized instances with reference count at +1 + */ + static OpaqueVal* Unserialize(const broker::data& data); + +protected: + friend class Val; + friend class OpaqueMgr; + OpaqueVal() { } + + /** + * Must be overridden to provide a serialized version of the derived + * class' state. + * + * @return the serialized data or an error if serialization + * isn't supported or failed. + */ + virtual broker::expected DoSerialize() const = 0; + + /** + * Must be overridden to recreate the the derived class' state from a + * serialization. + * + * @return true if successful. + */ + virtual bool DoUnserialize(const broker::data& data) = 0; + + /** + * Internal helper for the serialization machinery. Automatically + * overridden by the `DECLARE_OPAQUE_VALUE` macro. + */ + virtual const char* OpaqueName() const = 0; + + /** + * Provides an implementation of *Val::DoClone()* that leverages the + * serialization methods to deep-copy an instance. Derived classes + * may also override this with a more efficient custom clone + * implementation of their own. + */ + Val* DoClone(CloneState* state) override; + + /** + * Helper function for derived class that need to record a type + * during serialization. + */ + static broker::expected SerializeType(BroType* t); + + /** + * Helper function for derived class that need to restore a type + * during unserialization. Returns the type at reference count +1. + */ + static BroType* UnserializeType(const broker::data& data); +}; + namespace probabilistic { class BloomFilter; class CardinalityCounter; @@ -22,15 +161,13 @@ public: virtual StringVal* Get(); protected: - HashVal() { }; + HashVal() { valid = false; } explicit HashVal(OpaqueType* t); virtual bool DoInit(); virtual bool DoFeed(const void* data, size_t size); virtual StringVal* DoGet(); - DECLARE_SERIAL(HashVal); - private: // This flag exists because Get() can only be called once. bool valid; @@ -56,8 +193,7 @@ protected: bool DoFeed(const void* data, size_t size) override; StringVal* DoGet() override; - DECLARE_SERIAL(MD5Val); - + DECLARE_OPAQUE_VALUE(MD5Val) private: EVP_MD_CTX* ctx; }; @@ -78,8 +214,7 @@ protected: bool DoFeed(const void* data, size_t size) override; StringVal* DoGet() override; - DECLARE_SERIAL(SHA1Val); - + DECLARE_OPAQUE_VALUE(SHA1Val) private: EVP_MD_CTX* ctx; }; @@ -100,8 +235,7 @@ protected: bool DoFeed(const void* data, size_t size) override; StringVal* DoGet() override; - DECLARE_SERIAL(SHA256Val); - + DECLARE_OPAQUE_VALUE(SHA256Val) private: EVP_MD_CTX* ctx; }; @@ -110,8 +244,6 @@ class EntropyVal : public OpaqueVal { public: EntropyVal(); - Val* DoClone(CloneState* state) override; - bool Feed(const void* data, size_t size); bool Get(double *r_ent, double *r_chisq, double *r_mean, double *r_montepicalc, double *r_scc); @@ -119,8 +251,7 @@ public: protected: friend class Val; - DECLARE_SERIAL(EntropyVal); - + DECLARE_OPAQUE_VALUE(EntropyVal) private: RandTest state; }; @@ -149,8 +280,7 @@ protected: BloomFilterVal(); explicit BloomFilterVal(OpaqueType* t); - DECLARE_SERIAL(BloomFilterVal); - + DECLARE_OPAQUE_VALUE(BloomFilterVal) private: // Disable. BloomFilterVal(const BloomFilterVal&); @@ -174,18 +304,16 @@ public: BroType* Type() const; bool Typify(BroType* type); - probabilistic::CardinalityCounter* Get() { return c; }; protected: CardinalityVal(); + DECLARE_OPAQUE_VALUE(CardinalityVal) private: BroType* type; CompositeHash* hash; probabilistic::CardinalityCounter* c; - - DECLARE_SERIAL(CardinalityVal); }; #endif diff --git a/src/RE.cc b/src/RE.cc index b994f16cc2..98f120efe0 100644 --- a/src/RE.cc +++ b/src/RE.cc @@ -9,7 +9,7 @@ #include "DFA.h" #include "CCL.h" #include "EquivClass.h" -#include "Serializer.h" +#include "Reporter.h" CCL* curr_ccl = 0; @@ -469,57 +469,6 @@ int RE_Matcher::Compile(int lazy) return re_anywhere->Compile(lazy) && re_exact->Compile(lazy); } -bool RE_Matcher::Serialize(SerialInfo* info) const - { - return SerialObj::Serialize(info); - } - -RE_Matcher* RE_Matcher::Unserialize(UnserialInfo* info) - { - return (RE_Matcher*) SerialObj::Unserialize(info, SER_RE_MATCHER); - } - -IMPLEMENT_SERIAL(RE_Matcher, SER_RE_MATCHER); - -bool RE_Matcher::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_RE_MATCHER, SerialObj); - return SERIALIZE(re_anywhere->PatternText()) - && SERIALIZE(re_exact->PatternText()); - } - -bool RE_Matcher::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(SerialObj); - - re_anywhere = new Specific_RE_Matcher(MATCH_ANYWHERE); - re_exact = new Specific_RE_Matcher(MATCH_EXACTLY); - - const char* pat; - if ( ! UNSERIALIZE_STR(&pat, 0) ) - return false; - - re_anywhere->SetPat(pat); - if ( ! re_anywhere->Compile() ) - { - info->s->Error(fmt("Can't compile regexp '%s'", pat)); - return false; - } - - if ( ! UNSERIALIZE_STR(&pat, 0) ) - return false; - - re_exact->SetPat(pat); - if ( ! re_exact->Compile() ) - { - info->s->Error(fmt("Can't compile regexp '%s'", pat)); - return false; - } - - return true; - } - - static RE_Matcher* matcher_merge(const RE_Matcher* re1, const RE_Matcher* re2, const char* merge_op) { diff --git a/src/RE.h b/src/RE.h index 286eb1b44d..9386aa6f5f 100644 --- a/src/RE.h +++ b/src/RE.h @@ -171,12 +171,12 @@ protected: int current_pos; }; -class RE_Matcher : SerialObj { +class RE_Matcher { public: RE_Matcher(); explicit RE_Matcher(const char* pat); RE_Matcher(const char* exact_pat, const char* anywhere_pat); - virtual ~RE_Matcher() override; + virtual ~RE_Matcher(); void AddPat(const char* pat); @@ -212,9 +212,6 @@ public: const char* PatternText() const { return re_exact->PatternText(); } const char* AnywherePatternText() const { return re_anywhere->PatternText(); } - bool Serialize(SerialInfo* info) const; - static RE_Matcher* Unserialize(UnserialInfo* info); - unsigned int MemoryAllocation() const { return padded_sizeof(*this) @@ -223,8 +220,6 @@ public: } protected: - DECLARE_SERIAL(RE_Matcher); - Specific_RE_Matcher* re_anywhere; Specific_RE_Matcher* re_exact; }; diff --git a/src/Reassem.cc b/src/Reassem.cc index 7fa70091e0..d952f6e839 100644 --- a/src/Reassem.cc +++ b/src/Reassem.cc @@ -6,7 +6,6 @@ #include "zeek-config.h" #include "Reassem.h" -#include "Serializer.h" static const bool DEBUG_reassem = false; @@ -357,37 +356,3 @@ uint64 Reassembler::MemoryAllocation(ReassemblerType rtype) return Reassembler::sizes[rtype]; } -bool Reassembler::Serialize(SerialInfo* info) const - { - return SerialObj::Serialize(info); - } - -Reassembler* Reassembler::Unserialize(UnserialInfo* info) - { - return (Reassembler*) SerialObj::Unserialize(info, SER_REASSEMBLER); - } - -bool Reassembler::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_REASSEMBLER, BroObj); - - // I'm not sure if it makes sense to actually save the buffered data. - // For now, we just remember the seq numbers so that we don't get - // complaints about missing content. - return SERIALIZE(trim_seq) && SERIALIZE(int(0)); - } - -bool Reassembler::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroObj); - - blocks = last_block = 0; - - int dummy; // For backwards compatibility. - if ( ! UNSERIALIZE(&trim_seq) || ! UNSERIALIZE(&dummy) ) - return false; - - last_reassem_seq = trim_seq; - - return true; - } diff --git a/src/Reassem.h b/src/Reassem.h index 501cd23a18..ee5e4d42b6 100644 --- a/src/Reassem.h +++ b/src/Reassem.h @@ -62,9 +62,6 @@ public: void Describe(ODesc* d) const override; - bool Serialize(SerialInfo* info) const; - static Reassembler* Unserialize(UnserialInfo* info); - // Sum over all data buffered in some reassembler. static uint64 TotalMemoryAllocation() { return total_size; } @@ -76,8 +73,6 @@ public: protected: Reassembler() { } - DECLARE_ABSTRACT_SERIAL(Reassembler); - friend class DataBlock; virtual void Undelivered(uint64 up_to_seq); diff --git a/src/SerialInfo.h b/src/SerialInfo.h index 294c5747ba..04cd79daf7 100644 --- a/src/SerialInfo.h +++ b/src/SerialInfo.h @@ -3,13 +3,10 @@ #ifndef serialinfo_h #define serialinfo_h -#include "ChunkedIO.h" - class SerialInfo { public: SerialInfo(Serializer* arg_s) { - chunk = 0; s = arg_s; may_suspend = clear_containers = false; cache = globals_as_names = true; @@ -21,7 +18,6 @@ public: SerialInfo(const SerialInfo& info) { - chunk = info.chunk; s = info.s; may_suspend = info.may_suspend; cache = info.cache; @@ -49,8 +45,6 @@ public: // If true, we support keeping objs in cache permanently. bool new_cache_strategy; - ChunkedIO::Chunk* chunk; // chunk written right before the serialization - // Attributes set during serialization. SerialType type; // type of currently serialized object @@ -65,7 +59,6 @@ public: s = arg_s; cache = true; type = SER_NONE; - chunk = 0; install_globals = install_conns = true; install_uniques = false; ignore_callbacks = false; @@ -80,7 +73,6 @@ public: s = info.s; cache = info.cache; type = info.type; - chunk = info.chunk; install_globals = info.install_globals; install_uniques = info.install_uniques; install_conns = info.install_conns; @@ -96,8 +88,6 @@ public: bool cache; // if true, object caching is ok FILE* print; // print read objects to given file (human-readable) - ChunkedIO::Chunk* chunk; // chunk to parse (rather than reading one) - bool install_globals; // if true, install unknown globals // in global scope bool install_conns; // if true, add connections to session table diff --git a/src/SerialObj.cc b/src/SerialObj.cc deleted file mode 100644 index ab7f63e823..0000000000 --- a/src/SerialObj.cc +++ /dev/null @@ -1,277 +0,0 @@ -#include "SerialObj.h" -#include "Serializer.h" - -TransientID::ID TransientID::counter = 0; - -SerialObj::FactoryMap* SerialObj::factories = 0; -SerialObj::ClassNameMap* SerialObj::names = 0; -uint64 SerialObj::time_counter = NEVER + ALWAYS + 1; - -SerialObj* SerialObj::Instantiate(SerialType type) - { - FactoryMap::iterator f = factories->find(type & SER_TYPE_MASK_EXACT); - if ( f != factories->end() ) - { - SerialObj* o = (SerialObj*) (*f->second)(); -#ifdef DEBUG - o->serial_type = o->GetSerialType(); -#endif - return o; - } - - reporter->Error("Unknown object type 0x%08x", type); - return 0; - } - -const char* SerialObj::ClassName(SerialType type) - { - ClassNameMap::iterator f = names->find(type); - if ( f != names->end() ) - return f->second; - - reporter->Error("Unknown object type 0x%08x", type); - return ""; - } - -void SerialObj::Register(SerialType type, FactoryFunc f, const char* name) - { - if ( ! factories ) - { - factories = new FactoryMap; - names = new ClassNameMap; - } - - type = type & SER_TYPE_MASK_EXACT; - - FactoryMap::iterator i = factories->find(type); - if ( i != factories->end() ) - reporter->InternalError("SerialType 0x%08x registered twice", type); - - (*factories)[type] = f; - (*names)[type] = name; - } - -inline bool SerializePID(SerialInfo* info, bool full, SerializationCache::PermanentID pid) - { - if ( ! SERIALIZE(full) ) - return false; - - if ( ! info->pid_32bit ) - return SERIALIZE(pid); - - // Broccoli compatibility mode with 32bit pids. - uint32 tmp = uint32(pid); - return SERIALIZE(tmp); - } - -bool SerialObj::Serialize(SerialInfo* info) const - { - assert(info); - - if ( info->cont.NewInstance() ) - { - SerializationCache::PermanentID pid = SerializationCache::NONE; - - const TransientID* tid = GetTID(); - - if ( ! tid ) - reporter->InternalError("no tid - missing DECLARE_SERIAL?"); - - if ( info->cache ) - pid = info->s->Cache()->Lookup(*tid); - - if ( pid != SerializationCache::NONE && info->cache ) - { - DBG_LOG(DBG_SERIAL, "%s [%p, ref pid %lld, tid %lld]", __PRETTY_FUNCTION__, this, (long long) pid, tid->Value() ); - - DBG_LOG(DBG_SERIAL, "-- Caching"); - DBG_PUSH(DBG_SERIAL); - - if ( ! SerializePID(info, false, pid) ) - { - DBG_POP(DBG_SERIAL); - return false; - } - - DBG_POP(DBG_SERIAL); - return true; - } - - if ( info->cache ) - pid = info->s->Cache()->Register(this, - SerializationCache::NONE, - info->new_cache_strategy); - - DBG_LOG(DBG_SERIAL, "%s [%p, new pid %lld, tid %lld]", __PRETTY_FUNCTION__, this, (long long) pid, tid->Value() ); - DBG_LOG(DBG_SERIAL, "-- Caching"); - DBG_PUSH(DBG_SERIAL); - - if ( ! SerializePID(info, true, pid) ) - { - DBG_POP(DBG_SERIAL); - return false; - } - - info->type = SER_NONE; - DBG_POP(DBG_SERIAL); - } - - DBG_PUSH(DBG_SERIAL); - info->cont.SaveContext(); - bool ret = DoSerialize(info); - info->cont.RestoreContext(); - DBG_POP(DBG_SERIAL); - - if ( info->cont.ChildSuspended() ) - return ret; - -#ifdef DEBUG - if ( debug_logger.IsEnabled(DBG_SERIAL) && IsBroObj(serial_type) ) - { - ODesc desc(DESC_READABLE); - ((BroObj*)this)->Describe(&desc); - DBG_LOG(DBG_SERIAL, "-- Desc: %s", desc.Description()); - } -#endif - - return ret; - } - -SerialObj* SerialObj::Unserialize(UnserialInfo* info, SerialType type) - { - SerializationCache::PermanentID pid = SerializationCache::NONE; - - DBG_LOG(DBG_SERIAL, "%s", __PRETTY_FUNCTION__); - - bool full_obj; - - DBG_LOG(DBG_SERIAL, "-- Caching"); - DBG_PUSH(DBG_SERIAL); - - bool result; - - if ( ! info->pid_32bit ) - result = UNSERIALIZE(&full_obj) && UNSERIALIZE(&pid); - else - { - // Broccoli compatibility mode with 32bit pids. - uint32 tmp = 0; - result = UNSERIALIZE(&full_obj) && UNSERIALIZE(&tmp); - pid = tmp; - } - - if ( ! result ) - { - DBG_POP(DBG_SERIAL); - return 0; - } - - DBG_POP(DBG_SERIAL); - - DBG_LOG(DBG_SERIAL, "-- [%s pid %lld]", full_obj ? "obj" : "ref", (long long) pid); - - if ( ! full_obj ) - { - // FIXME: Yet another const_cast to check eventually... - SerialObj* obj = - const_cast(info->s->Cache()->Lookup(pid)); - if ( obj ) - { - if ( obj->IsBroObj() ) - Ref((BroObj*) obj); - return obj; - } - - // In the following we'd like the format specifier to match - // the type of pid; but pid is uint64, for which there's - // no portable format specifier. So we upcast it to long long, - // which is at least that size, and use a matching format. - info->s->Error(fmt("unknown object %lld referenced", - (long long) pid)); - return 0; - } - - uint16 stype; - if ( ! UNSERIALIZE(&stype) ) - return 0; - - SerialObj* obj = Instantiate(SerialType(stype)); - - if ( ! obj ) - { - info->s->Error("unknown object type"); - return 0; - } - -#ifdef DEBUG - obj->serial_type = stype; -#endif - - const TransientID* tid = obj->GetTID(); - if ( ! tid ) - reporter->InternalError("no tid - missing DECLARE_SERIAL?"); - - if ( info->cache ) - info->s->Cache()->Register(obj, pid, info->new_cache_strategy); - - info->type = stype; - - DBG_PUSH(DBG_SERIAL); - if ( ! obj->DoUnserialize(info) ) - { - DBG_POP(DBG_SERIAL); - return 0; - } - - DBG_POP(DBG_SERIAL); - - if ( ! SerialObj::CheckTypes(stype, type) ) - { - info->s->Error("type mismatch"); - return 0; - } - -#ifdef DEBUG - if ( debug_logger.IsEnabled(DBG_SERIAL) && IsBroObj(stype) ) - { - ODesc desc(DESC_READABLE); - ((BroObj*)obj)->Describe(&desc); - DBG_LOG(DBG_SERIAL, "-- Desc: %s", desc.Description()); - } -#endif - - assert(obj); - return obj; - } - -bool SerialObj::DoSerialize(SerialInfo* info) const - { - assert(info->type != SER_NONE); - -#ifdef DEBUG - const_cast(this)->serial_type = info->type; -#endif - - DBG_LOG(DBG_SERIAL, __PRETTY_FUNCTION__); - DBG_PUSH(DBG_SERIAL); - - uint16 stype = uint16(info->type); - - if ( ! info->new_cache_strategy ) - { - // This is a bit unfortunate: to make sure we're sending - // out the same types as in the past, we need to strip out - // the new cache stable bit. - stype &= ~SER_IS_CACHE_STABLE; - } - - bool ret = SERIALIZE(stype); - DBG_POP(DBG_SERIAL); - return ret; - } - -bool SerialObj::DoUnserialize(UnserialInfo* info) - { - DBG_LOG(DBG_SERIAL, __PRETTY_FUNCTION__); - return true; - } diff --git a/src/SerialObj.h b/src/SerialObj.h deleted file mode 100644 index 84334716de..0000000000 --- a/src/SerialObj.h +++ /dev/null @@ -1,382 +0,0 @@ -// Infrastructure for serializable objects. -// -// How to make objects of class Foo serializable: -// -// 1. Derive Foo (directly or indirectly) from SerialObj. -// 2. Add a SER_FOO constant to SerialTypes in SerialTypes.h. -// 3. Add DECLARE_SERIAL(Foo) into class definition. -// 4. Add a (preferably protected) default ctor if it doesn't already exist. -// 5. For non-abstract classes, add IMPLEMENT_SERIAL(Foo, SER_FOO) to *.cc -// 6. Add two methods like this to *.cc (keep names of arguments!) -// -// bool Foo::DoSerialize(SerialInfo* info) const -// { -// DO_SERIALIZE(SER_FOO, ParentClassOfFoo); -// <... serialize class members via methods in Serializer ...> -// return true if everything ok; -// } -// -// bool Foo::DoUnserialize(UnserialInfo* info) -// { -// DO_UNSERIALIZE(ParentClassOfFoo); -// <... unserialize class members via methods in Serializer ...> -// return true if everything ok; -// } -// -// (7. If no parent class of Foo already contains Serialize()/Unserialize() -// methods, these need to be added somewhere too. But most of the various -// parts of the class hierarchy already have them.) - - -#ifndef SERIALOBJ_H -#define SERIALOBJ_H - -#include -#include - -#include "DebugLogger.h" -#include "Continuation.h" -#include "SerialTypes.h" -#include "zeek-config.h" - -#if SIZEOF_LONG_LONG < 8 -# error "Serialization requires that sizeof(long long) is at least 8. (Remove this message only if you know what you're doing.)" -#endif - -class Serializer; -class SerialInfo; -class UnserialInfo; -class SerializationCache; - -// Per-process unique ID. -class TransientID { -public: - TransientID() { id = ++counter; } - - typedef unsigned long long ID; - ID Value() const { return id; } - -private: - ID id; - static ID counter; -}; - -// Abstract base class for serializable objects. -class SerialObj { -public: - virtual ~SerialObj() { } - - virtual const TransientID* GetTID() const { return 0; } - - virtual SerialType GetSerialType() const { return 0; } - - bool IsBroObj() const { return IsBroObj(GetSerialType()); } - bool IsCacheStable() const { return IsCacheStable(GetSerialType()); } - - static const uint64 NEVER = 0; - static const uint64 ALWAYS = 1; - - // Returns time of last modification. This "time" is a monotonically - // increasing counter which is incremented each time a modification is - // performed (more precisely: each time an object is modified which - // returns something different than NEVER). Such times can thus be - // compared to see whether some modification took place before another. - // - // There are two special values: - // NEVER: This object will never change. - // ALWAYS: Always consider this object as changed, i.e., don't - // cache it. - virtual uint64 LastModified() const { return NEVER; } - - // Instantiate an object of the given type. Return nil - // if unknown. - static SerialObj* Instantiate(SerialType type); - - static const char* ClassName(SerialType type); - - // Associate a "factory" function with the given type. - // A factory is a class or function that creates instances - // of a certain type. - - typedef SerialObj* (*FactoryFunc)(); - static void Register(SerialType type, FactoryFunc f, - const char* class_name); - - static bool IsBroObj(SerialType type) - { return type & SER_IS_BRO_OBJ; } - - static bool IsCacheStable(SerialType type) - { return type & SER_IS_CACHE_STABLE; } - - static bool CheckTypes(SerialType type1, SerialType type2) - { return (type1 & SER_TYPE_MASK_PARENT) == - (type2 & SER_TYPE_MASK_PARENT); } - -protected: - friend class SerializationCache; - - SerialObj() - { -#ifdef DEBUG - serial_type = 0; -#endif - } - - // Serializes this object. If info->cache is false, we can use - // DECLARE_NON_CACHEABLE_SERIAL (instead of DECLARE_SERIAL) which - // avoids storing a per-object id. - bool Serialize(SerialInfo* info) const; - - // Unserializes next object. - static SerialObj* Unserialize(UnserialInfo* info, - SerialType type); - - virtual bool DoSerialize(SerialInfo* info) const; - virtual bool DoUnserialize(UnserialInfo* info); - - typedef std::map FactoryMap; - static FactoryMap* factories; - - typedef std::map ClassNameMap; - static ClassNameMap* names; - - static uint64 time_counter; - static uint64 IncreaseTimeCounter() { return ++time_counter; } - static uint64 GetTimeCounter() { return time_counter; } - -#ifdef DEBUG - SerialType serial_type; -#endif -}; - -// A class that registers a factory function upon instantiation. -class SerialTypeRegistrator { -public: - SerialTypeRegistrator(SerialType type, SerialObj::FactoryFunc func, - const char* class_name) - { - SerialObj::Register(type, func, class_name); - } -}; - - -// Macro helpers. - -#define DECLARE_ABSTRACT_SERIAL(classname) \ - bool DoSerialize(SerialInfo*) const override; \ - bool DoUnserialize(UnserialInfo*) override; \ - -#define DECLARE_SERIAL(classname) \ - static classname* Instantiate(); \ - static SerialTypeRegistrator register_type; \ - bool DoSerialize(SerialInfo*) const override; \ - bool DoUnserialize(UnserialInfo*) override; \ - const TransientID* GetTID() const override { return &tid; } \ - SerialType GetSerialType() const override; \ - TransientID tid; - -// Only needed (and usable) for non-abstract classes. -#define IMPLEMENT_SERIAL(classname, classtype) \ - SerialTypeRegistrator classname::register_type(classtype, \ - FactoryFunc(&classname::Instantiate), #classname); \ - SerialType classname::GetSerialType() const { return classtype; }; \ - classname* classname::Instantiate() { return new classname(); } \ - -// Pushes debug level on instantiation and pops when it goes out of scope. -class AutoPush { -public: - AutoPush() { DBG_PUSH(DBG_SERIAL); } - ~AutoPush() { DBG_POP(DBG_SERIAL); } -}; - -// Note that by default we disable suspending. Use DO_SERIALIZE_WITH_SUSPEND -// to enable, but be careful to make sure that whomever calls us is aware of -// the fact (or has already disabled suspension itself). -#define DO_SERIALIZE(classtype, super) \ - DBG_LOG(DBG_SERIAL, __PRETTY_FUNCTION__); \ - if ( info->type == SER_NONE ) \ - info->type = classtype; \ - DisableSuspend suspend(info); \ - AutoPush auto_push; \ - if ( ! super::DoSerialize(info) ) \ - return false; - -// Unfortunately, this is getting quite long. :-( -#define DO_SERIALIZE_WITH_SUSPEND(classtype, super) \ - DBG_LOG(DBG_SERIAL, __PRETTY_FUNCTION__); \ - if ( info->type == SER_NONE ) \ - info->type = classtype; \ - AutoPush auto_push; \ - \ - bool call_super = info->cont.NewInstance(); \ - \ - if ( info->cont.ChildSuspended() ) \ - { \ - void* user_ptr = info->cont.RestoreState(); \ - if ( user_ptr == &call_super ) \ - call_super = true; \ - } \ - \ - if ( call_super ) \ - { \ - info->cont.SaveState(&call_super); \ - info->cont.SaveContext(); \ - bool result = super::DoSerialize(info); \ - info->cont.RestoreContext(); \ - if ( ! result ) \ - return false; \ - if ( info->cont.ChildSuspended() ) \ - return true; \ - info->cont.SaveState(0); \ - } \ - -#define DO_UNSERIALIZE(super) \ - DBG_LOG(DBG_SERIAL, __PRETTY_FUNCTION__); \ - AutoPush auto_push; \ - if ( ! super::DoUnserialize(info) ) \ - return false; - -#define SERIALIZE(x) \ - info->s->Write(x, #x) - -#define SERIALIZE_STR(x, y) \ - info->s->Write(x, y, #x) - -#define SERIALIZE_BIT(bit) \ - info->s->Write(bool(bit), #bit) - -#define UNSERIALIZE(x) \ - info->s->Read(x, #x) - -#define UNSERIALIZE_STR(x, y) \ - info->s->Read(x, y, #x) - -#define UNSERIALIZE_BIT(bit) \ - { \ - bool tmp; \ - if ( ! info->s->Read(&tmp, #bit) ) \ - return false; \ - bit = (unsigned int) tmp; \ - } - -// Some helpers for pointers which may be nil. -#define SERIALIZE_OPTIONAL(ptr) \ - { \ - if ( ptr ) \ - { \ - if ( ! info->cont.ChildSuspended() ) \ - if ( ! info->s->Write(true, "has_" #ptr) ) \ - return false; \ - \ - info->cont.SaveContext(); \ - bool result = ptr->Serialize(info); \ - info->cont.RestoreContext(); \ - if ( ! result ) \ - return false; \ - \ - if ( info->cont.ChildSuspended() ) \ - return true; \ - } \ - \ - else if ( ! info->s->Write(false, "has_" #ptr) ) \ - return false; \ - } - -#define SERIALIZE_OPTIONAL_STR(str) \ - { \ - if ( str ) \ - { \ - if ( ! (info->s->Write(true, "has_" #str) && info->s->Write(str, "str")) ) \ - return false; \ - } \ - \ - else if ( ! info->s->Write(false, "has_" #str) ) \ - return false; \ - } - -#define UNSERIALIZE_OPTIONAL(dst, unserialize) \ - { \ - bool has_it; \ - if ( ! info->s->Read(&has_it, "has_" #dst) ) \ - return false; \ - \ - if ( has_it ) \ - { \ - dst = unserialize; \ - if ( ! dst ) \ - return false; \ - } \ - \ - else \ - dst = 0; \ - } - -#define UNSERIALIZE_OPTIONAL_STR(dst) \ - { \ - bool has_it; \ - if ( ! info->s->Read(&has_it, "has_" #dst) ) \ - return false; \ - \ - if ( has_it ) \ - { \ - if ( ! info->s->Read(&dst, 0, "has_" #dst) ) \ - return false; \ - if ( ! dst ) \ - return false; \ - } \ - \ - else \ - dst = 0; \ - } - -#define UNSERIALIZE_OPTIONAL_STR_DEL(dst, del) \ - { \ - bool has_it; \ - if ( ! info->s->Read(&has_it, "has_" #dst) ) \ - { \ - delete del; \ - return 0; \ - } \ - \ - if ( has_it ) \ - { \ - if ( ! info->s->Read(&dst, 0, "has_" #dst) ) \ - { \ - delete del; \ - return 0; \ - } \ - if ( ! dst ) \ - { \ - delete del; \ - return 0; \ - } \ - } \ - \ - else \ - dst = 0; \ - } - -#define UNSERIALIZE_OPTIONAL_STATIC(dst, unserialize, del) \ - { \ - bool has_it; \ - if ( ! info->s->Read(&has_it, "has_" #dst) ) \ - { \ - delete del; \ - return 0; \ - } \ - \ - if ( has_it ) \ - { \ - dst = unserialize; \ - if ( ! dst ) \ - { \ - delete del; \ - return 0; \ - } \ - } \ - \ - else \ - dst = 0; \ - } - -#endif diff --git a/src/SerializationFormat.cc b/src/SerializationFormat.cc index d5f366f7fd..6505598fc8 100644 --- a/src/SerializationFormat.cc +++ b/src/SerializationFormat.cc @@ -2,7 +2,7 @@ #include "net_util.h" #include "SerializationFormat.h" -#include "Serializer.h" +#include "DebugLogger.h" #include "Reporter.h" const float SerializationFormat::GROWTH_FACTOR = 2.5; diff --git a/src/Serializer.cc b/src/Serializer.cc deleted file mode 100644 index 2b9860ca08..0000000000 --- a/src/Serializer.cc +++ /dev/null @@ -1,1059 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Serializer.h" -#include "Scope.h" -#include "Stmt.h" -#include "Reporter.h" -#include "Func.h" -#include "Event.h" -#include "EventRegistry.h" -#include "SerializationFormat.h" -#include "NetVar.h" -#include "Conn.h" -#include "Timer.h" -#include "iosource/Manager.h" - -Serializer::Serializer(SerializationFormat* arg_format) - { - if ( arg_format ) - format = arg_format; - else - format = new BinarySerializationFormat(); - - io = 0; - error_descr = 0; - current_cache = 0; - } - -Serializer::~Serializer() - { - delete format; - delete [] error_descr; - } - -bool Serializer::Read(string* s, const char* tag) - { - char* cstr; - int len; - if ( format->Read(&cstr, &len, tag) ) - { - s->assign(cstr, len); - delete [] cstr; - return true; - } - else - return false; - } - -bool Serializer::StartSerialization(SerialInfo* info, const char* descr, - char tag) - { - format->StartWrite(); - assert(current_cache); - SetErrorDescr(fmt("serializing %s", descr)); - if ( ! Write(tag, "tag") ) - { - Error(io->Error()); - return false; - } - - current_cache->Begin(info->new_cache_strategy); - return true; - } - -bool Serializer::EndSerialization(SerialInfo* info) - { - if ( info->chunk ) - { - if ( ! io->Write(info->chunk) ) - { - Error(io->Error()); - return false; - } - } - - ChunkedIO::Chunk* chunk = new ChunkedIO::Chunk; - chunk->len = format->EndWrite(&chunk->data); - chunk->free_func = ChunkedIO::Chunk::free_func_free; - - if ( ! io->Write(chunk) ) - { - Error(io->Error()); - return false; - } - - current_cache->End(info->new_cache_strategy); - return true; - } - -bool Serializer::Serialize(SerialInfo* info, const ID& id) - { - if ( info->cont.NewInstance() ) - { - if ( ! (id.IsGlobal() || id.IsEnumConst()) ) - { - Error("non-global identifiers cannot be serialized"); - return false; - } - - if ( ! StartSerialization(info, "ID", 'i') ) - return false; - } - - info->cont.SaveContext(); - bool result = id.Serialize(info); - info->cont.RestoreContext(); - - if ( ! result ) - { - Error("failed"); - return false; - } - - if ( info->cont.ChildSuspended() ) - return true; - - WriteSeparator(); - return EndSerialization(info); - } - -bool Serializer::Serialize(SerialInfo* info, const char* func, val_list* args) - { - DisableSuspend suspend(info); - - if ( ! StartSerialization(info, "call", 'e') ) - return false; - - WriteOpenTag("call"); - int a = args->length(); - Write(func, "name"); - Write(network_time, "time"); - Write(a, "len"); - - loop_over_list(*args, i) - { - if ( ! (*args)[i]->Serialize(info) ) - { - Error("failed"); - return false; - } - } - - WriteCloseTag("call"); - WriteSeparator(); - - return EndSerialization(info); - } - -bool Serializer::Serialize(SerialInfo* info, const StateAccess& s) - { - DisableSuspend suspend(info); - - if ( ! StartSerialization(info, "state access", 's') ) - return false; - - if ( ! s.Serialize(info) ) - { - Error("failed"); - return false; - } - - return EndSerialization(info); - } - -bool Serializer::Serialize(SerialInfo* info, const Timer& t) - { - DisableSuspend suspend(info); - - if ( ! StartSerialization(info, "timer", 't') ) - return false; - - if ( ! t.Serialize(info) ) - { - Error("failed"); - return false; - } - - return EndSerialization(info); - } - -bool Serializer::Serialize(SerialInfo* info, const Connection& c) - { - DisableSuspend suspend(info); - - if ( ! StartSerialization(info, "connection", 'c') ) - return false; - - if ( ! c.Serialize(info) ) - { - Error("failed"); - return false; - } - - return EndSerialization(info); - } - -bool Serializer::Serialize(SerialInfo* info, const Packet& p) - { - DisableSuspend suspend(info); - - if ( ! StartSerialization(info, "packet", 'p') ) - return false; - - if ( ! p.Serialize(info) ) - { - Error("failed"); - return false; - } - - return EndSerialization(info); - } - -int Serializer::Unserialize(UnserialInfo* info, bool block) - { - assert(current_cache); - - SetErrorDescr("unserializing"); - - current_cache->Begin(info->new_cache_strategy); - - ChunkedIO::Chunk* chunk = info->chunk; - - while ( ! chunk ) - { - if ( ! io->Read(&chunk) ) - { - if ( io->Eof() ) - return 0; - Error(io->Error()); - return -1; - } - - if ( ! chunk && ! block ) - return 0; - } - - format->StartRead(chunk->data, chunk->len); - - char type; - if ( ! format->Read(&type, "tag") ) - return -1; - -// DEBUG(fmt("parent: serialization of size %d", ); - - bool result; - switch ( type ) { - case 'i': - result = UnserializeID(info); - break; - - case 'e': - result = UnserializeCall(info); - break; - - case 's': - result = UnserializeStateAccess(info); - break; - - case 'c': - result = UnserializeConnection(info); - break; - - case 't': - result = UnserializeTimer(info); - break; - - case 'p': - result = UnserializePacket(info); - break; - - default: - Error(fmt("unknown serialization type %x", (int) type)); - result = false; - } - - format->EndRead(); - - if ( ! info->chunk ) - { // only delete if we allocated it ourselves - delete chunk; - } - - current_cache->End(info->new_cache_strategy); - - return result ? 1 : -1; - } - -bool Serializer::UnserializeID(UnserialInfo* info) - { - SetErrorDescr("unserializing ID"); - - ID* id = ID::Unserialize(info); - - if ( ! id ) - return false; - - if ( info->print ) - { - ODesc d; - d.SetQuotes(true); - d.SetIncludeStats(true); - d.SetShort(); - id->DescribeExtended(&d); - fprintf(info->print, "ID %s\n", d.Description()); - } - - if ( ! info->ignore_callbacks ) - GotID(id, id->ID_Val()); - else - Unref(id); - - return true; - } - -bool Serializer::UnserializeCall(UnserialInfo* info) - { - char* name; - int len; - double time; - - if ( ! (UNSERIALIZE_STR(&name, 0) && UNSERIALIZE(&time) && UNSERIALIZE(&len)) ) - return false; - - SetErrorDescr(fmt("unserializing event/function %s", name)); - - bool ignore = false; - FuncType* functype = 0; - type_list* types = 0; - - ID* id = global_scope()->Lookup(name); - - if ( id ) - { - if ( id->Type()->Tag() == TYPE_FUNC ) - { - functype = id->Type()->AsFuncType(); - types = functype->ArgTypes()->Types(); - if ( types->length() != len ) - { - Error("wrong number of arguments, ignoring"); - ignore = true; - } - } - else - { - Error("not a function/event, ignoring"); - ignore = true; - } - } - else - { - Error("unknown event/function, ignoring"); - ignore = true; - } - - ODesc d; - d.SetQuotes(true); - d.SetIncludeStats(true); - d.SetShort(); - - val_list* args = new val_list(len); - for ( int i = 0; i < len; ++i ) - { - Val* v = Val::Unserialize(info); - - if ( ! v ) - { - delete [] name; - delete_vals(args); - return false; - } - - if ( ! ignore ) - { - if ( v->Type()->Tag() != (*types)[i]->Tag() && - (*types)[i]->Tag() != TYPE_ANY ) - { - Error("mismatch in argument types; ignoring"); - ignore = true; - } - - if ( info->print && ! ignore ) - v->Describe(&d); - } - - args->append(v); - } - - if ( ! ignore ) - { - if ( info->print ) - fprintf(info->print, "%s [%.06f] %s(%s)\n", - functype->FlavorString().c_str(), - time, name, types ? d.Description() : ""); - - switch ( functype->Flavor() ) { - - case FUNC_FLAVOR_EVENT: - { - EventHandler* handler = event_registry->Lookup(name); - assert(handler); - - if ( ! info->ignore_callbacks ) - GotEvent(name, time, handler, args); - - break; - } - - case FUNC_FLAVOR_FUNCTION: - case FUNC_FLAVOR_HOOK: - if ( ! info->ignore_callbacks ) - GotFunctionCall(name, time, id->ID_Val()->AsFunc(), args); - break; - - default: - reporter->InternalError("unserialized call for invalid function flavor"); - break; - } - - if ( info->ignore_callbacks ) - delete_vals(args); - } - else - delete_vals(args); - - delete [] name; - - return true; - } - -bool Serializer::UnserializeStateAccess(UnserialInfo* info) - { - SetErrorDescr("unserializing state access"); - - StateAccess* s = StateAccess::Unserialize(info); - - if ( ! s ) - return false; - - if ( info->print ) - { - ODesc d; - d.SetQuotes(true); - d.SetIncludeStats(true); - d.SetShort(); - s->Describe(&d); - fprintf(info->print, "State access: %s\n", d.Description()); - } - - if ( ! info->ignore_callbacks ) - GotStateAccess(s); - else - delete s; - - return true; - } - -bool Serializer::UnserializeTimer(UnserialInfo* info) - { - SetErrorDescr("unserializing timer"); - - Timer* t = Timer::Unserialize(info); - - if ( ! t ) - return false; - - if ( info->print ) - { - ODesc d; - d.SetQuotes(true); - d.SetIncludeStats(true); - d.SetShort(); - t->Describe(&d); - fprintf(info->print, "Timer: %s\n", d.Description()); - } - - if ( ! info->ignore_callbacks ) - GotTimer(t); - - return true; - } - -bool Serializer::UnserializeConnection(UnserialInfo* info) - { - SetErrorDescr("unserializing connection"); - - Connection* c = Connection::Unserialize(info); - - if ( ! c ) - return false; - - if ( info->print ) - { - ODesc d; - d.SetQuotes(true); - d.SetIncludeStats(true); - d.SetShort(); - c->Describe(&d); - fprintf(info->print, "Connection: %s", d.Description()); - } - - if ( info->install_conns ) - { - Ref(c); - sessions->Insert(c); - } - else - // We finish the connection here because it's not part - // of the standard processing and most likely to be - // discarded pretty soon. - // Without the Done(), some cleanup may not take place. - c->Done(); - - if ( ! info->ignore_callbacks ) - GotConnection(c); - else - Unref(c); - - return true; - } - -bool Serializer::UnserializePacket(UnserialInfo* info) - { - SetErrorDescr("unserializing packet"); - - Packet* p = Packet::Unserialize(info); - - if ( ! p ) - return false; - - if ( info->print ) - { - ODesc d; - d.SetQuotes(true); - d.SetIncludeStats(true); - d.SetShort(); - p->Describe(&d); - fprintf(info->print, "Packet: %s", d.Description()); - } - - if ( ! info->ignore_callbacks ) - GotPacket(p); - else - delete p; - - return true; - } - -void Serializer::Error(const char* str) - { - char buffer[1024]; - safe_snprintf(buffer, sizeof(buffer), "%s%s%s", - error_descr ? error_descr : "", error_descr ? ": " : "", str); - ReportError(buffer); - } - -void Serializer::Warning(const char* str) - { - // We ignore these as there's no good place to report them. - } - -SerializationCache::SerializationCache(unsigned int arg_max_cache_size) - { - max_cache_size = arg_max_cache_size; - next_id = 1; - cache_stable.head = cache_stable.tail = 0; - cache_unstable.head = cache_unstable.tail = 0; - cache_stable.size = cache_unstable.size = 0; - } - -SerializationCache::~SerializationCache() - { - Clear(); - } - -SerializationCache::PermanentID -SerializationCache::Register(const SerialObj* obj, PermanentID pid, - bool new_cache_strategy) - { - if ( pid == NONE ) - pid = next_id++; - - PIDMap::iterator i = pid_map.find(pid); - assert(i == pid_map.end()); - - CacheList* cache = - (new_cache_strategy && obj->IsCacheStable()) ? - &cache_stable : &cache_unstable; - - CacheEntry* entry = new CacheEntry; - entry->obj.serial = obj; - entry->is_bro_obj = obj->IsBroObj(); - entry->pid = pid; - entry->tid = obj->GetTID()->Value(); - entry->time = SerialObj::GetTimeCounter(); - entry->prev = cache->tail; - entry->next = 0; - entry->cache = cache; - entry->stype = obj->GetSerialType(); - - if ( cache->tail ) - cache->tail->next = entry; - if ( ! cache->head ) - cache->head = entry; - - cache->tail = entry; - ++(cache->size); - - // This is a bit weird. If the TID is already contained in the map (i.e. - // we're re-registering), TIDMap::insert() will *not* override the old - // entry but set the bool to false and return it. - pair old = tid_map.insert(TIDMap::value_type(entry->tid, entry)); - if ( ! old.second ) - { - // Already existed. - old.first->second->tid = 0; // invalidate - old.first->second = entry; // replace - } - - pid_map.insert(PIDMap::value_type(pid, entry)); - - if ( entry->is_bro_obj ) - Ref(const_cast(entry->obj.bro)); - else - { - // Make sure it goes into unstable. - assert(! obj->IsCacheStable()); - - volatiles.push_back(entry); - } - - return entry->pid; - } - -void SerializationCache::UnlinkEntry(CacheEntry* e) - { - assert(e); - - // Remove from double-linked list. - if ( e == e->cache->head ) - { - e->cache->head = e->next; - if ( e->cache->head ) - e->cache->head->prev = 0; - } - else - e->prev->next = e->next; - - if ( e == e->cache->tail ) - { - e->cache->tail = e->prev; - if ( e->cache->tail ) - e->cache->tail->next = 0; - } - else - e->next->prev = e->prev; - - e->prev = e->next = 0; - } - -void SerializationCache::RemoveEntry(CacheEntry* e) - { - assert(e); - UnlinkEntry(e); - - if ( e->tid ) - tid_map.erase(e->tid); - - pid_map.erase(e->pid); - - if ( e->is_bro_obj ) - Unref(const_cast(e->obj.bro)); - - e->obj.serial = 0; // for debugging - --(e->cache->size); - delete e; - } - -void SerializationCache::MoveEntryToTail(CacheEntry* e) - { - assert(e); - UnlinkEntry(e); - e->prev = e->cache->tail; - e->next = 0; - - if ( e->cache->tail ) - e->cache->tail->next = e; - if ( ! e->cache->head ) - e->cache->head = e; - - e->cache->tail = e; - } - -void SerializationCache::Clear() - { - tid_map.clear(); - pid_map.clear(); - volatiles.clear(); - - while ( cache_stable.head ) - RemoveEntry(cache_stable.head); - - while ( cache_unstable.head ) - RemoveEntry(cache_unstable.head); - - assert(cache_stable.size == 0); - assert(cache_unstable.size == 0); - } - -void SerializationCache::End(bool new_cache_strategy) - { - // Remove objects not-derived from BroObj (they aren't ref'counted - // so it's not safe to keep them). - for ( VolatileList::iterator i = volatiles.begin(); - i != volatiles.end(); i++ ) - { - assert(*i); - RemoveEntry(*i); - } - - volatiles.clear(); - - if ( new_cache_strategy ) - { - while ( max_cache_size && cache_stable.head && - cache_stable.size > max_cache_size ) - RemoveEntry(cache_stable.head); - - while ( max_cache_size && cache_unstable.head && - cache_unstable.size > max_cache_size ) - RemoveEntry(cache_unstable.head); - } - - else - { - while ( max_cache_size && pid_map.size() > max_cache_size ) - RemoveEntry(cache_unstable.head); - } - } - -FileSerializer::FileSerializer(SerializationFormat* format) -: Serializer(format), cache(100) - { - file = 0; - fd = -1; - io = 0; - SetCache(&cache); - } - -FileSerializer::~FileSerializer() - { - if ( io ) - io->Flush(); - - delete [] file; - - if ( io ) - delete io; // destructor will call close() on fd - else if ( fd >= 0 ) - safe_close(fd); - } - -bool FileSerializer::Open(const char* file, bool pure) - { - if ( ! OpenFile(file, false) ) - return false; - - if ( pure ) - io->MakePure(); - - if ( ! PrepareForWriting() ) - return false; - - return true; - } - -bool FileSerializer::Close() - { - CloseFile(); - return true; - } - -bool FileSerializer::OpenFile(const char* arg_file, bool readonly, bool should_exist) - { - CloseFile(); - - cache.Clear(); - - file = copy_string(arg_file); - fd = open(file, readonly ? O_RDONLY : O_WRONLY | O_CREAT | O_TRUNC, 0600); - - if ( fd < 0 ) - { - if ( readonly && errno == ENOENT ) - { - // Only an error if we expect to exist. - if ( should_exist ) - { - Error(fmt("%s does not exist", file)); - return false; - } - - CloseFile(); - return true; - } - - Error(fmt("can't open file %s for %s: %s", - file, (readonly ? "reading" : "writing"), - strerror(errno))); - return false; - } - - io = new ChunkedIOFd(fd, "file"); - - return io != 0; - } - -void FileSerializer::CloseFile() - { - if ( io ) - io->Flush(); - - if ( fd >= 0 && ! io ) // destructor of io calls close() on fd - safe_close(fd); - fd = -1; - - delete [] file; - file = 0; - - delete io; - io = 0; - - cache.Clear(); - } - -bool FileSerializer::PrepareForWriting() - { - if ( ! io->IsPure() ) - { - // Write file header. - uint32 magic = htonl(MAGIC); - uint16 version = htons(DATA_FORMAT_VERSION); - uint32 time = htonl(uint32(::time(0))); - - if ( write(fd, &magic, sizeof(magic)) != sizeof(magic ) || - write(fd, &version, sizeof(version)) != sizeof(version) || - write(fd, &time, sizeof(time)) != sizeof(time)) - { - Error(fmt("can't write file header to %s: %s", - file, strerror(errno))); - return false; - } - } - - return true; - } - -bool FileSerializer::ReadHeader(UnserialInfo* info) - { - uint32 magic; - uint16 version; - uint32 time; - - if ( read(fd, &magic, sizeof(magic)) != sizeof(magic ) || - read(fd, &version, sizeof(version)) != sizeof(version) || - read(fd, &time, sizeof(time)) != sizeof(time) ) - { - Error(fmt("can't read file header from %s: %s", - file, strerror(errno))); - return false; - } - - version = ntohs(version); - time = ntohl(time); - - if ( info && info->print ) - { - time_t teatime = (time_t) time; - fprintf(stderr, "Date: %s", ctime(&teatime)); - } - - if ( magic != htonl(MAGIC) ) - { - Error(fmt("%s is not a Zeek state file", file)); - CloseFile(); - return false; - } - - if ( version != DATA_FORMAT_VERSION ) - { - Error(fmt("wrong data format, expected version %d but got version %d", DATA_FORMAT_VERSION, version)); - CloseFile(); - return false; - } - - return true; - } - -bool FileSerializer::Read(UnserialInfo* info, const char* file, bool header) - { - if ( ! OpenFile(file, true, info->print) ) - return false; - - // fprintf( stderr, "Reading %s\n", file ); - - if ( fd < 0 ) - // Not existent, but that's ok. - return true; - - if ( header && ! ReadHeader(info) ) - return false; - - int i; - while ( (i = Unserialize(info, true)) > 0 ) - ; - - CloseFile(); - - return i == 0; - } - -void FileSerializer::ReportError(const char* str) - { - reporter->Error("%s", str); - } - -void FileSerializer::GotID(ID* id, Val* val) - { - // Do nothing. - Unref(id); - } - -void FileSerializer::GotStateAccess(StateAccess* s) - { - delete s; - } - -void FileSerializer::GotEvent(const char* name, double time, - EventHandlerPtr event, val_list* args) - { - // Do nothing. - delete_vals(args); - } - -void FileSerializer::GotFunctionCall(const char* name, double time, - Func* func, val_list* args) - { - // Do nothing. - delete_vals(args); - } - -void FileSerializer::GotTimer(Timer* t) - { - // Do nothing. - delete t; - } - -void FileSerializer::GotConnection(Connection* c) - { - // Do nothing. - Unref(c); - } - -void FileSerializer::GotPacket(Packet* p) - { - // Do nothing. - delete p; - } - -EventPlayer::EventPlayer(const char* file) - : stream_time(), replay_time(), ne_time(), ne_handler(), ne_args() - { - if ( ! OpenFile(file, true) || fd < 0 ) - Error(fmt("event replayer: cannot open %s", file)); - - if ( ReadHeader() ) - iosource_mgr->Register(this); - } - -EventPlayer::~EventPlayer() - { - CloseFile(); - } - -void EventPlayer::GotEvent(const char* name, double time, - EventHandlerPtr event, val_list* args) - { - ne_time = time; - ne_handler = event; - ne_args = std::move(*args); - delete args; - } - -void EventPlayer::GotFunctionCall(const char* name, double time, - Func* func, val_list* args) - { - // We don't replay function calls. - } - -void EventPlayer::GetFds(iosource::FD_Set* read, iosource::FD_Set* write, - iosource::FD_Set* except) - { - read->Insert(fd); - } - -double EventPlayer::NextTimestamp(double* local_network_time) - { - if ( ne_time ) - return ne_time; - - if ( ! io ) - return -1; - - // Read next event if we don't have one waiting. - if ( ! ne_time ) - { - UnserialInfo info(this); - Unserialize(&info); - SetClosed(io->Eof()); - } - - if ( ! ne_time ) - return -1; - - if ( ! network_time ) - { - // Network time not initialized yet. - stream_time = replay_time = ne_time; - return ne_time; - } - - if ( ! stream_time ) - { - // Init base times. - stream_time = ne_time; - replay_time = network_time; - } - - // Scale time. - ne_time = ne_time - stream_time + network_time; - return ne_time; - } - -void EventPlayer::Process() - { - if ( ! (io && ne_time) ) - return; - - Event* event = new Event(ne_handler, std::move(ne_args)); - mgr.Dispatch(event); - - ne_time = 0; - } diff --git a/src/Serializer.h b/src/Serializer.h deleted file mode 100644 index 2c30ef5443..0000000000 --- a/src/Serializer.h +++ /dev/null @@ -1,363 +0,0 @@ -#ifndef SERIALIZER_H -#define SERIALIZER_H - -#include -#include -#include - -#include "ID.h" -#include "List.h" -#include "Expr.h" -#include "ChunkedIO.h" -#include "SerializationFormat.h" -#include "StateAccess.h" -#include "PriorityQueue.h" -#include "SerialInfo.h" -#include "IP.h" -#include "Timer.h" -#include "iosource/IOSource.h" -#include "Reporter.h" - -class SerializationCache; -class SerialInfo; - -class Connection; -class Timer; -class Packet; - -class Serializer { -public: - // Currently ID serialization is the only method which may suspend. - bool Serialize(SerialInfo* info, const ID& id); - bool Serialize(SerialInfo* info, const char* func, val_list* args); - bool Serialize(SerialInfo* info, const StateAccess& s); - bool Serialize(SerialInfo* info, const Connection& c); - bool Serialize(SerialInfo* info, const Timer& t); - bool Serialize(SerialInfo* info, const Packet& p); - - // Access to the current cache. - SerializationCache* Cache() { return current_cache; } - void SetCache(SerializationCache* cache) - { current_cache = cache; } - - // Input/output methods. - -#define DECLARE_READ(type) \ - bool Read(type* v, const char* tag) { return format->Read(v, tag); } - -#define DECLARE_WRITE(type) \ - bool Write(type v, const char* tag) \ - { return format->Write(v, tag); } - -#define DECLARE_IO(type) \ - DECLARE_READ(type) \ - DECLARE_WRITE(type) - - DECLARE_IO(int) - DECLARE_IO(uint16) - DECLARE_IO(uint32) - DECLARE_IO(int64) - DECLARE_IO(uint64) - DECLARE_IO(char) - DECLARE_IO(bool) - DECLARE_IO(double) - - bool Read(char** str, int* len, const char* tag) - { return format->Read(str, len, tag); } - bool Read(const char** str, int* len, const char* tag) - // This cast is ok. - { return format->Read(const_cast(str), len, tag); } - - bool Read(string* s, const char* tag); - bool Read(IPAddr* a, const char* tag) { return format->Read(a, tag); } - bool Read(IPPrefix* p, const char* tag) { return format->Read(p, tag); } - - bool Write(const char* s, const char* tag) - { return format->Write(s, tag); } - bool Write(const char* buf, int len, const char* tag) - { return format->Write(buf, len, tag); } - bool Write(const string& s, const char* tag) - { return format->Write(s.data(), s.size(), tag); } - bool Write(const IPAddr& a, const char* tag) { return format->Write(a, tag); } - bool Write(const IPPrefix& p, const char* tag) { return format->Write(p, tag); } - - bool WriteOpenTag(const char* tag) - { return format->WriteOpenTag(tag); } - bool WriteCloseTag(const char* tag) - { return format->WriteCloseTag(tag); } - - bool WriteSeparator() { return format->WriteSeparator(); } - - void Error(const char* msg); - void Warning(const char* msg); - - void SetErrorDescr(const char* descr) - { delete [] error_descr; error_descr = copy_string(descr); } - -protected: - // Format defaults to binary serialization. - explicit Serializer(SerializationFormat* format = 0); - virtual ~Serializer(); - - // Reads next object. - // If 'block' is true, wait until an object can be read. - // Returns 0 if no more object available, -1 on error. - int Unserialize(UnserialInfo* info, bool block = false); - - // Callback for error messages. - virtual void ReportError(const char* msg) = 0; - - // Callbacks for unserialized objects. - - // id points to ID in global scope, val is unserialized value. - virtual void GotID(ID* id, Val* val) = 0; - virtual void GotEvent(const char* name, double time, - EventHandlerPtr event, val_list* args) = 0; - virtual void GotFunctionCall(const char* name, double time, - Func* func, val_list* args) = 0; - virtual void GotStateAccess(StateAccess* s) = 0; - virtual void GotTimer(Timer* t) = 0; - virtual void GotConnection(Connection* c) = 0; - virtual void GotPacket(Packet* packet) = 0; - - // Magic to recognize state files. - static const uint32 MAGIC = 0x42525354; - - // This will be increased whenever there is an incompatible change - // in the data format. - static const uint32 DATA_FORMAT_VERSION = 26; - - ChunkedIO* io; - -private: - bool StartSerialization(SerialInfo* info, const char* descr, char tag); - bool EndSerialization(SerialInfo* info); - - bool UnserializeID(UnserialInfo* info); - bool UnserializeCall(UnserialInfo* info); - bool UnserializeStateAccess(UnserialInfo* info); - bool UnserializeTimer(UnserialInfo* info); - bool UnserializeConnection(UnserialInfo* info); - bool UnserializePacket(UnserialInfo* info); - - SerializationFormat* format; - SerializationCache* current_cache; - const char* error_descr; // used in error messages -}; - - - -// We maintain an LRU-cache for some of the objects which have already been -// serialized. For the cache, we need two types of IDs: TransientIDs (defined -// in SerialObj.cc) uniquely reference an object during the lifetime of a -// process. PermanentIDs uniquely reference an object within a serialization. - -class SerializationCache { -public: - typedef uint64 PermanentID; - static const PermanentID NONE = 0; - - // If max_cache_size is greater than zero, we'll remove old entries - // automatically if limit is reached (LRU expiration). - explicit SerializationCache(unsigned int max_cache_size = 0); - ~SerializationCache(); - - PermanentID Register(const SerialObj* obj, PermanentID pid, - bool new_cache_strategy); - - const SerialObj* Lookup(PermanentID pid) - { - PIDMap::const_iterator i = pid_map.find(pid); - if ( i == pid_map.end() ) - return 0; - - assert(i->second); - MoveEntryToTail(i->second); - return i->second->obj.serial; - } - - PermanentID Lookup(const TransientID& tid) - { - TIDMap::const_iterator i = tid_map.find(tid.Value()); - if ( i == tid_map.end() ) - return 0; - - uint64 modified = i->second->obj.serial->LastModified(); - if ( modified == SerialObj::ALWAYS || modified > i->second->time ) - return 0; - - assert(i->second); - MoveEntryToTail(i->second); - return i->second->pid; - } - - unsigned int GetMaxCacheSize() const { return max_cache_size; } - void SetMaxCacheSize(unsigned int size) { max_cache_size = size; } - - // These methods have to be called at the start/end of the - // serialization of an entity. The cache guarentees that objects - // registered after Begin() remain valid until End() is called. - // After End(), objects which are not derived from BroObj are - // discarded; others *may* remain valid. - void Begin(bool can_keep_in_cache) { End(can_keep_in_cache); } - void End(bool can_keep_in_cache); - - void Clear(); - -private: - - struct CacheList; - - struct CacheEntry { - union { - const SerialObj* serial; - const BroObj* bro; - } obj; - - bool is_bro_obj; - PermanentID pid; - TransientID::ID tid; - uint64 time; - struct CacheList* cache; - CacheEntry* prev; - CacheEntry* next; - - SerialType stype; // primarily for debugging - }; - - // We maintain two LRU-sorted lists, one for often-changing objects and - // one for only rarely changing objects; - struct CacheList { - CacheEntry* head; - CacheEntry* tail; - unsigned int size; - }; - - void RemoveEntry(CacheEntry* e); - void UnlinkEntry(CacheEntry* e); - void MoveEntryToTail(CacheEntry* e); - - unsigned int max_cache_size; - - typedef map PIDMap; - typedef map TIDMap; - - TIDMap tid_map; - PIDMap pid_map; - - CacheList cache_stable; - CacheList cache_unstable; - - // Objects in the cache which aren't derived from BroObj. These are - // always stored in the unstable cache. - typedef list VolatileList; - VolatileList volatiles; - - PermanentID next_id; -}; - -// A serializer for cloning objects. Objects can be serialized into -// the serializer and unserialized into new objects. An absolutely -// minimal implementation of Serializer! -class CloneSerializer : public Serializer { -public: - explicit CloneSerializer(SerializationFormat* format = 0) : Serializer(format) { } - ~CloneSerializer() override - { } - -protected: - void ReportError(const char* msg) override { reporter->Error("%s", msg); } - void GotID(ID* id, Val* val) override { } - void GotEvent(const char* name, double time, EventHandlerPtr event, val_list* args) override { } - void GotFunctionCall(const char* name, double time, - Func* func, val_list* args) override { } - void GotStateAccess(StateAccess* s) override { delete s; } - void GotTimer(Timer* t) override { } - void GotConnection(Connection* c) override { } - void GotPacket(Packet* packet) override { } -}; - -// Write values/events to file or fd. -class FileSerializer : public Serializer { -public: - explicit FileSerializer(SerializationFormat* format = 0); - ~FileSerializer() override; - - // Opens the file for serialization. - bool Open(const char* file, bool pure = false); - bool Close(); - - // Reads the file. - bool Read(UnserialInfo* info, const char* file, bool header = true); - -protected: - void ReportError(const char* msg) override; - void GotID(ID* id, Val* val) override; - void GotEvent(const char* name, double time, - EventHandlerPtr event, val_list* args) override; - void GotFunctionCall(const char* name, double time, - Func* func, val_list* args) override; - void GotStateAccess(StateAccess* s) override; - void GotTimer(Timer* t) override; - void GotConnection(Connection* c) override; - void GotPacket(Packet* packet) override; - - bool OpenFile(const char* file, bool readonly, bool should_exist = false); - void CloseFile(); - bool ReadFile(const char* file); - bool PrepareForWriting(); - bool ReadHeader(UnserialInfo* info = 0); - - SerializationCache cache; - const char* file; - int fd; -}; - -// Abstract interface class for external sources providing a stream of events. -class EventSource { -public: - virtual ~EventSource() { } - - // Returns time of the oldest event (0 if none available). - virtual double NextTimestamp(double* local_network_time) = 0; - - // Dispatches the oldest event and removes it. - virtual void DispatchNextEvent() = 0; - - // Returns true if there are more events to expect from this source. - virtual bool IsActive() = 0; -}; - -// Plays a file of events back. -class EventPlayer : public FileSerializer, public iosource::IOSource { -public: - explicit EventPlayer(const char* file); - ~EventPlayer() override; - - void GetFds(iosource::FD_Set* read, iosource::FD_Set* write, - iosource::FD_Set* except) override; - double NextTimestamp(double* local_network_time) override; - void Process() override; - const char* Tag() override { return "EventPlayer"; } - -protected: - void GotID(ID* id, Val* val) override {} - void GotEvent(const char* name, double time, - EventHandlerPtr event, val_list* args) override; - void GotFunctionCall(const char* name, double time, - Func* func, val_list* args) override; - - double stream_time; // time of first captured event - double replay_time; // network time of replay start - - // Next event waiting to be dispatched. - double ne_time; - EventHandlerPtr ne_handler; - val_list ne_args; - -}; - -extern FileSerializer* event_serializer; -extern FileSerializer* state_serializer; - -#endif diff --git a/src/StateAccess.cc b/src/StateAccess.cc index abcd7aad66..997eb5a48d 100644 --- a/src/StateAccess.cc +++ b/src/StateAccess.cc @@ -1,6 +1,5 @@ #include "Val.h" #include "StateAccess.h" -#include "Serializer.h" #include "Event.h" #include "NetVar.h" #include "DebugLogger.h" @@ -72,7 +71,6 @@ StateAccess::StateAccess(Opcode arg_opcode, } StateAccess::StateAccess(const StateAccess& sa) -: SerialObj() { opcode = sa.opcode; target_type = sa.target_type; @@ -343,146 +341,6 @@ ID* StateAccess::Target() const return target_type == TYPE_ID ? target.id : target.val->UniqueID(); } -bool StateAccess::Serialize(SerialInfo* info) const - { - return SerialObj::Serialize(info); - } - -StateAccess* StateAccess::Unserialize(UnserialInfo* info) - { - StateAccess* sa = - (StateAccess*) SerialObj::Unserialize(info, SER_STATE_ACCESS); - return sa; - } - -IMPLEMENT_SERIAL(StateAccess, SER_STATE_ACCESS); - -bool StateAccess::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_STATE_ACCESS, SerialObj); - - if ( ! SERIALIZE(char(opcode)) ) - return false; - - const ID* id = - target_type == TYPE_ID ? target.id : target.val->UniqueID(); - - if ( ! SERIALIZE(id->Name()) ) - return false; - - if ( op1_type == TYPE_KEY ) - { - Val* index = - id->ID_Val()->AsTableVal()->RecoverIndex(this->op1.key); - - if ( ! index ) - return false; - if ( ! index->Serialize(info) ) - return false; - - Unref(index); - } - - else if ( ! op1.val->Serialize(info) ) - return false; - - // Don't send the "old" operand if we don't want consistency checks. - // Unfortunately, it depends on the opcode which operand that actually - // is. - - const Val* null = 0; - - switch ( opcode ) { - case OP_PRINT: - case OP_EXPIRE: - case OP_READ_IDX: - // No old. - SERIALIZE_OPTIONAL(null); - SERIALIZE_OPTIONAL(null); - break; - - case OP_INCR: - case OP_INCR_IDX: - // Always need old. - SERIALIZE_OPTIONAL(op2); - SERIALIZE_OPTIONAL(op3); - break; - - case OP_ASSIGN: - case OP_ADD: - case OP_DEL: - // Op2 is old. - SERIALIZE_OPTIONAL(null); - SERIALIZE_OPTIONAL(null); - break; - - case OP_ASSIGN_IDX: - // Op3 is old. - SERIALIZE_OPTIONAL(op2); - SERIALIZE_OPTIONAL(null); - break; - - default: - reporter->InternalError("StateAccess::DoSerialize: unknown opcode"); - } - - return true; - } - -bool StateAccess::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(SerialObj); - - char c; - if ( ! UNSERIALIZE(&c) ) - return false; - - opcode = Opcode(c); - - const char* name; - if ( ! UNSERIALIZE_STR(&name, 0) ) - return false; - - target_type = TYPE_ID; - target.id = global_scope()->Lookup(name); - - if ( target.id ) - // Otherwise, we'll delete it below. - delete [] name; - - op1_type = TYPE_VAL; - op1.val = Val::Unserialize(info); - if ( ! op1.val ) - return false; - - UNSERIALIZE_OPTIONAL(op2, Val::Unserialize(info)); - UNSERIALIZE_OPTIONAL(op3, Val::Unserialize(info)); - - if ( target.id ) - Ref(target.id); - else - { - // This may happen as long as we haven't agreed on the - // unique name for an ID during initial synchronization, or if - // the local peer has already deleted the ID. - DBG_LOG(DBG_STATE, "state access referenced unknown id %s", name); - - if ( info->install_uniques ) - { - target.id = new ID(name, SCOPE_GLOBAL, true); - Ref(target.id); - global_scope()->Insert(name, target.id); -#ifdef USE_PERFTOOLS_DEBUG - heap_checker->IgnoreObject(target.id); -#endif - } - - delete [] name; - } - - return true; - } - void StateAccess::Describe(ODesc* d) const { const ID* id; diff --git a/src/StateAccess.h b/src/StateAccess.h index 1ef1e2b8da..15e2ae4676 100644 --- a/src/StateAccess.h +++ b/src/StateAccess.h @@ -7,14 +7,11 @@ #include #include -#include "SerialObj.h" - class Val; class ID; class MutableVal; class HashKey; class ODesc; -class Serializer; class TableVal; enum Opcode { // Op1 Op2 Op3 (Vals) @@ -30,7 +27,7 @@ enum Opcode { // Op1 Op2 Op3 (Vals) OP_READ_IDX, // idx }; -class StateAccess : public SerialObj { +class StateAccess { public: StateAccess(Opcode opcode, const ID* target, const Val* op1, const Val* op2 = 0, const Val* op3 = 0); @@ -48,7 +45,7 @@ public: StateAccess(const StateAccess& sa); - ~StateAccess() override; + virtual ~StateAccess(); // Replays this access in the our environment. void Replay(); @@ -58,9 +55,6 @@ public: void Describe(ODesc* d) const; - bool Serialize(SerialInfo* info) const; - static StateAccess* Unserialize(UnserialInfo* info); - // Main entry point when StateAcesses are performed. // For every state-changing operation, this has to be called. static void Log(StateAccess* access); @@ -74,8 +68,6 @@ private: StateAccess() { target.id = 0; op1.val = op2 = op3 = 0; } void RefThem(); - DECLARE_SERIAL(StateAccess); - Opcode opcode; union { ID* id; diff --git a/src/Stmt.cc b/src/Stmt.cc index 5960747d05..58bca4fc5b 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -117,47 +117,6 @@ void Stmt::AccessStats(ODesc* d) const } } -bool Stmt::Serialize(SerialInfo* info) const - { - return SerialObj::Serialize(info); - } - -Stmt* Stmt::Unserialize(UnserialInfo* info, BroStmtTag want) - { - Stmt* stmt = (Stmt*) SerialObj::Unserialize(info, SER_STMT); - - if ( want != STMT_ANY && stmt->tag != want ) - { - info->s->Error("wrong stmt type"); - Unref(stmt); - return 0; - } - - return stmt; - } - -bool Stmt::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_STMT, BroObj); - - return SERIALIZE(char(tag)) && SERIALIZE(last_access) - && SERIALIZE(access_count); - } - -bool Stmt::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroObj); - - char c; - if ( ! UNSERIALIZE(&c) ) - return 0; - - tag = BroStmtTag(c); - - return UNSERIALIZE(&last_access) && UNSERIALIZE(&access_count); - } - - ExprListStmt::ExprListStmt(BroStmtTag t, ListExpr* arg_l) : Stmt(t) { @@ -207,19 +166,6 @@ void ExprListStmt::PrintVals(ODesc* d, val_list* vals, int offset) const describe_vals(vals, d, offset); } -bool ExprListStmt::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_EXPR_LIST_STMT, Stmt); - return l->Serialize(info); - } - -bool ExprListStmt::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Stmt); - l = (ListExpr*) Expr::Unserialize(info, EXPR_LIST); - return l != 0; - } - TraversalCode ExprListStmt::Traverse(TraversalCallback* cb) const { TraversalCode tc = cb->PreStmt(this); @@ -305,20 +251,6 @@ Val* PrintStmt::DoExec(val_list* vals, stmt_flow_type& /* flow */) const return 0; } -IMPLEMENT_SERIAL(PrintStmt, SER_PRINT_STMT); - -bool PrintStmt::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_PRINT_STMT, ExprListStmt); - return true; - } - -bool PrintStmt::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(ExprListStmt); - return true; - } - ExprStmt::ExprStmt(Expr* arg_e) : Stmt(STMT_EXPR) { e = arg_e; @@ -404,22 +336,6 @@ TraversalCode ExprStmt::Traverse(TraversalCallback* cb) const HANDLE_TC_STMT_POST(tc); } -IMPLEMENT_SERIAL(ExprStmt, SER_EXPR_STMT); - -bool ExprStmt::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_EXPR_STMT, Stmt); - SERIALIZE_OPTIONAL(e); - return true; - } - -bool ExprStmt::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Stmt); - UNSERIALIZE_OPTIONAL(e, Expr::Unserialize(info)); - return true; - } - IfStmt::IfStmt(Expr* test, Stmt* arg_s1, Stmt* arg_s2) : ExprStmt(STMT_IF, test) { s1 = arg_s1; @@ -507,25 +423,6 @@ TraversalCode IfStmt::Traverse(TraversalCallback* cb) const HANDLE_TC_STMT_POST(tc); } -IMPLEMENT_SERIAL(IfStmt, SER_IF_STMT); - -bool IfStmt::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_IF_STMT, ExprStmt); - return s1->Serialize(info) && s2->Serialize(info); - } - -bool IfStmt::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(ExprStmt); - s1 = Stmt::Unserialize(info); - if ( ! s1 ) - return false; - - s2 = Stmt::Unserialize(info); - return s2 != 0; - } - static BroStmtTag get_last_stmt_tag(const Stmt* stmt) { if ( ! stmt ) @@ -655,67 +552,6 @@ TraversalCode Case::Traverse(TraversalCallback* cb) const return TC_CONTINUE; } -bool Case::Serialize(SerialInfo* info) const - { - return SerialObj::Serialize(info); - } - -Case* Case::Unserialize(UnserialInfo* info) - { - return (Case*) SerialObj::Unserialize(info, SER_CASE); - } - -IMPLEMENT_SERIAL(Case, SER_CASE); - -bool Case::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_CASE, BroObj); - - if ( ! expr_cases->Serialize(info) ) - return false; - - id_list empty; - id_list* types = (type_cases ? type_cases : &empty); - - if ( ! SERIALIZE(types->length()) ) - return false; - - loop_over_list((*types), i) - { - if ( ! (*types)[i]->Serialize(info) ) - return false; - } - - return this->s->Serialize(info); - } - -bool Case::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroObj); - - expr_cases = (ListExpr*) Expr::Unserialize(info, EXPR_LIST); - if ( ! expr_cases ) - return false; - - int len; - if ( ! UNSERIALIZE(&len) ) - return false; - - type_cases = new id_list(len); - - while ( len-- ) - { - ID* id = ID::Unserialize(info); - if ( ! id ) - return false; - - type_cases->append(id); - } - - this->s = Stmt::Unserialize(info); - return this->s != 0; - } - static void int_del_func(void* v) { delete (int*) v; @@ -1028,66 +864,6 @@ TraversalCode SwitchStmt::Traverse(TraversalCallback* cb) const HANDLE_TC_STMT_POST(tc); } -IMPLEMENT_SERIAL(SwitchStmt, SER_SWITCH_STMT); - -bool SwitchStmt::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_SWITCH_STMT, ExprStmt); - - if ( ! SERIALIZE(cases->length()) ) - return false; - - loop_over_list((*cases), i) - if ( ! (*cases)[i]->Serialize(info) ) - return false; - - if ( ! SERIALIZE(default_case_idx) ) - return false; - - return true; - } - -bool SwitchStmt::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(ExprStmt); - - Init(); - - int len; - if ( ! UNSERIALIZE(&len) ) - return false; - - while ( len-- ) - { - Case* c = Case::Unserialize(info); - if ( ! c ) - return false; - - cases->append(c); - } - - if ( ! UNSERIALIZE(&default_case_idx) ) - return false; - - loop_over_list(*cases, i) - { - const ListExpr* le = (*cases)[i]->ExprCases(); - - if ( ! le ) - continue; - - const expr_list& exprs = le->Exprs(); - - loop_over_list(exprs, j) - { - if ( ! AddCaseLabelValueMapping(exprs[j]->ExprVal(), i) ) - return false; - } - } - - return true; - } - AddStmt::AddStmt(Expr* arg_e) : ExprStmt(STMT_ADD, arg_e) { if ( ! e->CanAdd() ) @@ -1121,20 +897,6 @@ TraversalCode AddStmt::Traverse(TraversalCallback* cb) const HANDLE_TC_STMT_POST(tc); } -IMPLEMENT_SERIAL(AddStmt, SER_ADD_STMT); - -bool AddStmt::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_ADD_STMT, ExprStmt); - return true; - } - -bool AddStmt::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(ExprStmt); - return true; - } - DelStmt::DelStmt(Expr* arg_e) : ExprStmt(STMT_DELETE, arg_e) { if ( e->IsError() ) @@ -1170,20 +932,6 @@ TraversalCode DelStmt::Traverse(TraversalCallback* cb) const HANDLE_TC_STMT_POST(tc); } -IMPLEMENT_SERIAL(DelStmt, SER_DEL_STMT); - -bool DelStmt::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_DEL_STMT, ExprStmt); - return true; - } - -bool DelStmt::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(ExprStmt); - return true; - } - EventStmt::EventStmt(EventExpr* arg_e) : ExprStmt(STMT_EVENT, arg_e) { event_expr = arg_e; @@ -1218,22 +966,6 @@ TraversalCode EventStmt::Traverse(TraversalCallback* cb) const HANDLE_TC_STMT_POST(tc); } -IMPLEMENT_SERIAL(EventStmt, SER_EVENT_STMT); - -bool EventStmt::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_EVENT_STMT, ExprStmt); - return event_expr->Serialize(info); - } - -bool EventStmt::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(ExprStmt); - - event_expr = (EventExpr*) Expr::Unserialize(info, EXPR_EVENT); - return event_expr != 0; - } - WhileStmt::WhileStmt(Expr* arg_loop_condition, Stmt* arg_body) : loop_condition(arg_loop_condition), body(arg_body) { @@ -1319,30 +1051,6 @@ Val* WhileStmt::Exec(Frame* f, stmt_flow_type& flow) const return rval; } -IMPLEMENT_SERIAL(WhileStmt, SER_WHILE_STMT); - -bool WhileStmt::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_WHILE_STMT, Stmt); - - if ( ! loop_condition->Serialize(info) ) - return false; - - return body->Serialize(info); - } - -bool WhileStmt::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Stmt); - loop_condition = Expr::Unserialize(info); - - if ( ! loop_condition ) - return false; - - body = Stmt::Unserialize(info); - return body != 0; - } - ForStmt::ForStmt(id_list* arg_loop_vars, Expr* loop_expr) : ExprStmt(STMT_FOR, loop_expr) { @@ -1607,47 +1315,6 @@ TraversalCode ForStmt::Traverse(TraversalCallback* cb) const HANDLE_TC_STMT_POST(tc); } -IMPLEMENT_SERIAL(ForStmt, SER_FOR_STMT); - -bool ForStmt::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_FOR_STMT, ExprStmt); - - if ( ! SERIALIZE(loop_vars->length()) ) - return false; - - loop_over_list((*loop_vars), i) - { - if ( ! (*loop_vars)[i]->Serialize(info) ) - return false; - } - - return body->Serialize(info); - } - -bool ForStmt::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(ExprStmt); - - int len; - if ( ! UNSERIALIZE(&len) ) - return false; - - loop_vars = new id_list(len); - - while ( len-- ) - { - ID* id = ID::Unserialize(info); - if ( ! id ) - return false; - - loop_vars->append(id); - } - - body = Stmt::Unserialize(info); - return body != 0; - } - Val* NextStmt::Exec(Frame* /* f */, stmt_flow_type& flow) const { RegisterAccess(); @@ -1675,20 +1342,6 @@ TraversalCode NextStmt::Traverse(TraversalCallback* cb) const HANDLE_TC_STMT_POST(tc); } -IMPLEMENT_SERIAL(NextStmt, SER_NEXT_STMT); - -bool NextStmt::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_NEXT_STMT, Stmt); - return true; - } - -bool NextStmt::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Stmt); - return true; - } - Val* BreakStmt::Exec(Frame* /* f */, stmt_flow_type& flow) const { RegisterAccess(); @@ -1716,20 +1369,6 @@ TraversalCode BreakStmt::Traverse(TraversalCallback* cb) const HANDLE_TC_STMT_POST(tc); } -IMPLEMENT_SERIAL(BreakStmt, SER_BREAK_STMT); - -bool BreakStmt::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_BREAK_STMT, Stmt); - return true; - } - -bool BreakStmt::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Stmt); - return true; - } - Val* FallthroughStmt::Exec(Frame* /* f */, stmt_flow_type& flow) const { RegisterAccess(); @@ -1757,20 +1396,6 @@ TraversalCode FallthroughStmt::Traverse(TraversalCallback* cb) const HANDLE_TC_STMT_POST(tc); } -IMPLEMENT_SERIAL(FallthroughStmt, SER_FALLTHROUGH_STMT); - -bool FallthroughStmt::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_FALLTHROUGH_STMT, Stmt); - return true; - } - -bool FallthroughStmt::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Stmt); - return true; - } - ReturnStmt::ReturnStmt(Expr* arg_e) : ExprStmt(STMT_RETURN, arg_e) { Scope* s = current_scope(); @@ -1838,20 +1463,6 @@ void ReturnStmt::Describe(ODesc* d) const DescribeDone(d); } -IMPLEMENT_SERIAL(ReturnStmt, SER_RETURN_STMT); - -bool ReturnStmt::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_RETURN_STMT, ExprStmt); - return true; - } - -bool ReturnStmt::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(ExprStmt); - return true; - } - StmtList::StmtList() : Stmt(STMT_LIST) { } @@ -1941,43 +1552,6 @@ TraversalCode StmtList::Traverse(TraversalCallback* cb) const HANDLE_TC_STMT_POST(tc); } -IMPLEMENT_SERIAL(StmtList, SER_STMT_LIST); - -bool StmtList::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_STMT_LIST, Stmt); - - if ( ! SERIALIZE(stmts.length()) ) - return false; - - loop_over_list(stmts, i) - if ( ! stmts[i]->Serialize(info) ) - return false; - - return true; - } - -bool StmtList::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Stmt); - - int len; - if ( ! UNSERIALIZE(&len) ) - return false; - - while ( len-- ) - { - Stmt* stmt = Stmt::Unserialize(info); - if ( ! stmt ) - return false; - - stmts.append(stmt); - } - - return true; - } - - Val* EventBodyList::Exec(Frame* f, stmt_flow_type& flow) const { RegisterAccess(); @@ -2036,20 +1610,6 @@ void EventBodyList::Describe(ODesc* d) const StmtList::Describe(d); } -IMPLEMENT_SERIAL(EventBodyList, SER_EVENT_BODY_LIST); - -bool EventBodyList::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_EVENT_BODY_LIST, StmtList); - return SERIALIZE(topmost); - } - -bool EventBodyList::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(StmtList); - return UNSERIALIZE(&topmost); - } - InitStmt::~InitStmt() { loop_over_list(*inits, i) @@ -2123,45 +1683,6 @@ TraversalCode InitStmt::Traverse(TraversalCallback* cb) const HANDLE_TC_STMT_POST(tc); } -IMPLEMENT_SERIAL(InitStmt, SER_INIT_STMT); - -bool InitStmt::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_INIT_STMT, Stmt); - - if ( ! SERIALIZE(inits->length()) ) - return false; - - loop_over_list((*inits), i) - { - if ( ! (*inits)[i]->Serialize(info) ) - return false; - } - - return true; - } - -bool InitStmt::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Stmt); - - int len; - if ( ! UNSERIALIZE(&len) ) - return false; - - inits = new id_list(len); - - while ( len-- ) - { - ID* id = ID::Unserialize(info); - if ( ! id ) - return false; - inits->append(id); - } - return true; - } - - Val* NullStmt::Exec(Frame* /* f */, stmt_flow_type& flow) const { RegisterAccess(); @@ -2191,20 +1712,6 @@ TraversalCode NullStmt::Traverse(TraversalCallback* cb) const HANDLE_TC_STMT_POST(tc); } -IMPLEMENT_SERIAL(NullStmt, SER_NULL_STMT); - -bool NullStmt::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_NULL_STMT, Stmt); - return true; - } - -bool NullStmt::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Stmt); - return true; - } - WhenStmt::WhenStmt(Expr* arg_cond, Stmt* arg_s1, Stmt* arg_s2, Expr* arg_timeout, bool arg_is_return) : Stmt(STMT_WHEN) @@ -2320,35 +1827,3 @@ TraversalCode WhenStmt::Traverse(TraversalCallback* cb) const HANDLE_TC_STMT_POST(tc); } -IMPLEMENT_SERIAL(WhenStmt, SER_WHEN_STMT); - -bool WhenStmt::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_WHEN_STMT, Stmt); - - if ( cond->Serialize(info) && s1->Serialize(info) ) - return false; - - SERIALIZE_OPTIONAL(s2); - SERIALIZE_OPTIONAL(timeout); - - return true; - } - -bool WhenStmt::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Stmt); - - cond = Expr::Unserialize(info); - if ( ! cond ) - return false; - - s1 = Stmt::Unserialize(info); - if ( ! s1 ) - return false; - - UNSERIALIZE_OPTIONAL(s2, Stmt::Unserialize(info)); - UNSERIALIZE_OPTIONAL(timeout, Expr::Unserialize(info)); - - return true; - } diff --git a/src/Stmt.h b/src/Stmt.h index c3ee6611fe..7136ffe306 100644 --- a/src/Stmt.h +++ b/src/Stmt.h @@ -71,9 +71,6 @@ public: virtual unsigned int BPCount() const { return breakpoint_count; } - bool Serialize(SerialInfo* info) const; - static Stmt* Unserialize(UnserialInfo* info, BroStmtTag want = STMT_ANY); - virtual TraversalCode Traverse(TraversalCallback* cb) const = 0; protected: @@ -83,8 +80,6 @@ protected: void AddTag(ODesc* d) const; void DescribeDone(ODesc* d) const; - DECLARE_ABSTRACT_SERIAL(Stmt); - BroStmtTag tag; int breakpoint_count; // how many breakpoints on this statement @@ -111,8 +106,6 @@ protected: void Describe(ODesc* d) const override; void PrintVals(ODesc* d, val_list* vals, int offset) const; - DECLARE_ABSTRACT_SERIAL(ExprListStmt); - ListExpr* l; }; @@ -125,8 +118,6 @@ protected: PrintStmt() {} Val* DoExec(val_list* vals, stmt_flow_type& flow) const override; - - DECLARE_SERIAL(PrintStmt); }; class ExprStmt : public Stmt { @@ -151,8 +142,6 @@ protected: int IsPure() const override; - DECLARE_SERIAL(ExprStmt); - Expr* e; }; @@ -175,8 +164,6 @@ protected: Val* DoExec(Frame* f, Val* v, stmt_flow_type& flow) const override; int IsPure() const override; - DECLARE_SERIAL(IfStmt); - Stmt* s1; Stmt* s2; }; @@ -197,17 +184,12 @@ public: void Describe(ODesc* d) const override; - bool Serialize(SerialInfo* info) const; - static Case* Unserialize(UnserialInfo* info); - TraversalCode Traverse(TraversalCallback* cb) const; protected: friend class Stmt; Case() { expr_cases = 0; type_cases = 0; s = 0; } - DECLARE_SERIAL(Case); - ListExpr* expr_cases; id_list* type_cases; Stmt* s; @@ -234,8 +216,6 @@ protected: Val* DoExec(Frame* f, Val* v, stmt_flow_type& flow) const override; int IsPure() const override; - DECLARE_SERIAL(SwitchStmt); - // Initialize composite hash and case label map. void Init(); @@ -274,8 +254,6 @@ public: protected: friend class Stmt; AddStmt() {} - - DECLARE_SERIAL(AddStmt); }; class DelStmt : public ExprStmt { @@ -290,8 +268,6 @@ public: protected: friend class Stmt; DelStmt() {} - - DECLARE_SERIAL(DelStmt); }; class EventStmt : public ExprStmt { @@ -306,8 +282,6 @@ protected: friend class Stmt; EventStmt() { event_expr = 0; } - DECLARE_SERIAL(EventStmt); - EventExpr* event_expr; }; @@ -331,8 +305,6 @@ protected: Val* Exec(Frame* f, stmt_flow_type& flow) const override; - DECLARE_SERIAL(WhileStmt); - Expr* loop_condition; Stmt* body; }; @@ -362,8 +334,6 @@ protected: Val* DoExec(Frame* f, Val* v, stmt_flow_type& flow) const override; - DECLARE_SERIAL(ForStmt); - id_list* loop_vars; Stmt* body; // Stores the value variable being used for a key value for loop. @@ -383,7 +353,6 @@ public: TraversalCode Traverse(TraversalCallback* cb) const override; protected: - DECLARE_SERIAL(NextStmt); }; class BreakStmt : public Stmt { @@ -398,7 +367,6 @@ public: TraversalCode Traverse(TraversalCallback* cb) const override; protected: - DECLARE_SERIAL(BreakStmt); }; class FallthroughStmt : public Stmt { @@ -413,7 +381,6 @@ public: TraversalCode Traverse(TraversalCallback* cb) const override; protected: - DECLARE_SERIAL(FallthroughStmt); }; class ReturnStmt : public ExprStmt { @@ -427,8 +394,6 @@ public: protected: friend class Stmt; ReturnStmt() {} - - DECLARE_SERIAL(ReturnStmt); }; class StmtList : public Stmt { @@ -448,8 +413,6 @@ public: protected: int IsPure() const override; - DECLARE_SERIAL(StmtList); - stmt_list stmts; }; @@ -467,9 +430,6 @@ public: // bool IsTopmost() { return topmost; } protected: - - DECLARE_SERIAL(EventBodyList); - bool topmost; }; @@ -496,8 +456,6 @@ protected: friend class Stmt; InitStmt() { inits = 0; } - DECLARE_SERIAL(InitStmt); - id_list* inits; }; @@ -511,9 +469,6 @@ public: void Describe(ODesc* d) const override; TraversalCode Traverse(TraversalCallback* cb) const override; - -protected: - DECLARE_SERIAL(NullStmt); }; class WhenStmt : public Stmt { @@ -537,8 +492,6 @@ public: protected: WhenStmt() { cond = 0; s1 = s2 = 0; timeout = 0; is_return = 0; } - DECLARE_SERIAL(WhenStmt); - Expr* cond; Stmt* s1; Stmt* s2; diff --git a/src/Timer.cc b/src/Timer.cc index 9041b822c8..1138deec79 100644 --- a/src/Timer.cc +++ b/src/Timer.cc @@ -5,7 +5,6 @@ #include "util.h" #include "Timer.h" #include "Desc.h" -#include "Serializer.h" #include "broker/Manager.h" // Names of timers in same order than in TimerType. @@ -53,41 +52,6 @@ void Timer::Describe(ODesc* d) const d->Add(Time()); } -bool Timer::Serialize(SerialInfo* info) const - { - return SerialObj::Serialize(info); - } - -Timer* Timer::Unserialize(UnserialInfo* info) - { - Timer* timer = (Timer*) SerialObj::Unserialize(info, SER_TIMER); - if ( ! timer ) - return 0; - - timer_mgr->Add(timer); - - return timer; - } - -bool Timer::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_TIMER, SerialObj); - char tmp = type; - return SERIALIZE(tmp) && SERIALIZE(time); - } - -bool Timer::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(SerialObj); - - char tmp; - if ( ! UNSERIALIZE(&tmp) ) - return false; - type = tmp; - - return UNSERIALIZE(&time); - } - unsigned int TimerMgr::current_timers[NUM_TIMER_TYPES]; TimerMgr::~TimerMgr() diff --git a/src/Timer.h b/src/Timer.h index 2ce9f56e0b..02ebb2773c 100644 --- a/src/Timer.h +++ b/src/Timer.h @@ -6,7 +6,6 @@ #include #include -#include "SerialObj.h" #include "PriorityQueue.h" extern "C" { @@ -49,10 +48,9 @@ const int NUM_TIMER_TYPES = int(TIMER_TIMERMGR_EXPIRE) + 1; extern const char* timer_type_to_string(TimerType type); -class Serializer; class ODesc; -class Timer : public SerialObj, public PQ_Element { +class Timer : public PQ_Element { public: Timer(double t, TimerType arg_type) : PQ_Element(t) { type = (char) arg_type; } @@ -67,14 +65,9 @@ public: void Describe(ODesc* d) const; - bool Serialize(SerialInfo* info) const; - static Timer* Unserialize(UnserialInfo* info); - protected: Timer() {} - DECLARE_ABSTRACT_SERIAL(Timer); - unsigned int type:8; }; diff --git a/src/Type.cc b/src/Type.cc index 094a974255..7b65dd0ffa 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -6,7 +6,6 @@ #include "Attr.h" #include "Expr.h" #include "Scope.h" -#include "Serializer.h" #include "Reporter.h" #include "zeekygen/Manager.h" #include "zeekygen/utils.h" @@ -122,27 +121,30 @@ BroType::BroType(TypeTag t, bool arg_base_type) } -BroType* BroType::Clone() const +BroType* BroType::ShallowClone() { - SerializationFormat* form = new BinarySerializationFormat(); - form->StartWrite(); - CloneSerializer ss(form); - SerialInfo sinfo(&ss); - sinfo.cache = false; + switch ( tag ) { + case TYPE_VOID: + case TYPE_BOOL: + case TYPE_INT: + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + case TYPE_STRING: + case TYPE_PATTERN: + case TYPE_TIMER: + case TYPE_PORT: + case TYPE_ADDR: + case TYPE_SUBNET: + case TYPE_ANY: + return new BroType(tag, base_type); - this->Serialize(&sinfo); - char* data; - uint32 len = form->EndWrite(&data); - form->StartRead(data, len); - - UnserialInfo uinfo(&ss); - uinfo.cache = false; - - BroType* rval = this->Unserialize(&uinfo, false); - assert(rval != this); - - free(data); - return rval; + default: + reporter->InternalError("cloning illegal base BroType"); + } + return nullptr; } int BroType::MatchesIndex(ListExpr*& index) const @@ -203,124 +205,6 @@ unsigned int BroType::MemoryAllocation() const return padded_sizeof(*this); } -bool BroType::Serialize(SerialInfo* info) const - { - // We always send full types (see below). - if ( ! SERIALIZE(true) ) - return false; - - bool ret = SerialObj::Serialize(info); - return ret; - } - -BroType* BroType::Unserialize(UnserialInfo* info, bool use_existing) - { - // To avoid external Broccoli clients needing to always send full type - // objects, we allow them to give us only the name of a type. To - // differentiate between the two cases, we exchange a flag first. - bool full_type = true;; - if ( ! UNSERIALIZE(&full_type) ) - return 0; - - if ( ! full_type ) - { - const char* name; - if ( ! UNSERIALIZE_STR(&name, 0) ) - return 0; - - ID* id = global_scope()->Lookup(name); - if ( ! id ) - { - info->s->Error(fmt("unknown type %s", name)); - return 0; - } - - BroType* t = id->AsType(); - if ( ! t ) - { - info->s->Error(fmt("%s is not a type", name)); - return 0; - } - - return t->Ref(); - } - - BroType* t = (BroType*) SerialObj::Unserialize(info, SER_BRO_TYPE); - - if ( ! t || ! use_existing ) - return t; - - if ( ! t->name.empty() ) - { - // Avoid creating a new type if it's known by name. - // Also avoids loss of base type name alias (from condition below). - ID* id = global_scope()->Lookup(t->name.c_str()); - BroType* t2 = id ? id->AsType() : 0; - - if ( t2 ) - { - Unref(t); - return t2->Ref(); - } - } - - if ( t->base_type ) - { - BroType* t2 = ::base_type(TypeTag(t->tag)); - Unref(t); - assert(t2); - return t2; - } - - assert(t); - return t; - } - -IMPLEMENT_SERIAL(BroType, SER_BRO_TYPE) - -bool BroType::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_BRO_TYPE, BroObj); - - info->s->WriteOpenTag("Type"); - - if ( ! (SERIALIZE(char(tag)) && SERIALIZE(char(internal_tag))) ) - return false; - - if ( ! (SERIALIZE(is_network_order) && SERIALIZE(base_type)) ) - return false; - - SERIALIZE_STR(name.c_str(), name.size()); - - info->s->WriteCloseTag("Type"); - - return true; - } - -bool BroType::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroObj); - - char c1, c2; - if ( ! (UNSERIALIZE(&c1) && UNSERIALIZE(&c2) ) ) - return 0; - - tag = (TypeTag) c1; - internal_tag = (InternalTypeTag) c2; - - if ( ! (UNSERIALIZE(&is_network_order) && UNSERIALIZE(&base_type)) ) - return 0; - - const char* n; - if ( ! UNSERIALIZE_STR(&n, 0) ) - return false; - - name = n; - delete [] n; - - return true; - } - TypeList::~TypeList() { loop_over_list(types, i) @@ -383,47 +267,6 @@ void TypeList::Describe(ODesc* d) const } } -IMPLEMENT_SERIAL(TypeList, SER_TYPE_LIST); - -bool TypeList::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_TYPE_LIST, BroType); - - SERIALIZE_OPTIONAL(pure_type); - - if ( ! SERIALIZE(types.length()) ) - return false; - - loop_over_list(types, j) - { - if ( ! types[j]->Serialize(info) ) - return false; - } - - return true; - } - -bool TypeList::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroType); - - UNSERIALIZE_OPTIONAL(pure_type, BroType::Unserialize(info)); - - int len; - if ( ! UNSERIALIZE(&len) ) - return false; - - while ( len-- ) - { - BroType* t = BroType::Unserialize(info); - if ( ! t ) - return false; - - types.append(t); - } - return true; - } - IndexType::~IndexType() { Unref(indices); @@ -530,25 +373,6 @@ bool IndexType::IsSubNetIndex() const return false; } -IMPLEMENT_SERIAL(IndexType, SER_INDEX_TYPE); - -bool IndexType::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_INDEX_TYPE, BroType); - - SERIALIZE_OPTIONAL(yield_type); - return indices->Serialize(info); - } - -bool IndexType::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroType); - - UNSERIALIZE_OPTIONAL(yield_type, BroType::Unserialize(info)); - indices = (TypeList*) BroType::Unserialize(info); - return indices != 0; - } - TableType::TableType(TypeList* ind, BroType* yield) : IndexType(TYPE_TABLE, ind, yield) { @@ -577,6 +401,16 @@ TableType::TableType(TypeList* ind, BroType* yield) } } +TableType* TableType::ShallowClone() + { + if ( indices ) + indices->Ref(); + if ( yield_type ) + yield_type->Ref(); + + return new TableType(indices, yield_type); + } + bool TableType::IsUnspecifiedTable() const { // Unspecified types have an empty list of indices. @@ -650,18 +484,17 @@ SetType::SetType(TypeList* ind, ListExpr* arg_elements) : TableType(ind, 0) } } -IMPLEMENT_SERIAL(TableType, SER_TABLE_TYPE); - -bool TableType::DoSerialize(SerialInfo* info) const +SetType* SetType::ShallowClone() { - DO_SERIALIZE(SER_TABLE_TYPE, IndexType); - return true; - } + // constructor only consumes indices when elements + // is set + if ( elements && indices ) + { + elements->Ref(); + indices->Ref(); + } -bool TableType::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(IndexType); - return true; + return new SetType(indices, elements); } SetType::~SetType() @@ -669,24 +502,6 @@ SetType::~SetType() Unref(elements); } -IMPLEMENT_SERIAL(SetType, SER_SET_TYPE); - -bool SetType::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_SET_TYPE, TableType); - - SERIALIZE_OPTIONAL(elements); - return true; - } - -bool SetType::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(TableType); - - UNSERIALIZE_OPTIONAL(elements, (ListExpr*) Expr::Unserialize(info, EXPR_LIST)); - return true; - } - FuncType::FuncType(RecordType* arg_args, BroType* arg_yield, function_flavor arg_flavor) : BroType(TYPE_FUNC) { @@ -716,6 +531,16 @@ FuncType::FuncType(RecordType* arg_args, BroType* arg_yield, function_flavor arg } } +FuncType* FuncType::ShallowClone() + { + auto f = new FuncType(); + f->args = args->Ref()->AsRecordType(); + f->arg_types = arg_types->Ref()->AsTypeList(); + f->yield = yield->Ref(); + f->flavor = flavor; + return f; + } + string FuncType::FlavorString() const { switch ( flavor ) { @@ -822,80 +647,6 @@ void FuncType::DescribeReST(ODesc* d, bool roles_only) const } } -IMPLEMENT_SERIAL(FuncType, SER_FUNC_TYPE); - -bool FuncType::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_FUNC_TYPE, BroType); - - assert(args); - assert(arg_types); - - SERIALIZE_OPTIONAL(yield); - - int ser_flavor = 0; - - switch ( flavor ) { - - case FUNC_FLAVOR_FUNCTION: - ser_flavor = 0; - break; - - case FUNC_FLAVOR_EVENT: - ser_flavor = 1; - break; - - case FUNC_FLAVOR_HOOK: - ser_flavor = 2; - break; - - default: - reporter->InternalError("Invalid function flavor serialization"); - break; - } - - return args->Serialize(info) && - arg_types->Serialize(info) && - SERIALIZE(ser_flavor); - } - -bool FuncType::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroType); - - UNSERIALIZE_OPTIONAL(yield, BroType::Unserialize(info)); - - args = (RecordType*) BroType::Unserialize(info); - if ( ! args ) - return false; - - arg_types = (TypeList*) BroType::Unserialize(info); - if ( ! arg_types ) - return false; - - int ser_flavor = 0; - - if ( ! UNSERIALIZE(&ser_flavor) ) - return false; - - switch ( ser_flavor ) { - case 0: - flavor = FUNC_FLAVOR_FUNCTION; - break; - case 1: - flavor = FUNC_FLAVOR_EVENT; - break; - case 2: - flavor = FUNC_FLAVOR_HOOK; - break; - default: - reporter->InternalError("Invalid function flavor unserialization"); - break; - } - - return true; - } - TypeDecl::TypeDecl(BroType* t, const char* i, attr_list* arg_attrs, bool in_record) { type = t; @@ -921,35 +672,6 @@ TypeDecl::~TypeDecl() delete [] id; } -bool TypeDecl::Serialize(SerialInfo* info) const - { - assert(type); - assert(id); - - SERIALIZE_OPTIONAL(attrs); - - if ( ! (type->Serialize(info) && SERIALIZE(id)) ) - return false; - - return true; - } - -TypeDecl* TypeDecl::Unserialize(UnserialInfo* info) - { - TypeDecl* t = new TypeDecl(0, 0, 0); - - UNSERIALIZE_OPTIONAL_STATIC(t->attrs, Attributes::Unserialize(info), t); - t->type = BroType::Unserialize(info); - - if ( ! (t->type && UNSERIALIZE_STR(&t->id, 0)) ) - { - delete t; - return 0; - } - - return t; - } - void TypeDecl::DescribeReST(ODesc* d, bool roles_only) const { d->Add(id); @@ -977,6 +699,16 @@ RecordType::RecordType(type_decl_list* arg_types) : BroType(TYPE_RECORD) num_fields = types ? types->length() : 0; } +// in this case the clone is actually not so shallow, since +// it gets modified by everyone. +RecordType* RecordType::ShallowClone() + { + auto pass = new type_decl_list(); + loop_over_list(*types, i) + pass->append(new TypeDecl(*(*types)[i])); + return new RecordType(pass); + } + RecordType::~RecordType() { if ( types ) @@ -1253,67 +985,6 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const d->PopIndentNoNL(); } -IMPLEMENT_SERIAL(RecordType, SER_RECORD_TYPE) - -bool RecordType::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_RECORD_TYPE, BroType); - - if ( ! SERIALIZE(num_fields) ) - return false; - - if ( types ) - { - if ( ! (SERIALIZE(true) && SERIALIZE(types->length())) ) - return false; - - loop_over_list(*types, i) - { - if ( ! (*types)[i]->Serialize(info) ) - return false; - } - } - - else if ( ! SERIALIZE(false) ) - return false; - - return true; - } - -bool RecordType::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroType); - - if ( ! UNSERIALIZE(&num_fields) ) - return false; - - bool has_it; - if ( ! UNSERIALIZE(&has_it) ) - return false; - - if ( has_it ) - { - int len; - if ( ! UNSERIALIZE(&len) ) - return false; - - types = new type_decl_list(len); - - while ( len-- ) - { - TypeDecl* t = TypeDecl::Unserialize(info); - if ( ! t ) - return false; - - types->append(t); - } - } - else - types = 0; - - return true; - } - SubNetType::SubNetType() : BroType(TYPE_SUBNET) { } @@ -1326,20 +997,6 @@ void SubNetType::Describe(ODesc* d) const d->Add(int(Tag())); } -IMPLEMENT_SERIAL(SubNetType, SER_SUBNET_TYPE); - -bool SubNetType::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_SUBNET_TYPE, BroType); - return true; - } - -bool SubNetType::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroType); - return true; - } - FileType::FileType(BroType* yield_type) : BroType(TYPE_FILE) { @@ -1370,24 +1027,6 @@ void FileType::Describe(ODesc* d) const } } -IMPLEMENT_SERIAL(FileType, SER_FILE_TYPE); - -bool FileType::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_FILE_TYPE, BroType); - - assert(yield); - return yield->Serialize(info); - } - -bool FileType::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroType); - - yield = BroType::Unserialize(info); - return yield != 0; - } - OpaqueType::OpaqueType(const string& arg_name) : BroType(TYPE_OPAQUE) { name = arg_name; @@ -1408,28 +1047,6 @@ void OpaqueType::DescribeReST(ODesc* d, bool roles_only) const d->Add(fmt(":zeek:type:`%s` of %s", type_name(Tag()), name.c_str())); } -IMPLEMENT_SERIAL(OpaqueType, SER_OPAQUE_TYPE); - -bool OpaqueType::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_OPAQUE_TYPE, BroType); - return SERIALIZE_STR(name.c_str(), name.size()); - } - -bool OpaqueType::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroType); - - const char* n; - if ( ! UNSERIALIZE_STR(&n, 0) ) - return false; - - name = n; - delete [] n; - - return true; - } - EnumType::EnumType(const string& name) : BroType(TYPE_ENUM) { @@ -1437,18 +1054,26 @@ EnumType::EnumType(const string& name) SetName(name); } -EnumType::EnumType(EnumType* e) +EnumType::EnumType(const EnumType* e) : BroType(TYPE_ENUM) { counter = e->counter; SetName(e->GetName()); - for ( NameMap::iterator it = e->names.begin(); it != e->names.end(); ++it ) + for ( auto it = e->names.begin(); it != e->names.end(); ++it ) names[it->first] = it->second; vals = e->vals; } +EnumType* EnumType::ShallowClone() + { + if ( counter == 0 ) + return new EnumType(GetName()); + + return new EnumType(this); + } + EnumType::~EnumType() { for ( auto& kv : vals ) @@ -1672,64 +1297,16 @@ void EnumType::DescribeReST(ODesc* d, bool roles_only) const } } -IMPLEMENT_SERIAL(EnumType, SER_ENUM_TYPE); - -bool EnumType::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_ENUM_TYPE, BroType); - - if ( ! (SERIALIZE(counter) && SERIALIZE((unsigned int) names.size()) && - // Dummy boolean for backwards compatibility. - SERIALIZE(false)) ) - return false; - - for ( NameMap::const_iterator iter = names.begin(); - iter != names.end(); ++iter ) - { - if ( ! SERIALIZE(iter->first) || ! SERIALIZE(iter->second) ) - return false; - } - - return true; - } - -bool EnumType::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroType); - - unsigned int len; - bool dummy; - if ( ! UNSERIALIZE(&counter) || - ! UNSERIALIZE(&len) || - // Dummy boolean for backwards compatibility. - ! UNSERIALIZE(&dummy) ) - return false; - - while ( len-- ) - { - const char* name; - bro_int_t val; - if ( ! (UNSERIALIZE_STR(&name, 0) && UNSERIALIZE(&val)) ) - return false; - - names[name] = val; - delete [] name; // names[name] converts to std::string - // note: the 'vals' map gets populated lazily, which works fine and - // also happens to avoid a leak due to circular reference between the - // types and vals (there's a special case for unserializing a known - // type that will unserialze and then immediately want to unref the - // type if we already have it, except that won't delete it as intended - // if we've already created circular references to it here). - } - - return true; - } - VectorType::VectorType(BroType* element_type) : BroType(TYPE_VECTOR), yield_type(element_type) { } +VectorType* VectorType::ShallowClone() + { + return new VectorType(yield_type); + } + VectorType::~VectorType() { Unref(yield_type); @@ -1793,21 +1370,6 @@ bool VectorType::IsUnspecifiedVector() const return yield_type->Tag() == TYPE_VOID; } -IMPLEMENT_SERIAL(VectorType, SER_VECTOR_TYPE); - -bool VectorType::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_VECTOR_TYPE, BroType); - return yield_type->Serialize(info); - } - -bool VectorType::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroType); - yield_type = BroType::Unserialize(info); - return yield_type != 0; - } - void VectorType::Describe(ODesc* d) const { if ( d->IsReadable() ) diff --git a/src/Type.h b/src/Type.h index c537bb6203..043ec5c928 100644 --- a/src/Type.h +++ b/src/Type.h @@ -72,7 +72,6 @@ class SubNetType; class FuncType; class ListExpr; class EnumType; -class Serializer; class VectorType; class TypeType; class OpaqueType; @@ -87,7 +86,15 @@ public: explicit BroType(TypeTag tag, bool base_type = false); ~BroType() override { } - BroType* Clone() const; + // Performs a shallow clone operation of the Bro type. + // This especially means that especially for tables the types + // are not recursively cloned; altering one type will in this case + // alter one of them. + // The main use for this is alias tracking. + // Clone operations will mostly be implemented in the derived classes; + // in addition cloning will be limited to classes that can be reached by + // the script-level. + virtual BroType* ShallowClone(); TypeTag Tag() const { return tag; } InternalTypeTag InternalType() const { return internal_tag; } @@ -108,7 +115,7 @@ public: // this type is a table[string] of port, then returns the "port" // type. Returns nil if this is not an index type. virtual BroType* YieldType(); - const BroType* YieldType() const + virtual const BroType* YieldType() const { return ((BroType*) this)->YieldType(); } // Returns true if this type is a record and contains the @@ -256,9 +263,6 @@ public: virtual unsigned MemoryAllocation() const; - bool Serialize(SerialInfo* info) const; - static BroType* Unserialize(UnserialInfo* info, bool use_existing = true); - void SetName(const string& arg_name) { name = arg_name; } string GetName() const { return name; } @@ -275,8 +279,6 @@ protected: void SetError(); - DECLARE_SERIAL(BroType) - private: TypeTag tag; InternalTypeTag internal_tag; @@ -325,8 +327,6 @@ public: } protected: - DECLARE_SERIAL(TypeList) - BroType* pure_type; type_list types; }; @@ -338,7 +338,7 @@ public: TypeList* Indices() const { return indices; } const type_list* IndexTypes() const { return indices->Types(); } BroType* YieldType() override; - const BroType* YieldType() const; + const BroType* YieldType() const override; void Describe(ODesc* d) const override; void DescribeReST(ODesc* d, bool roles_only = false) const override; @@ -356,8 +356,6 @@ protected: } ~IndexType() override; - DECLARE_SERIAL(IndexType) - TypeList* indices; BroType* yield_type; }; @@ -366,6 +364,8 @@ class TableType : public IndexType { public: TableType(TypeList* ind, BroType* yield); + TableType* ShallowClone() override; + // Returns true if this table type is "unspecified", which is // what one gets using an empty "set()" or "table()" constructor. bool IsUnspecifiedTable() const; @@ -374,8 +374,6 @@ protected: TableType() {} TypeList* ExpandRecordIndex(RecordType* rt) const; - - DECLARE_SERIAL(TableType) }; class SetType : public TableType { @@ -383,25 +381,26 @@ public: SetType(TypeList* ind, ListExpr* arg_elements); ~SetType() override; + SetType* ShallowClone() override; + ListExpr* SetElements() const { return elements; } protected: SetType() {} ListExpr* elements; - - DECLARE_SERIAL(SetType) }; class FuncType : public BroType { public: FuncType(RecordType* args, BroType* yield, function_flavor f); + FuncType* ShallowClone() override; ~FuncType() override; RecordType* Args() const { return args; } BroType* YieldType() override; - const BroType* YieldType() const; + const BroType* YieldType() const override; void SetYieldType(BroType* arg_yield) { yield = arg_yield; } function_flavor Flavor() const { return flavor; } string FlavorString() const; @@ -419,9 +418,7 @@ public: void DescribeReST(ODesc* d, bool roles_only = false) const override; protected: - FuncType() { args = 0; arg_types = 0; yield = 0; flavor = FUNC_FLAVOR_FUNCTION; } - DECLARE_SERIAL(FuncType) - + FuncType() : BroType(TYPE_FUNC) { args = 0; arg_types = 0; yield = 0; flavor = FUNC_FLAVOR_FUNCTION; } RecordType* args; TypeList* arg_types; BroType* yield; @@ -431,6 +428,7 @@ protected: class TypeType : public BroType { public: explicit TypeType(BroType* t) : BroType(TYPE_TYPE) { type = t->Ref(); } + TypeType* ShallowClone() override { return new TypeType(type); } ~TypeType() override { Unref(type); } BroType* Type() { return type; } @@ -450,9 +448,6 @@ public: const Attr* FindAttr(attr_tag a) const { return attrs ? attrs->FindAttr(a) : 0; } - bool Serialize(SerialInfo* info) const; - static TypeDecl* Unserialize(UnserialInfo* info); - virtual void DescribeReST(ODesc* d, bool roles_only = false) const; BroType* type; @@ -466,6 +461,7 @@ typedef PList(TypeDecl) type_decl_list; class RecordType : public BroType { public: explicit RecordType(type_decl_list* types); + RecordType* ShallowClone() override; ~RecordType() override; @@ -501,8 +497,6 @@ public: protected: RecordType() { types = 0; } - DECLARE_SERIAL(RecordType) - int num_fields; type_decl_list* types; }; @@ -511,13 +505,12 @@ class SubNetType : public BroType { public: SubNetType(); void Describe(ODesc* d) const override; -protected: - DECLARE_SERIAL(SubNetType) }; class FileType : public BroType { public: explicit FileType(BroType* yield_type); + FileType* ShallowClone() override { return new FileType(yield->Ref()); } ~FileType() override; BroType* YieldType() override; @@ -527,14 +520,13 @@ public: protected: FileType() { yield = 0; } - DECLARE_SERIAL(FileType) - BroType* yield; }; class OpaqueType : public BroType { public: explicit OpaqueType(const string& name); + OpaqueType* ShallowClone() override { return new OpaqueType(name); } ~OpaqueType() override { }; const string& Name() const { return name; } @@ -545,8 +537,6 @@ public: protected: OpaqueType() { } - DECLARE_SERIAL(OpaqueType) - string name; }; @@ -554,8 +544,9 @@ class EnumType : public BroType { public: typedef std::list > enum_name_list; - explicit EnumType(EnumType* e); + explicit EnumType(const EnumType* e); explicit EnumType(const string& arg_name); + EnumType* ShallowClone() override; ~EnumType() override; // The value of this name is next internal counter value, starting @@ -582,8 +573,6 @@ public: protected: EnumType() { counter = 0; } - DECLARE_SERIAL(EnumType) - void AddNameInternal(const string& module_name, const char* name, bro_int_t val, bool is_export); @@ -609,9 +598,10 @@ protected: class VectorType : public BroType { public: explicit VectorType(BroType* t); + VectorType* ShallowClone() override; ~VectorType() override; BroType* YieldType() override; - const BroType* YieldType() const; + const BroType* YieldType() const override; int MatchesIndex(ListExpr*& index) const override; @@ -625,8 +615,6 @@ public: protected: VectorType() { yield_type = 0; } - DECLARE_SERIAL(VectorType) - BroType* yield_type; }; diff --git a/src/Val.cc b/src/Val.cc index 21a2d4a7db..409268d801 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -20,7 +20,6 @@ #include "Scope.h" #include "NetVar.h" #include "Expr.h" -#include "Serializer.h" #include "PrefixTable.h" #include "Conn.h" #include "Reporter.h" @@ -74,7 +73,8 @@ Val::~Val() Val* Val::Clone() { Val::CloneState state; - return Clone(&state); + auto v = Clone(&state); + return v; } Val* Val::Clone(CloneState* state) @@ -85,7 +85,10 @@ Val* Val::Clone(CloneState* state) return i->second->Ref(); auto c = DoClone(state); - assert(c); + + if ( ! c ) + reporter->RuntimeError(GetLocationInfo(), "cannot clone value"); + return c; } @@ -130,222 +133,6 @@ Val* Val::DoClone(CloneState* state) return nullptr; } -bool Val::Serialize(SerialInfo* info) const - { - return SerialObj::Serialize(info); - } - -Val* Val::Unserialize(UnserialInfo* info, TypeTag type, const BroType* exact_type) - { - Val* v = (Val*) SerialObj::Unserialize(info, SER_VAL); - if ( ! v ) - return 0; - - if ( type != TYPE_ANY && (v->Type()->Tag() != type - || (exact_type && ! same_type(exact_type, v->Type()))) ) - { - info->s->Error("type mismatch for value"); - Unref(v); - return 0; - } - - // For MutableVals, we may get a value which, by considering the - // globally unique ID, we already know. To keep references correct, - // we have to bind to the local version. (FIXME: This is not the - // nicest solution. Ideally, DoUnserialize() should be able to pass - // us an alternative ptr to the correct object.) - if ( v->IsMutableVal() ) - { - MutableVal* mv = v->AsMutableVal(); - if ( mv->HasUniqueID() ) - { - ID* current = - global_scope()->Lookup(mv->UniqueID()->Name()); - - if ( current && current != mv->UniqueID() ) - { - DBG_LOG(DBG_STATE, "binding to already existing ID %s\n", current->Name()); - assert(current->ID_Val()); - - // Need to unset the ID here. Otherwise, - // when the SerializationCache destroys - // the value, the global name will disappear. - mv->SetID(0); - Unref(v); - return current->ID_Val()->Ref(); - } - } - } - - // An enum may be bound to a different internal number remotely than we - // do for the same identifier. Check if this is the case, and, if yes, - // rebind to our value. - if ( v->Type()->Tag() == TYPE_ENUM ) - { - int rv = v->AsEnum(); - EnumType* rt = v->Type()->AsEnumType(); - - const char* name = rt->Lookup(rv); - if ( name ) - { - // See if we know the enum locally. - ID* local = global_scope()->Lookup(name); - if ( local && local->IsEnumConst() ) - { - EnumType* lt = local->Type()->AsEnumType(); - int lv = lt->Lookup(local->ModuleName(), - local->Name()); - - // Compare. - if ( rv != lv ) - { - // Different, so let's bind the val - // to the local type. - v->val.int_val = lv; - Unref(rt); - v->type = lt; - ::Ref(lt); - } - } - } - - } - - return v; - } - -IMPLEMENT_SERIAL(Val, SER_VAL); - -bool Val::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_VAL, BroObj); - - if ( ! type->Serialize(info) ) - return false; - - switch ( type->InternalType() ) { - case TYPE_INTERNAL_VOID: - info->s->Error("type is void"); - return false; - - case TYPE_INTERNAL_INT: - return SERIALIZE(val.int_val); - - case TYPE_INTERNAL_UNSIGNED: - return SERIALIZE(val.uint_val); - - case TYPE_INTERNAL_DOUBLE: - return SERIALIZE(val.double_val); - - case TYPE_INTERNAL_STRING: - return SERIALIZE_STR((const char*) val.string_val->Bytes(), - val.string_val->Len()); - - case TYPE_INTERNAL_ADDR: - return SERIALIZE(*val.addr_val); - - case TYPE_INTERNAL_SUBNET: - return SERIALIZE(*val.subnet_val); - - case TYPE_INTERNAL_OTHER: - // Derived classes are responsible for this. - // Exception: Functions and files. There aren't any derived - // classes. - if ( type->Tag() == TYPE_FUNC ) - if ( ! AsFunc()->Serialize(info) ) - return false; - - if ( type->Tag() == TYPE_FILE ) - if ( ! AsFile()->Serialize(info) ) - return false; - return true; - - case TYPE_INTERNAL_ERROR: - info->s->Error("type is error"); - return false; - - default: - info->s->Error("type is out of range"); - return false; - } - - return false; - } - -bool Val::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BroObj); - - if ( type ) - Unref(type); - - if ( ! (type = BroType::Unserialize(info)) ) - return false; - - switch ( type->InternalType() ) { - case TYPE_INTERNAL_VOID: - info->s->Error("type is void"); - return false; - - case TYPE_INTERNAL_INT: - return UNSERIALIZE(&val.int_val); - - case TYPE_INTERNAL_UNSIGNED: - return UNSERIALIZE(&val.uint_val); - - case TYPE_INTERNAL_DOUBLE: - return UNSERIALIZE(&val.double_val); - - case TYPE_INTERNAL_STRING: - const char* str; - int len; - if ( ! UNSERIALIZE_STR(&str, &len) ) - return false; - - val.string_val = new BroString((u_char*) str, len, 1); - delete [] str; - return true; - - case TYPE_INTERNAL_ADDR: - { - val.addr_val = new IPAddr(); - return UNSERIALIZE(val.addr_val); - } - - case TYPE_INTERNAL_SUBNET: - { - val.subnet_val = new IPPrefix(); - return UNSERIALIZE(val.subnet_val); - } - - case TYPE_INTERNAL_OTHER: - // Derived classes are responsible for this. - // Exception: Functions and files. There aren't any derived - // classes. - if ( type->Tag() == TYPE_FUNC ) - { - val.func_val = Func::Unserialize(info); - return val.func_val != 0; - } - else if ( type->Tag() == TYPE_FILE ) - { - val.file_val = BroFile::Unserialize(info); - return val.file_val != 0; - } - return true; - - case TYPE_INTERNAL_ERROR: - info->s->Error("type is error"); - return false; - - default: - info->s->Error("type out of range"); - return false; - } - - return false; - } - int Val::IsZero() const { switch ( type->InternalType() ) { @@ -715,61 +502,6 @@ void MutableVal::TransferUniqueID(MutableVal* mv) mv->id = 0; } -IMPLEMENT_SERIAL(MutableVal, SER_MUTABLE_VAL); - -bool MutableVal::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_MUTABLE_VAL, Val); - - if ( ! SERIALIZE(props) ) - return false; - - // Don't use ID::Serialize here, that would loop. All we - // need is the name, anyway. - const char* name = id ? id->Name() : ""; - if ( ! SERIALIZE(name) ) - return false; - - return true; - } - -bool MutableVal::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Val); - - if ( ! UNSERIALIZE(&props) ) - return false; - - id = 0; - - const char* name; - if ( ! UNSERIALIZE_STR(&name, 0) ) - return false; - - if ( *name ) - { - id = new ID(name, SCOPE_GLOBAL, true); - id->SetVal(this, OP_NONE, true); - - ID* current = global_scope()->Lookup(name); - if ( ! current ) - { - global_scope()->Insert(name, id); - DBG_LOG(DBG_STATE, "installed formerly unknown ID %s", id->Name()); - } - else - { - DBG_LOG(DBG_STATE, "got already known ID %s", current->Name()); - // This means that we already know the value and - // that in fact we should bind to the local value. - // Val::Unserialize() will take care of this. - } - } - - delete [] name; - return true; - } - IntervalVal::IntervalVal(double quantity, double units) : Val(quantity * units, TYPE_INTERVAL) { @@ -812,20 +544,6 @@ void IntervalVal::ValDescribe(ODesc* d) const DO_UNIT(Microseconds, "usec") } -IMPLEMENT_SERIAL(IntervalVal, SER_INTERVAL_VAL); - -bool IntervalVal::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_INTERVAL_VAL, Val); - return true; - } - -bool IntervalVal::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Val); - return true; - } - PortVal* PortManager::Get(uint32 port_num) const { return val_mgr->GetPort(port_num); @@ -930,20 +648,6 @@ Val* PortVal::DoClone(CloneState* state) return Ref(); } -IMPLEMENT_SERIAL(PortVal, SER_PORT_VAL); - -bool PortVal::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_PORT_VAL, Val); - return true; - } - -bool PortVal::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Val); - return true; - } - AddrVal::AddrVal(const char* text) : Val(TYPE_ADDR) { val.addr_val = new IPAddr(text); @@ -994,20 +698,6 @@ Val* AddrVal::DoClone(CloneState* state) return Ref(); } -IMPLEMENT_SERIAL(AddrVal, SER_ADDR_VAL); - -bool AddrVal::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_ADDR_VAL, Val); - return true; - } - -bool AddrVal::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Val); - return true; - } - SubNetVal::SubNetVal(const char* text) : Val(TYPE_SUBNET) { string s(text); @@ -1124,20 +814,6 @@ Val* SubNetVal::DoClone(CloneState* state) return Ref(); } -IMPLEMENT_SERIAL(SubNetVal, SER_SUBNET_VAL); - -bool SubNetVal::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_SUBNET_VAL, Val); - return true; - } - -bool SubNetVal::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Val); - return true; - } - StringVal::StringVal(BroString* s) : Val(TYPE_STRING) { val.string_val = s; @@ -1190,20 +866,6 @@ Val* StringVal::DoClone(CloneState* state) val.string_val->Len(), 1))); } -IMPLEMENT_SERIAL(StringVal, SER_STRING_VAL); - -bool StringVal::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_STRING_VAL, Val); - return true; - } - -bool StringVal::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Val); - return true; - } - PatternVal::PatternVal(RE_Matcher* re) : Val(base_type(TYPE_PATTERN)) { val.re_val = re; @@ -1263,22 +925,6 @@ Val* PatternVal::DoClone(CloneState* state) return state->NewClone(this, new PatternVal(re)); } -IMPLEMENT_SERIAL(PatternVal, SER_PATTERN_VAL); - -bool PatternVal::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_PATTERN_VAL, Val); - return AsPattern()->Serialize(info); - } - -bool PatternVal::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Val); - - val.re_val = RE_Matcher::Unserialize(info); - return val.re_val != 0; - } - ListVal::ListVal(TypeTag t) : Val(new TypeList(t == TYPE_ANY ? 0 : base_type_no_ref(t))) { @@ -1373,52 +1019,6 @@ Val* ListVal::DoClone(CloneState* state) return lv; } -IMPLEMENT_SERIAL(ListVal, SER_LIST_VAL); - -bool ListVal::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_LIST_VAL, Val); - - if ( ! (SERIALIZE(char(tag)) && SERIALIZE(vals.length())) ) - return false; - - loop_over_list(vals, i) - { - if ( ! vals[i]->Serialize(info) ) - return false; - } - - return true; - } - -bool ListVal::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Val); - - char t; - int len; - - if ( ! (UNSERIALIZE(&t) && UNSERIALIZE(&len)) ) - return false; - - tag = TypeTag(t); - - while ( len-- ) - { - Val* v = Val::Unserialize(info, TYPE_ANY); - if ( ! v ) - return false; - - vals.append(v); - } - - // Our dtor will do Unref(type) in addition to Val's dtor. - if ( type ) - type->Ref(); - - return true; - } - unsigned int ListVal::MemoryAllocation() const { unsigned int size = 0; @@ -1676,7 +1276,6 @@ int TableVal::Assign(Val* index, HashKey* k, Val* new_val, Opcode op) delete old_entry_val; } - Modified(); return 1; } @@ -2158,7 +1757,6 @@ Val* TableVal::Delete(const Val* index) delete k; delete v; - Modified(); return va; } @@ -2180,7 +1778,6 @@ Val* TableVal::Delete(const HashKey* k) if ( LoggingAccess() ) StateAccess::Log(new StateAccess(OP_DEL, this, k)); - Modified(); return va; } @@ -2450,7 +2047,6 @@ void TableVal::DoExpire(double t) tbl->RemoveEntry(k); Unref(v->Value()); delete v; - Modified(); } delete k; @@ -2623,236 +2219,6 @@ Val* TableVal::DoClone(CloneState* state) return tv; } -IMPLEMENT_SERIAL(TableVal, SER_TABLE_VAL); - -// This is getting rather complex due to the ability to suspend even within -// deeply-nested values. -bool TableVal::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE_WITH_SUSPEND(SER_TABLE_VAL, MutableVal); - - // The current state of the serialization. - struct State { - IterCookie* c; - TableEntryVal* v; // current value - bool did_index; // already wrote the val's index - }* state = 0; - - PDict(TableEntryVal)* tbl = - const_cast(this)->AsNonConstTable(); - - if ( info->cont.NewInstance() ) - { - // For simplicity, we disable suspension for the objects - // serialized here. (In fact we know that *currently* - // they won't even try). - DisableSuspend suspend(info); - - state = new State; - state->c = tbl->InitForIteration(); - tbl->MakeRobustCookie(state->c); - state->v = 0; - state->did_index = false; - info->s->WriteOpenTag(table_type->IsSet() ? "set" : "table"); - - SERIALIZE_OPTIONAL(attrs); - SERIALIZE_OPTIONAL(expire_time); - SERIALIZE_OPTIONAL(expire_func); - - // Make sure nobody kills us in between. - const_cast(this)->Ref(); - } - - else if ( info->cont.ChildSuspended() ) - state = (State*) info->cont.RestoreState(); - - else if ( info->cont.Resuming() ) - { - info->cont.Resume(); - state = (State*) info->cont.RestoreState(); - } - else - reporter->InternalError("unknown continuation state"); - - HashKey* k = 0; - int count = 0; - - assert((!info->cont.ChildSuspended()) || state->v); - - while ( true ) - { - if ( ! state->v ) - { - state->v = tbl->NextEntry(k, state->c); - if ( ! state->c ) - { - // No next one. - if ( ! SERIALIZE(false) ) - { - delete k; - return false; - } - - break; - } - - // There's a value coming. - if ( ! SERIALIZE(true) ) - { - delete k; - return false; - } - - if ( state->v->Value() ) - state->v->Ref(); - - state->did_index = false; - } - - // Serialize index. - if ( k && ! state->did_index ) - { - // Indices are rather small, so we disable suspension - // here again. - DisableSuspend suspend(info); - info->s->WriteOpenTag("key"); - ListVal* index = table_hash->RecoverVals(k)->AsListVal(); - delete k; - - if ( ! index->Serialize(info) ) - return false; - - Unref(index); - info->s->WriteCloseTag("key"); - - state->did_index = true; - - // Start serializing data. - if ( ! type->IsSet() ) - info->s->WriteOpenTag("value"); - } - - if ( ! type->IsSet() ) - { - info->cont.SaveState(state); - info->cont.SaveContext(); - bool result = state->v->val->Serialize(info); - info->cont.RestoreContext(); - - if ( ! result ) - return false; - - if ( info->cont.ChildSuspended() ) - return true; - } - - double eat = state->v->ExpireAccessTime(); - - if ( ! (SERIALIZE(state->v->last_access_time) && - SERIALIZE(eat)) ) - return false; - - info->s->WriteCloseTag("value"); - - if ( state->v->Value() ) - state->v->Unref(); - state->v = 0; // Next value. - - // Suspend if we've done enough for now (which means we - // have serialized more than table_incremental_step entries - // in a row; if an entry has suspended itself in between, - // we start counting from 0). - if ( info->may_suspend && ++count > table_incremental_step) - { - info->cont.SaveState(state); - info->cont.Suspend(); - reporter->Info("TableVals serialization suspended right in the middle."); - return true; - } - } - - info->s->WriteCloseTag(table_type->IsSet() ? "set" : "table"); - delete state; - - Unref(const_cast(this)); - return true; - } - -bool TableVal::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(MutableVal); - - Init((TableType*) type); - - UNSERIALIZE_OPTIONAL(attrs, Attributes::Unserialize(info)); - UNSERIALIZE_OPTIONAL(expire_time, Expr::Unserialize(info)); - UNSERIALIZE_OPTIONAL(expire_func, Expr::Unserialize(info)); - - while ( true ) - { - // Anymore? - bool next; - if ( ! UNSERIALIZE(&next) ) - return false; - - if ( ! next ) - break; - - // Unserialize index. - ListVal* index = - (ListVal*) Val::Unserialize(info, table_type->Indices()); - if ( ! index ) - return false; - - // Unserialize data. - Val* entry; - if ( ! table_type->IsSet() ) - { - entry = Val::Unserialize(info, type->YieldType()); - if ( ! entry ) - return false; - } - else - entry = 0; - - TableEntryVal* entry_val = new TableEntryVal(entry); - - double eat; - - if ( ! UNSERIALIZE(&entry_val->last_access_time) || - ! UNSERIALIZE(&eat) ) - { - entry_val->Unref(); - delete entry_val; - return false; - } - - entry_val->SetExpireAccess(eat); - - HashKey* key = ComputeHash(index); - TableEntryVal* old_entry_val = - AsNonConstTable()->Insert(key, entry_val); - assert(! old_entry_val); - - delete key; - - if ( subnets ) - subnets->Insert(index, entry_val); - - Unref(index); - } - - // If necessary, activate the expire timer. - if ( attrs ) - { - CheckExpireAttr(ATTR_EXPIRE_READ); - CheckExpireAttr(ATTR_EXPIRE_WRITE); - CheckExpireAttr(ATTR_EXPIRE_CREATE); - } - - return true; - } - bool TableVal::AddProperties(Properties arg_props) { if ( ! MutableVal::AddProperties(arg_props) ) @@ -2995,7 +2361,6 @@ void RecordVal::Assign(int field, Val* new_val, Opcode op) } Unref(old_val); - Modified(); } Val* RecordVal::Lookup(int field) const @@ -3205,59 +2570,6 @@ Val* RecordVal::DoClone(CloneState* state) return rv; } -IMPLEMENT_SERIAL(RecordVal, SER_RECORD_VAL); - -bool RecordVal::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_RECORD_VAL, MutableVal); - - // We could use the type name as a tag here. - info->s->WriteOpenTag("record"); - auto record_type = Type()->AsRecordType(); - - // FIXME: What about origin? - - if ( ! SERIALIZE(val.val_list_val->length()) ) - return false; - - loop_over_list(*val.val_list_val, i) - { - info->s->WriteOpenTag(record_type->FieldName(i)); - Val* v = (*val.val_list_val)[i]; - SERIALIZE_OPTIONAL(v); - info->s->WriteCloseTag(record_type->FieldName(i)); - } - - info->s->WriteCloseTag("record"); - - return true; - } - -bool RecordVal::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(MutableVal); - - origin = 0; - - int len; - if ( ! UNSERIALIZE(&len) ) - { - val.val_list_val = new val_list; - return false; - } - - val.val_list_val = new val_list(len); - - for ( int i = 0; i < len; ++i ) - { - Val* v; - UNSERIALIZE_OPTIONAL(v, Val::Unserialize(info)); - AsNonConstRecord()->append(v); // correct for v==0, too. - } - - return true; - } - bool RecordVal::AddProperties(Properties arg_props) { if ( ! MutableVal::AddProperties(arg_props) ) @@ -3325,20 +2637,6 @@ Val* EnumVal::DoClone(CloneState* state) return Ref(); } -IMPLEMENT_SERIAL(EnumVal, SER_ENUM_VAL); - -bool EnumVal::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_ENUM_VAL, Val); - return true; - } - -bool EnumVal::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Val); - return true; - } - VectorVal::VectorVal(VectorType* t) : MutableVal(t) { vector_type = t->Ref()->AsVectorType(); @@ -3391,7 +2689,6 @@ bool VectorVal::Assign(unsigned int index, Val* element, Opcode op) // to do it similarly. (*val.vector_val)[index] = element; - Modified(); return true; } @@ -3428,7 +2725,6 @@ bool VectorVal::Insert(unsigned int index, Val* element) // to do it similarly. val.vector_val->insert(it, element); - Modified(); return true; } @@ -3442,7 +2738,6 @@ bool VectorVal::Remove(unsigned int index) val.vector_val->erase(it); Unref(val_at_index); - Modified(); return true; } @@ -3541,51 +2836,6 @@ Val* VectorVal::DoClone(CloneState* state) return vv; } -IMPLEMENT_SERIAL(VectorVal, SER_VECTOR_VAL); - -bool VectorVal::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_VECTOR_VAL, MutableVal); - - info->s->WriteOpenTag("vector"); - - if ( ! SERIALIZE(unsigned(val.vector_val->size())) ) - return false; - - for ( unsigned int i = 0; i < val.vector_val->size(); ++i ) - { - info->s->WriteOpenTag("value"); - Val* v = (*val.vector_val)[i]; - SERIALIZE_OPTIONAL(v); - info->s->WriteCloseTag("value"); - } - - info->s->WriteCloseTag("vector"); - - return true; - } - -bool VectorVal::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(MutableVal); - - val.vector_val = new vector; - vector_type = type->Ref()->AsVectorType(); - - int len; - if ( ! UNSERIALIZE(&len) ) - return false; - - for ( int i = 0; i < len; ++i ) - { - Val* v; - UNSERIALIZE_OPTIONAL(v, Val::Unserialize(info, TYPE_ANY)); // accept any type - Assign(i, v); - } - - return true; - } - void VectorVal::ValDescribe(ODesc* d) const { d->Add("["); @@ -3605,34 +2855,6 @@ void VectorVal::ValDescribe(ODesc* d) const d->Add("]"); } -OpaqueVal::OpaqueVal(OpaqueType* t) : Val(t) - { - } - -OpaqueVal::~OpaqueVal() - { - } - -Val* OpaqueVal::DoClone(CloneState* state) - { - reporter->InternalError("cloning opaque type without clone implementation"); - return nullptr; - } - -IMPLEMENT_SERIAL(OpaqueVal, SER_OPAQUE_VAL); - -bool OpaqueVal::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_OPAQUE_VAL, Val); - return true; - } - -bool OpaqueVal::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Val); - return true; - } - Val* check_and_promote(Val* v, const BroType* t, int is_init, const Location* expr_location) { if ( ! v ) diff --git a/src/Val.h b/src/Val.h index 66ad7e907e..8168113acd 100644 --- a/src/Val.h +++ b/src/Val.h @@ -8,6 +8,9 @@ #include #include #include +#include + +#include #include "net_util.h" #include "Type.h" @@ -20,6 +23,7 @@ #include "Scope.h" #include "StateAccess.h" #include "IPAddr.h" +#include "DebugLogger.h" // We have four different port name spaces: TCP, UDP, ICMP, and UNKNOWN. // We distinguish between them based on the bits specified in the *_PORT_MASK @@ -36,7 +40,6 @@ class Func; class BroFile; class RE_Matcher; class PrefixTable; -class SerialInfo; class PortVal; class AddrVal; @@ -50,6 +53,7 @@ class ListVal; class StringVal; class EnumVal; class MutableVal; +class OpaqueVal; class StateAccess; @@ -304,6 +308,7 @@ public: CONVERTER(TYPE_STRING, StringVal*, AsStringVal) CONVERTER(TYPE_VECTOR, VectorVal*, AsVectorVal) CONVERTER(TYPE_ENUM, EnumVal*, AsEnumVal) + CONVERTER(TYPE_OPAQUE, OpaqueVal*, AsOpaqueVal) #define CONST_CONVERTER(tag, ctype, name) \ const ctype name() const \ @@ -321,6 +326,7 @@ public: CONST_CONVERTER(TYPE_LIST, ListVal*, AsListVal) CONST_CONVERTER(TYPE_STRING, StringVal*, AsStringVal) CONST_CONVERTER(TYPE_VECTOR, VectorVal*, AsVectorVal) + CONST_CONVERTER(TYPE_OPAQUE, OpaqueVal*, AsOpaqueVal) bool IsMutableVal() const { @@ -344,14 +350,6 @@ public: void Describe(ODesc* d) const override; virtual void DescribeReST(ODesc* d) const; - bool Serialize(SerialInfo* info) const; - static Val* Unserialize(UnserialInfo* info, TypeTag type = TYPE_ANY) - { return Unserialize(info, type, 0); } - static Val* Unserialize(UnserialInfo* info, const BroType* exact_type) - { return Unserialize(info, exact_type->Tag(), exact_type); } - - DECLARE_SERIAL(Val); - #ifdef DEBUG // For debugging, we keep a reference to the global ID to which a // value has been bound *last*. @@ -420,10 +418,6 @@ protected: ACCESSOR(TYPE_TABLE, PDict(TableEntryVal)*, table_val, AsNonConstTable) ACCESSOR(TYPE_RECORD, val_list*, val_list_val, AsNonConstRecord) - // Just an internal helper. - static Val* Unserialize(UnserialInfo* info, TypeTag type, - const BroType* exact_type); - // For internal use by the Val::Clone() methods. struct CloneState { // Caches a cloned value for later reuse during the same @@ -566,18 +560,10 @@ public: #endif } - uint64 LastModified() const override { return last_modified; } - - // Mark value as changed. - void Modified() - { - last_modified = IncreaseTimeCounter(); - } - protected: explicit MutableVal(BroType* t) : Val(t) - { props = 0; id = 0; last_modified = SerialObj::ALWAYS; } - MutableVal() { props = 0; id = 0; last_modified = SerialObj::ALWAYS; } + { props = 0; id = 0; } + MutableVal() { props = 0; id = 0; } ~MutableVal() override; friend class ID; @@ -585,8 +571,6 @@ protected: void SetID(ID* arg_id) { Unref(id); id = arg_id; } - DECLARE_SERIAL(MutableVal); - private: ID* Bind() const; @@ -611,8 +595,6 @@ protected: IntervalVal() {} void ValDescribe(ODesc* d) const override; - - DECLARE_SERIAL(IntervalVal); }; @@ -659,8 +641,6 @@ protected: void ValDescribe(ODesc* d) const override; Val* DoClone(CloneState* state) override; - - DECLARE_SERIAL(PortVal); }; class AddrVal : public Val { @@ -685,8 +665,6 @@ protected: explicit AddrVal(BroType* t) : Val(t) { } Val* DoClone(CloneState* state) override; - - DECLARE_SERIAL(AddrVal); }; class SubNetVal : public Val { @@ -715,8 +693,6 @@ protected: void ValDescribe(ODesc* d) const override; Val* DoClone(CloneState* state) override; - - DECLARE_SERIAL(SubNetVal); }; class StringVal : public Val { @@ -748,8 +724,6 @@ protected: void ValDescribe(ODesc* d) const override; Val* DoClone(CloneState* state) override; - - DECLARE_SERIAL(StringVal); }; class PatternVal : public Val { @@ -769,8 +743,6 @@ protected: void ValDescribe(ODesc* d) const override; Val* DoClone(CloneState* state) override; - - DECLARE_SERIAL(PatternVal); }; // ListVals are mainly used to index tables that have more than one @@ -816,8 +788,6 @@ protected: Val* DoClone(CloneState* state) override; - DECLARE_SERIAL(ListVal); - val_list vals; TypeTag tag; }; @@ -1036,8 +1006,6 @@ protected: Val* DoClone(CloneState* state) override; - DECLARE_SERIAL(TableVal); - TableType* table_type; CompositeHash* table_hash; Attributes* attrs; @@ -1110,8 +1078,7 @@ protected: Val* DoClone(CloneState* state) override; - DECLARE_SERIAL(RecordVal); - + RecordType* record_type; BroObj* origin; static vector parse_time_records; @@ -1141,8 +1108,6 @@ protected: void ValDescribe(ODesc* d) const override; Val* DoClone(CloneState* state) override; - - DECLARE_SERIAL(EnumVal); }; @@ -1209,28 +1174,9 @@ protected: void ValDescribe(ODesc* d) const override; Val* DoClone(CloneState* state) override; - DECLARE_SERIAL(VectorVal); - VectorType* vector_type; }; -// Base class for values with types that are managed completely internally, -// with no further script-level operators provided (other than bif -// functions). See OpaqueVal.h for derived classes. -class OpaqueVal : public Val { -public: - explicit OpaqueVal(OpaqueType* t); - ~OpaqueVal() override; - -protected: - friend class Val; - OpaqueVal() { } - - Val* DoClone(CloneState* state) override; - - DECLARE_SERIAL(OpaqueVal); -}; - // Checks the given value for consistency with the given type. If an // exact match, returns it. If promotable, returns the promoted version, // Unref()'ing the original. If not a match, generates an error message diff --git a/src/Var.cc b/src/Var.cc index 16ced341c1..dcdd1cc279 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -6,7 +6,6 @@ #include "Func.h" #include "Stmt.h" #include "Scope.h" -#include "Serializer.h" #include "EventRegistry.h" #include "Traverse.h" @@ -274,7 +273,7 @@ void add_type(ID* id, BroType* t, attr_list* attr) tnew = t; else // Clone the type to preserve type name aliasing. - tnew = t->Clone(); + tnew = t->ShallowClone(); BroType::AddAlias(new_type_name, tnew); diff --git a/src/analyzer/protocol/krb/KRB.cc b/src/analyzer/protocol/krb/KRB.cc index 4ee663dcf1..e6bd42b961 100644 --- a/src/analyzer/protocol/krb/KRB.cc +++ b/src/analyzer/protocol/krb/KRB.cc @@ -1,5 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include + #include "KRB.h" #include "types.bif.h" #include "events.bif.h" diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.cc b/src/analyzer/protocol/tcp/TCP_Reassembler.cc index bfcb2c9c5f..939497b904 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.cc +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.cc @@ -441,20 +441,6 @@ void TCP_Reassembler::Overlap(const u_char* b1, const u_char* b2, uint64 n) } } -IMPLEMENT_SERIAL(TCP_Reassembler, SER_TCP_REASSEMBLER); - -bool TCP_Reassembler::DoSerialize(SerialInfo* info) const - { - reporter->InternalError("TCP_Reassembler::DoSerialize not implemented"); - return false; // Cannot be reached. - } - -bool TCP_Reassembler::DoUnserialize(UnserialInfo* info) - { - reporter->InternalError("TCP_Reassembler::DoUnserialize not implemented"); - return false; // Cannot be reached. - } - void TCP_Reassembler::Deliver(uint64 seq, int len, const u_char* data) { if ( type == Direct ) diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.h b/src/analyzer/protocol/tcp/TCP_Reassembler.h index bacfa663e0..f4512e4503 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.h +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.h @@ -89,8 +89,6 @@ public: private: TCP_Reassembler() { } - DECLARE_SERIAL(TCP_Reassembler); - void Undelivered(uint64 up_to_seq) override; void Gap(uint64 seq, uint64 len); diff --git a/src/broker/Data.cc b/src/broker/Data.cc index 849bad5d9b..966fed6426 100644 --- a/src/broker/Data.cc +++ b/src/broker/Data.cc @@ -126,15 +126,6 @@ struct val_converter { return rval->Ref(); } - case TYPE_OPAQUE: - { - SerializationFormat* form = new BinarySerializationFormat(); - form->StartRead(a.data(), a.size()); - CloneSerializer ss(form); - UnserialInfo uinfo(&ss); - uinfo.cache = false; - return Val::Unserialize(&uinfo, type->Tag()); - } default: return nullptr; } @@ -434,6 +425,8 @@ struct val_converter { auto rval = new PatternVal(re); return rval; } + else if ( type->Tag() == TYPE_OPAQUE ) + return OpaqueVal::Unserialize(a); return nullptr; } @@ -508,16 +501,6 @@ struct type_checker { return true; } - case TYPE_OPAQUE: - { - // TODO - SerializationFormat* form = new BinarySerializationFormat(); - form->StartRead(a.data(), a.size()); - CloneSerializer ss(form); - UnserialInfo uinfo(&ss); - uinfo.cache = false; - return Val::Unserialize(&uinfo, type->Tag()); - } default: return false; } @@ -764,6 +747,15 @@ struct type_checker { return true; } + else if ( type->Tag() == TYPE_OPAQUE ) + { + // TODO: Could avoid doing the full unserialization here + // and just check if the type is a correct match. + auto ov = OpaqueVal::Unserialize(a); + auto rval = ov != nullptr; + Unref(ov); + return rval; + } return false; } @@ -980,21 +972,14 @@ broker::expected bro_broker::val_to_data(Val* v) } case TYPE_OPAQUE: { - SerializationFormat* form = new BinarySerializationFormat(); - form->StartWrite(); - CloneSerializer ss(form); - SerialInfo sinfo(&ss); - sinfo.cache = false; - sinfo.include_locations = false; + auto c = v->AsOpaqueVal()->Serialize(); + if ( ! c ) + { + reporter->Error("unsupported opaque type for serialization"); + break; + } - if ( ! v->Serialize(&sinfo) ) - return broker::ec::invalid_data; - - char* data; - uint32 len = form->EndWrite(&data); - string rval(data, len); - free(data); - return {std::move(rval)}; + return {c}; } default: reporter->Error("unsupported Broker::Data type: %s", @@ -1131,39 +1116,114 @@ Val* bro_broker::DataVal::castTo(BroType* t) return data_to_val(data, t); } -IMPLEMENT_SERIAL(bro_broker::DataVal, SER_COMM_DATA_VAL); +IMPLEMENT_OPAQUE_VALUE(bro_broker::DataVal) -bool bro_broker::DataVal::DoSerialize(SerialInfo* info) const +broker::expected bro_broker::DataVal::DoSerialize() const { - DO_SERIALIZE(SER_COMM_DATA_VAL, OpaqueVal); - - std::string buffer; - caf::containerbuf sb{buffer}; - caf::stream_serializer&> serializer{sb}; - serializer << data; - - if ( ! SERIALIZE_STR(buffer.data(), buffer.size()) ) - return false; + return data; + } +bool bro_broker::DataVal::DoUnserialize(const broker::data& data_) + { + data = data_; return true; } -bool bro_broker::DataVal::DoUnserialize(UnserialInfo* info) +IMPLEMENT_OPAQUE_VALUE(bro_broker::SetIterator) + +broker::expected bro_broker::SetIterator::DoSerialize() const { - DO_UNSERIALIZE(OpaqueVal); + return broker::vector{dat, *it}; + } - const char* serial; - int len; - - if ( ! UNSERIALIZE_STR(&serial, &len) ) +bool bro_broker::SetIterator::DoUnserialize(const broker::data& data) + { + auto v = caf::get_if(&data); + if ( ! (v && v->size() == 2) ) return false; - caf::arraybuf sb{const_cast(serial), // will not write - static_cast(len)}; - caf::stream_deserializer&> deserializer{sb}; - deserializer >> data; + auto x = caf::get_if(&(*v)[0]); - delete [] serial; + // We set the iterator by finding the element it used to point to. + // This is not perfect, as there's no guarantee that the restored + // container will list the elements in the same order. But it's as + // good as we can do, and it should generally work out. + if( x->find((*v)[1]) == x->end() ) + return false; + + dat = *x; + it = dat.find((*v)[1]); + return true; + } + +IMPLEMENT_OPAQUE_VALUE(bro_broker::TableIterator) + +broker::expected bro_broker::TableIterator::DoSerialize() const + { + return broker::vector{dat, it->first}; + } + +bool bro_broker::TableIterator::DoUnserialize(const broker::data& data) + { + auto v = caf::get_if(&data); + if ( ! (v && v->size() == 2) ) + return false; + + auto x = caf::get_if(&(*v)[0]); + + // We set the iterator by finding the element it used to point to. + // This is not perfect, as there's no guarantee that the restored + // container will list the elements in the same order. But it's as + // good as we can do, and it should generally work out. + if( x->find((*v)[1]) == x->end() ) + return false; + + dat = *x; + it = dat.find((*v)[1]); + return true; + } + +IMPLEMENT_OPAQUE_VALUE(bro_broker::VectorIterator) + +broker::expected bro_broker::VectorIterator::DoSerialize() const + { + broker::integer difference = it - dat.begin(); + return broker::vector{dat, difference}; + } + +bool bro_broker::VectorIterator::DoUnserialize(const broker::data& data) + { + auto v = caf::get_if(&data); + if ( ! (v && v->size() == 2) ) + return false; + + auto x = caf::get_if(&(*v)[0]); + auto y = caf::get_if(&(*v)[1]); + + dat = *x; + it = dat.begin() + *y; + return true; + } + +IMPLEMENT_OPAQUE_VALUE(bro_broker::RecordIterator) + +broker::expected bro_broker::RecordIterator::DoSerialize() const + { + broker::integer difference = it - dat.begin(); + return broker::vector{dat, difference}; + } + +bool bro_broker::RecordIterator::DoUnserialize(const broker::data& data) + { + auto v = caf::get_if(&data); + if ( ! (v && v->size() == 2) ) + return false; + + auto x = caf::get_if(&(*v)[0]); + auto y = caf::get_if(&(*v)[1]); + + dat = *x; + it = dat.begin() + *y; return true; } diff --git a/src/broker/Data.h b/src/broker/Data.h index e2a5968a82..eda8f6550c 100644 --- a/src/broker/Data.h +++ b/src/broker/Data.h @@ -2,7 +2,7 @@ #define BRO_COMM_DATA_H #include -#include "Val.h" +#include "OpaqueVal.h" #include "Reporter.h" #include "Frame.h" #include "Expr.h" @@ -120,14 +120,14 @@ public: return script_data_type; } - DECLARE_SERIAL(DataVal); - broker::data data; protected: DataVal() {} + DECLARE_OPAQUE_VALUE(bro_broker::DataVal) + static BroType* script_data_type; }; @@ -242,6 +242,10 @@ public: broker::set dat; broker::set::iterator it; + +protected: + SetIterator() {} + DECLARE_OPAQUE_VALUE(bro_broker::SetIterator) }; class TableIterator : public OpaqueVal { @@ -255,6 +259,10 @@ public: broker::table dat; broker::table::iterator it; + +protected: + TableIterator() {} + DECLARE_OPAQUE_VALUE(bro_broker::TableIterator) }; class VectorIterator : public OpaqueVal { @@ -268,6 +276,10 @@ public: broker::vector dat; broker::vector::iterator it; + +protected: + VectorIterator() {} + DECLARE_OPAQUE_VALUE(bro_broker::VectorIterator) }; class RecordIterator : public OpaqueVal { @@ -281,6 +293,10 @@ public: broker::vector dat; broker::vector::iterator it; + +protected: + RecordIterator() {} + DECLARE_OPAQUE_VALUE(bro_broker::RecordIterator) }; } // namespace bro_broker diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 2cf0f8e3f2..e4ef3d570c 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -18,6 +18,7 @@ #include "logging/Manager.h" #include "DebugLogger.h" #include "iosource/Manager.h" +#include "SerializationFormat.h" using namespace std; diff --git a/src/broker/Store.cc b/src/broker/Store.cc index 200e1b6abf..2f61b14d37 100644 --- a/src/broker/Store.cc +++ b/src/broker/Store.cc @@ -49,46 +49,18 @@ void StoreHandleVal::ValDescribe(ODesc* d) const d->Add("}"); } -IMPLEMENT_SERIAL(StoreHandleVal, SER_COMM_STORE_HANDLE_VAL); +IMPLEMENT_OPAQUE_VALUE(StoreHandleVal) -bool StoreHandleVal::DoSerialize(SerialInfo* info) const +broker::expected StoreHandleVal::DoSerialize() const { - DO_SERIALIZE(SER_COMM_STORE_HANDLE_VAL, OpaqueVal); - - auto name = store.name(); - if ( ! SERIALIZE_STR(name.data(), name.size()) ) - return false; - - return true; + // Cannot serialize. + return broker::ec::invalid_data; } -bool StoreHandleVal::DoUnserialize(UnserialInfo* info) +bool StoreHandleVal::DoUnserialize(const broker::data& data) { - DO_UNSERIALIZE(OpaqueVal); - - const char* name_str; - int len; - - if ( ! UNSERIALIZE_STR(&name_str, &len) ) - return false; - - std::string name(name_str, len); - delete [] name_str; - - auto handle = broker_mgr->LookupStore(name); - if ( ! handle ) - { - // Passing serialized version of store handles to other Bro processes - // doesn't make sense, only allow local clones of the handle val. - reporter->Error("failed to look up unserialized store handle %s", - name.c_str()); - return false; - } - - store = handle->store; - proxy = broker::store::proxy{store}; - - return true; + // Cannot unserialize. + return false; } broker::backend to_backend_type(BifEnum::Broker::BackendType type) diff --git a/src/broker/Store.h b/src/broker/Store.h index 1df60584fd..7e5b2bde07 100644 --- a/src/broker/Store.h +++ b/src/broker/Store.h @@ -5,7 +5,7 @@ #include "broker/data.bif.h" #include "Reporter.h" #include "Type.h" -#include "Val.h" +#include "OpaqueVal.h" #include "Trigger.h" #include @@ -116,13 +116,13 @@ public: void ValDescribe(ODesc* d) const override; - DECLARE_SERIAL(StoreHandleVal); - broker::store store; broker::store::proxy proxy; protected: StoreHandleVal() = default; + + DECLARE_OPAQUE_VALUE(StoreHandleVal) }; // Helper function to construct a broker backend type from script land. diff --git a/src/broker/data.bif b/src/broker/data.bif index 2c3f922e1d..f0862c0f66 100644 --- a/src/broker/data.bif +++ b/src/broker/data.bif @@ -41,6 +41,20 @@ function Broker::__data_type%(d: Broker::Data%): Broker::DataType return bro_broker::get_data_type(d->AsRecordVal(), frame); %} +# For testing only. +function Broker::__opaque_clone_through_serialization%(d: any%): any + %{ + auto x = d->AsOpaqueVal()->Serialize(); + + if ( ! x ) + { + builtin_error("cannot serialize object to clone"); + return val_mgr->GetFalse(); + } + + return OpaqueVal::Unserialize(std::move(*x)); + %} + function Broker::__set_create%(%): Broker::Data %{ return bro_broker::make_data_val(broker::set()); diff --git a/src/file_analysis/FileReassembler.cc b/src/file_analysis/FileReassembler.cc index ba15086320..41a37c52fd 100644 --- a/src/file_analysis/FileReassembler.cc +++ b/src/file_analysis/FileReassembler.cc @@ -110,19 +110,4 @@ void FileReassembler::Overlap(const u_char* b1, const u_char* b2, uint64 n) { // Not doing anything here yet. } - -IMPLEMENT_SERIAL(FileReassembler, SER_FILE_REASSEMBLER); - -bool FileReassembler::DoSerialize(SerialInfo* info) const - { - reporter->InternalError("FileReassembler::DoSerialize not implemented"); - return false; // Cannot be reached. - } - -bool FileReassembler::DoUnserialize(UnserialInfo* info) - { - reporter->InternalError("FileReassembler::DoUnserialize not implemented"); - return false; // Cannot be reached. - } - } // end file_analysis diff --git a/src/file_analysis/FileReassembler.h b/src/file_analysis/FileReassembler.h index c6143a5565..79aff34829 100644 --- a/src/file_analysis/FileReassembler.h +++ b/src/file_analysis/FileReassembler.h @@ -50,8 +50,6 @@ public: protected: FileReassembler(); - DECLARE_SERIAL(FileReassembler); - void Undelivered(uint64 up_to_seq) override; void BlockInserted(DataBlock* b) override; void Overlap(const u_char* b1, const u_char* b2, uint64 n) override; diff --git a/src/file_analysis/analyzer/x509/OCSP.cc b/src/file_analysis/analyzer/x509/OCSP.cc index d55931c946..6833d5d8de 100644 --- a/src/file_analysis/analyzer/x509/OCSP.cc +++ b/src/file_analysis/analyzer/x509/OCSP.cc @@ -28,8 +28,6 @@ X509* helper_sk_X509_value(const STACK_OF(X509)* certs, int i) using namespace file_analysis; -IMPLEMENT_SERIAL(OCSP_RESPVal, SER_OCSP_RESP_VAL); - #define OCSP_STRING_BUF_SIZE 2048 static Val* get_ocsp_type(RecordVal* args, const char* name) @@ -177,9 +175,8 @@ bool file_analysis::OCSP::EndOfFile() return false; } - OCSP_RESPVal* resp_val = new OCSP_RESPVal(resp); // resp_val takes ownership - ParseResponse(resp_val); - Unref(resp_val); + ParseResponse(resp); + OCSP_RESPONSE_free(resp); } return true; @@ -451,9 +448,8 @@ void file_analysis::OCSP::ParseRequest(OCSP_REQUEST* req) BIO_free(bio); } -void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val) +void file_analysis::OCSP::ParseResponse(OCSP_RESPONSE *resp) { - OCSP_RESPONSE *resp = resp_val->GetResp(); //OCSP_RESPBYTES *resp_bytes = resp->responseBytes; OCSP_BASICRESP *basic_resp = nullptr; OCSP_RESPDATA *resp_data = nullptr; @@ -508,7 +504,6 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val) #endif vl.append(GetFile()->GetVal()->Ref()); - vl.append(resp_val->Ref()); vl.append(status_val); #if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) @@ -692,52 +687,3 @@ void file_analysis::OCSP::ParseExtensionsSpecific(X509_EXTENSION* ex, bool globa ParseSignedCertificateTimestamps(ex); } -OCSP_RESPVal::OCSP_RESPVal(OCSP_RESPONSE* arg_ocsp_resp) : OpaqueVal(ocsp_resp_opaque_type) - { - ocsp_resp = arg_ocsp_resp; - } - -OCSP_RESPVal::OCSP_RESPVal() : OpaqueVal(ocsp_resp_opaque_type) - { - ocsp_resp = nullptr; - } - -OCSP_RESPVal::~OCSP_RESPVal() - { - if (ocsp_resp) - OCSP_RESPONSE_free(ocsp_resp); - } - -OCSP_RESPONSE* OCSP_RESPVal::GetResp() const - { - return ocsp_resp; - } - -bool OCSP_RESPVal::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_OCSP_RESP_VAL, OpaqueVal); - unsigned char *buf = nullptr; - int length = i2d_OCSP_RESPONSE(ocsp_resp, &buf); - if ( length < 0 ) - return false; - bool res = SERIALIZE_STR(reinterpret_cast(buf), length); - OPENSSL_free(buf); - return res; - } - -bool OCSP_RESPVal::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(OpaqueVal) - - int length; - unsigned char *ocsp_resp_buf, *opensslbuf; - - if ( ! UNSERIALIZE_STR(reinterpret_cast(&ocsp_resp_buf), &length) ) - return false; - opensslbuf = ocsp_resp_buf; // OpenSSL likes to shift pointers around. really. - ocsp_resp = d2i_OCSP_RESPONSE(nullptr, const_cast(&opensslbuf), length); - delete [] ocsp_resp_buf; - if ( ! ocsp_resp ) - return false; - return true; - } diff --git a/src/file_analysis/analyzer/x509/OCSP.h b/src/file_analysis/analyzer/x509/OCSP.h index eb6499794c..c2cc0f6e5d 100644 --- a/src/file_analysis/analyzer/x509/OCSP.h +++ b/src/file_analysis/analyzer/x509/OCSP.h @@ -5,7 +5,6 @@ #include -#include "Val.h" #include "../File.h" #include "Analyzer.h" #include "X509Common.h" @@ -14,8 +13,6 @@ namespace file_analysis { -class OCSP_RESPVal; - class OCSP : public file_analysis::X509Common { public: bool DeliverStream(const u_char* data, uint64 len) override; @@ -29,7 +26,7 @@ protected: OCSP(RecordVal* args, File* file, bool request); private: - void ParseResponse(OCSP_RESPVal*); + void ParseResponse(OCSP_RESPONSE*); void ParseRequest(OCSP_REQUEST*); void ParseExtensionsSpecific(X509_EXTENSION* ex, bool, ASN1_OBJECT*, const char*) override; @@ -37,18 +34,6 @@ private: bool request = false; // true if ocsp request, false if reply }; -class OCSP_RESPVal: public OpaqueVal { -public: - explicit OCSP_RESPVal(OCSP_RESPONSE *); - ~OCSP_RESPVal() override; - OCSP_RESPONSE *GetResp() const; -protected: - OCSP_RESPVal(); -private: - OCSP_RESPONSE *ocsp_resp; - DECLARE_SERIAL(OCSP_RESPVal); -}; - } #endif diff --git a/src/file_analysis/analyzer/x509/X509.cc b/src/file_analysis/analyzer/x509/X509.cc index dfbea70686..74b903f585 100644 --- a/src/file_analysis/analyzer/x509/X509.cc +++ b/src/file_analysis/analyzer/x509/X509.cc @@ -18,8 +18,6 @@ using namespace file_analysis; -IMPLEMENT_SERIAL(X509Val, SER_X509_VAL); - file_analysis::X509::X509(RecordVal* args, file_analysis::File* file) : file_analysis::X509Common::X509Common(file_mgr->GetComponentTag("X509"), args, file) { @@ -491,39 +489,29 @@ Val* X509Val::DoClone(CloneState* state) return certificate; } -bool X509Val::DoSerialize(SerialInfo* info) const +IMPLEMENT_OPAQUE_VALUE(X509Val) + +broker::expected X509Val::DoSerialize() const { - DO_SERIALIZE(SER_X509_VAL, OpaqueVal); - unsigned char *buf = NULL; - int length = i2d_X509(certificate, &buf); if ( length < 0 ) - return false; - - bool res = SERIALIZE_STR(reinterpret_cast(buf), length); + return broker::ec::invalid_data; + auto d = std::string(reinterpret_cast(buf), length); OPENSSL_free(buf); - return res; + + return {std::move(d)}; } -bool X509Val::DoUnserialize(UnserialInfo* info) +bool X509Val::DoUnserialize(const broker::data& data) { - DO_UNSERIALIZE(OpaqueVal) - - int length; - unsigned char *certbuf, *opensslbuf; - - if ( ! UNSERIALIZE_STR(reinterpret_cast(&certbuf), &length) ) + auto s = caf::get_if(&data); + if ( ! s ) return false; - opensslbuf = certbuf; // OpenSSL likes to shift pointers around. really. - certificate = d2i_X509(NULL, const_cast(&opensslbuf), length); - delete[] certbuf; - - if ( !certificate ) - return false; - - return true; + auto opensslbuf = reinterpret_cast(s->data()); + certificate = d2i_X509(NULL, &opensslbuf, s->size()); + return (certificate != nullptr); } diff --git a/src/file_analysis/analyzer/x509/X509.h b/src/file_analysis/analyzer/x509/X509.h index 01ec5bc6ae..f20712cab2 100644 --- a/src/file_analysis/analyzer/x509/X509.h +++ b/src/file_analysis/analyzer/x509/X509.h @@ -5,7 +5,7 @@ #include -#include "Val.h" +#include "OpaqueVal.h" #include "X509Common.h" #if ( OPENSSL_VERSION_NUMBER < 0x10002000L ) || defined(LIBRESSL_VERSION_NUMBER) @@ -151,10 +151,9 @@ protected: */ X509Val(); + DECLARE_OPAQUE_VALUE(X509Val) private: ::X509* certificate; // the wrapped certificate - - DECLARE_SERIAL(X509Val); }; } diff --git a/src/file_analysis/analyzer/x509/ocsp_events.bif b/src/file_analysis/analyzer/x509/ocsp_events.bif index 564126b2bb..fe17344490 100644 --- a/src/file_analysis/analyzer/x509/ocsp_events.bif +++ b/src/file_analysis/analyzer/x509/ocsp_events.bif @@ -52,9 +52,6 @@ event ocsp_response_status%(f: fa_file, status: string%); ## ## f: The file. ## -## req_ref: An opaque pointer to the underlying OpenSSL data structure of the -## OCSP response. -## ## status: The status of the OCSP response (e.g. succesful, malformedRequest, tryLater). ## ## version: Version of the OCSP response (typically - for version 1). @@ -71,7 +68,7 @@ event ocsp_response_status%(f: fa_file, status: string%); ## .. zeek:see:: ocsp_request ocsp_request_certificate ocsp_response_status ## ocsp_response_certificate ocsp_extension ## x509_ocsp_ext_signed_certificate_timestamp -event ocsp_response_bytes%(f: fa_file, resp_ref: opaque of ocsp_resp, status: string, version: count, responderId: string, producedAt: time, signatureAlgorithm: string, certs: x509_opaque_vector%); +event ocsp_response_bytes%(f: fa_file, status: string, version: count, responderId: string, producedAt: time, signatureAlgorithm: string, certs: x509_opaque_vector%); ## This event is raised for each SingleResponse contained in an OCSP response. ## See :rfc:`6960` for more details on OCSP. diff --git a/src/iosource/Packet.cc b/src/iosource/Packet.cc index 54d1cc6f27..e2bbe99c30 100644 --- a/src/iosource/Packet.cc +++ b/src/iosource/Packet.cc @@ -2,8 +2,6 @@ #include "Packet.h" #include "Sessions.h" #include "iosource/Manager.h" -#include "SerialInfo.h" -#include "Serializer.h" extern "C" { #ifdef HAVE_NET_ETHERNET_H @@ -673,66 +671,3 @@ void Packet::Describe(ODesc* d) const d->Add(ip.DstAddr()); } -bool Packet::Serialize(SerialInfo* info) const - { - return SERIALIZE(uint32(ts.tv_sec)) && - SERIALIZE(uint32(ts.tv_usec)) && - SERIALIZE(uint32(len)) && - SERIALIZE(link_type) && - info->s->Write(tag.c_str(), tag.length(), "tag") && - info->s->Write((const char*)data, cap_len, "data"); - } - -#ifdef DEBUG -static iosource::PktDumper* dump = 0; -#endif - -Packet* Packet::Unserialize(UnserialInfo* info) - { - pkt_timeval ts; - uint32 len, link_type; - - if ( ! (UNSERIALIZE((uint32 *)&ts.tv_sec) && - UNSERIALIZE((uint32 *)&ts.tv_usec) && - UNSERIALIZE(&len) && - UNSERIALIZE(&link_type)) ) - return 0; - - char* tag; - if ( ! info->s->Read((char**) &tag, 0, "tag") ) - return 0; - - const u_char* pkt; - int caplen; - if ( ! info->s->Read((char**) &pkt, &caplen, "data") ) - { - delete [] tag; - return 0; - } - - Packet *p = new Packet(link_type, &ts, caplen, len, pkt, true, - std::string(tag)); - delete [] tag; - - // For the global timer manager, we take the global network_time as the - // packet's timestamp for feeding it into our packet loop. - if ( p->tag == "" ) - p->time = timer_mgr->Time(); - else - p->time = p->ts.tv_sec + double(p->ts.tv_usec) / 1e6; - -#ifdef DEBUG - if ( debug_logger.IsEnabled(DBG_TM) ) - { - if ( ! dump ) - dump = iosource_mgr->OpenPktDumper("tm.pcap", true); - - if ( dump ) - { - dump->Dump(p); - } - } -#endif - - return p; - } diff --git a/src/iosource/Packet.h b/src/iosource/Packet.h index ec29f39ff5..3ca24cb737 100644 --- a/src/iosource/Packet.h +++ b/src/iosource/Packet.h @@ -24,12 +24,6 @@ enum Layer3Proto { /** * A link-layer packet. - * - * Note that for serialization we don't use much of the support provided by - * the serialization framework. Serialize/Unserialize do all the work by - * themselves. In particular, Packets aren't derived from SerialObj. They are - * completely seperate and self-contained entities, and we don't need any of - * the sophisticated features like object caching. */ class Packet { public: @@ -144,16 +138,6 @@ public: */ void Describe(ODesc* d) const; - /** - * Serializes the packet, with standard signature. - */ - bool Serialize(SerialInfo* info) const; - - /** - * Unserializes the packet, with standard signature. - */ - static Packet* Unserialize(UnserialInfo* info); - /** * Maximal length of a layer 2 address. */ diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index 7bede8f6e6..162b19d26a 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -4,7 +4,6 @@ #include "util.h" #include "threading/SerialTypes.h" -#include "SerializationFormat.h" #include "Manager.h" #include "WriterBackend.h" @@ -70,58 +69,6 @@ public: using namespace logging; -bool WriterBackend::WriterInfo::Read(SerializationFormat* fmt) - { - int size; - - string tmp_path; - - if ( ! (fmt->Read(&tmp_path, "path") && - fmt->Read(&rotation_base, "rotation_base") && - fmt->Read(&rotation_interval, "rotation_interval") && - fmt->Read(&network_time, "network_time") && - fmt->Read(&size, "config_size")) ) - return false; - - path = copy_string(tmp_path.c_str()); - - config.clear(); - - while ( size-- ) - { - string value; - string key; - - if ( ! (fmt->Read(&value, "config-value") && fmt->Read(&key, "config-key")) ) - return false; - - config.insert(std::make_pair(copy_string(value.c_str()), copy_string(key.c_str()))); - } - - return true; - } - - -bool WriterBackend::WriterInfo::Write(SerializationFormat* fmt) const - { - int size = config.size(); - - if ( ! (fmt->Write(path, "path") && - fmt->Write(rotation_base, "rotation_base") && - fmt->Write(rotation_interval, "rotation_interval") && - fmt->Write(network_time, "network_time") && - fmt->Write(size, "config_size")) ) - return false; - - for ( config_map::const_iterator i = config.begin(); i != config.end(); ++i ) - { - if ( ! (fmt->Write(i->first, "config-value") && fmt->Write(i->second, "config-key")) ) - return false; - } - - return true; - } - broker::data WriterBackend::WriterInfo::ToBroker() const { auto t = broker::table(); diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index 187a1957d7..35cf401199 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -112,8 +112,6 @@ public: // Note, these need to be adapted when changing the struct's // fields. They serialize/deserialize the struct. - bool Read(SerializationFormat* fmt); - bool Write(SerializationFormat* fmt) const; broker::data ToBroker() const; bool FromBroker(broker::data d); diff --git a/src/main.cc b/src/main.cc index 90d7ebd6b4..e69aa70909 100644 --- a/src/main.cc +++ b/src/main.cc @@ -38,7 +38,6 @@ extern "C" { #include "DFA.h" #include "RuleMatcher.h" #include "Anon.h" -#include "Serializer.h" #include "EventRegistry.h" #include "Stats.h" #include "Brofiler.h" @@ -99,9 +98,6 @@ name_list prefixes; Stmt* stmts; EventHandlerPtr net_done = 0; RuleMatcher* rule_matcher = 0; -FileSerializer* event_serializer = 0; -FileSerializer* state_serializer = 0; -EventPlayer* event_player = 0; EventRegistry* event_registry = 0; ProfileLogger* profiling_logger = 0; ProfileLogger* segment_logger = 0; @@ -168,7 +164,6 @@ void usage(int code = 1) fprintf(stderr, " -t|--tracefile | activate execution tracing\n"); fprintf(stderr, " -v|--version | print version and exit\n"); fprintf(stderr, " -w|--writefile | write to given tcpdump file\n"); - fprintf(stderr, " -x|--print-state | print contents of state file\n"); #ifdef DEBUG fprintf(stderr, " -B|--debug | Enable debugging output for selected streams ('-B help' for help)\n"); #endif @@ -180,7 +175,6 @@ void usage(int code = 1) fprintf(stderr, " -N|--print-plugins | print available plugins and exit (-NN for verbose)\n"); fprintf(stderr, " -P|--prime-dns | prime DNS\n"); fprintf(stderr, " -Q|--time | print execution time summary to stderr\n"); - fprintf(stderr, " -R|--replay | replay events\n"); fprintf(stderr, " -S|--debug-rules | enable rule debugging\n"); fprintf(stderr, " -T|--re-level | set 'RE_level' for rules\n"); fprintf(stderr, " -U|--status-file | Record process status in file\n"); @@ -350,8 +344,6 @@ void terminate_bro() delete zeekygen_mgr; delete timer_mgr; - delete event_serializer; - delete state_serializer; delete event_registry; delete analyzer_mgr; delete file_mgr; @@ -421,9 +413,7 @@ int main(int argc, char** argv) name_list interfaces; name_list read_files; name_list rule_files; - char* bst_file = 0; char* id_name = 0; - char* events_file = 0; char* seed_load_file = zeekenv("ZEEK_SEED_FILE"); char* seed_save_file = 0; @@ -453,7 +443,6 @@ int main(int argc, char** argv) {"tracefile", required_argument, 0, 't'}, {"writefile", required_argument, 0, 'w'}, {"version", no_argument, 0, 'v'}, - {"print-state", required_argument, 0, 'x'}, {"no-checksums", no_argument, 0, 'C'}, {"force-dns", no_argument, 0, 'F'}, {"load-seeds", required_argument, 0, 'G'}, @@ -461,7 +450,6 @@ int main(int argc, char** argv) {"print-plugins", no_argument, 0, 'N'}, {"prime-dns", no_argument, 0, 'P'}, {"time", no_argument, 0, 'Q'}, - {"replay", required_argument, 0, 'R'}, {"debug-rules", no_argument, 0, 'S'}, {"re-level", required_argument, 0, 'T'}, {"watchdog", no_argument, 0, 'W'}, @@ -512,7 +500,7 @@ int main(int argc, char** argv) opterr = 0; char opts[256]; - safe_strncpy(opts, "B:e:f:G:H:I:i:n:p:R:r:s:T:t:U:w:x:X:CFNPQSWabdghv", + safe_strncpy(opts, "B:e:f:G:H:I:i:n:p:r:s:T:t:U:w:X:CFNPQSWabdhv", sizeof(opts)); #ifdef USE_PERFTOOLS_DEBUG @@ -577,10 +565,6 @@ int main(int argc, char** argv) writefile = optarg; break; - case 'x': - bst_file = optarg; - break; - case 'B': debug_streams = optarg; break; @@ -627,10 +611,6 @@ int main(int argc, char** argv) time_bro = 1; break; - case 'R': - events_file = optarg; - break; - case 'S': rule_debug = 1; break; @@ -743,7 +723,7 @@ int main(int argc, char** argv) if ( optind == argc && read_files.length() == 0 && interfaces.length() == 0 && - ! (id_name || bst_file) && ! command_line_policy && ! print_plugins ) + ! id_name && ! command_line_policy && ! print_plugins ) add_input_file("-"); // Process remaining arguments. X=Y arguments indicate script @@ -795,9 +775,6 @@ int main(int argc, char** argv) plugin_mgr->ActivateDynamicPlugins(! bare_mode); - if ( events_file ) - event_player = new EventPlayer(events_file); - init_event_handlers(); md5_type = new OpaqueType("md5"); @@ -969,19 +946,6 @@ int main(int argc, char** argv) exit(0); } - // Just read state file from disk. - if ( bst_file ) - { - FileSerializer s; - UnserialInfo info(&s); - info.print = stdout; - info.install_uniques = true; - if ( ! s.Read(&info, bst_file) ) - reporter->Error("Failed to read events from %s\n", bst_file); - - exit(0); - } - // Print the ID. if ( id_name ) { diff --git a/src/probabilistic/BitVector.cc b/src/probabilistic/BitVector.cc index 7fa80c206b..0025e417cb 100644 --- a/src/probabilistic/BitVector.cc +++ b/src/probabilistic/BitVector.cc @@ -5,7 +5,6 @@ #include #include "BitVector.h" -#include "Serializer.h" #include "digest.h" using namespace probabilistic; @@ -506,6 +505,47 @@ uint64 BitVector::Hash() const return digest; } +broker::expected BitVector::Serialize() const + { + broker::vector v = {static_cast(num_bits), static_cast(bits.size())}; + v.reserve(2 + bits.size()); + + for ( size_t i = 0; i < bits.size(); ++i ) + v.emplace_back(static_cast(bits[i])); + + return {std::move(v)}; + } + +std::unique_ptr BitVector::Unserialize(const broker::data& data) + { + auto v = caf::get_if(&data); + if ( ! (v && v->size() >= 2) ) + return nullptr; + + auto num_bits = caf::get_if(&(*v)[0]); + auto size = caf::get_if(&(*v)[1]); + + if ( ! (num_bits && size) ) + return nullptr; + + if ( v->size() != 2 + *size ) + return nullptr; + + auto bv = std::unique_ptr(new BitVector()); + bv->num_bits = *num_bits; + + for ( size_t i = 0; i < *size; ++i ) + { + auto x = caf::get_if(&(*v)[2 + i]); + if ( ! x ) + return nullptr; + + bv->bits.push_back(*x); + } + + return bv; + } + BitVector::size_type BitVector::lowest_bit(block_type block) { block_type x = block - (block & (block - 1)); @@ -539,56 +579,3 @@ BitVector::size_type BitVector::find_from(size_type i) const return i * bits_per_block + lowest_bit(bits[i]); } -bool BitVector::Serialize(SerialInfo* info) const - { - return SerialObj::Serialize(info); - } - -BitVector* BitVector::Unserialize(UnserialInfo* info) - { - return reinterpret_cast(SerialObj::Unserialize(info, SER_BITVECTOR)); - } - -IMPLEMENT_SERIAL(BitVector, SER_BITVECTOR); - -bool BitVector::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_BITVECTOR, SerialObj); - - if ( ! SERIALIZE(static_cast(bits.size())) ) - return false; - - for ( size_t i = 0; i < bits.size(); ++i ) - if ( ! SERIALIZE(static_cast(bits[i])) ) - return false; - - return SERIALIZE(static_cast(num_bits)); - } - -bool BitVector::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(SerialObj); - - uint64 size; - if ( ! UNSERIALIZE(&size) ) - return false; - - bits.resize(static_cast(size)); - - for ( size_t i = 0; i < bits.size(); ++i ) - { - uint64 block; - if ( ! UNSERIALIZE(&block) ) - return false; - - bits[i] = static_cast(block); - } - - uint64 n; - if ( ! UNSERIALIZE(&n) ) - return false; - - num_bits = static_cast(n); - - return true; - } diff --git a/src/probabilistic/BitVector.h b/src/probabilistic/BitVector.h index a1ff0c9ad9..12d628cacf 100644 --- a/src/probabilistic/BitVector.h +++ b/src/probabilistic/BitVector.h @@ -6,16 +6,16 @@ #include #include -#include "SerialObj.h" +#include namespace probabilistic { /** * A vector of bits. */ -class BitVector : public SerialObj { +class BitVector { public: - typedef uint64 block_type; + typedef uint64_t block_type; typedef size_t size_type; typedef bool const_reference; @@ -281,28 +281,10 @@ public: * * @return The hash. */ - uint64 Hash() const; + uint64_t Hash() const; - /** - * Serializes the bit vector. - * - * @param info The serializaton informationt to use. - * - * @return True if successful. - */ - bool Serialize(SerialInfo* info) const; - - /** - * Unserialize the bit vector. - * - * @param info The serializaton informationt to use. - * - * @return The unserialized bit vector, or null if an error occured. - */ - static BitVector* Unserialize(UnserialInfo* info); - -protected: - DECLARE_SERIAL(BitVector); + broker::expected Serialize() const; + static std::unique_ptr Unserialize(const broker::data& data); private: /** diff --git a/src/probabilistic/BloomFilter.cc b/src/probabilistic/BloomFilter.cc index ef671268b9..f449fad8b6 100644 --- a/src/probabilistic/BloomFilter.cc +++ b/src/probabilistic/BloomFilter.cc @@ -7,9 +7,9 @@ #include "BloomFilter.h" #include "CounterVector.h" -#include "Serializer.h" #include "../util.h" +#include "../Reporter.h" using namespace probabilistic; @@ -28,29 +28,53 @@ BloomFilter::~BloomFilter() delete hasher; } -bool BloomFilter::Serialize(SerialInfo* info) const +broker::expected BloomFilter::Serialize() const { - return SerialObj::Serialize(info); + auto h = hasher->Serialize(); + + if ( ! h ) + return broker::ec::invalid_data; // Cannot serialize + + auto d = DoSerialize(); + + if ( ! d ) + return broker::ec::invalid_data; // Cannot serialize + + return {broker::vector{static_cast(Type()), std::move(*h), std::move(*d)}}; } -BloomFilter* BloomFilter::Unserialize(UnserialInfo* info) +std::unique_ptr BloomFilter::Unserialize(const broker::data& data) { - return reinterpret_cast(SerialObj::Unserialize(info, SER_BLOOMFILTER)); + auto v = caf::get_if(&data); + + if ( ! (v && v->size() == 3) ) + return nullptr; + + auto type = caf::get_if(&(*v)[0]); + if ( ! type ) + return nullptr; + + auto hasher_ = Hasher::Unserialize((*v)[1]); + if ( ! hasher_ ) + return nullptr; + + std::unique_ptr bf; + + switch ( *type ) { + case Basic: + bf = std::unique_ptr(new BasicBloomFilter()); + break; + + case Counting: + bf = std::unique_ptr(new CountingBloomFilter()); + break; } -bool BloomFilter::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_BLOOMFILTER, SerialObj); + if ( ! bf->DoUnserialize((*v)[2]) ) + return nullptr; - return hasher->Serialize(info); - } - -bool BloomFilter::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(SerialObj); - - hasher = Hasher::Unserialize(info); - return hasher != 0; + bf->hasher = hasher_.release(); + return bf; } size_t BasicBloomFilter::M(double fp, size_t capacity) @@ -130,21 +154,6 @@ BasicBloomFilter::~BasicBloomFilter() delete bits; } -IMPLEMENT_SERIAL(BasicBloomFilter, SER_BASICBLOOMFILTER) - -bool BasicBloomFilter::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_BASICBLOOMFILTER, BloomFilter); - return bits->Serialize(info); - } - -bool BasicBloomFilter::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BloomFilter); - bits = BitVector::Unserialize(info); - return (bits != 0); - } - void BasicBloomFilter::Add(const HashKey* key) { Hasher::digest_vector h = hasher->Hash(key); @@ -166,6 +175,22 @@ size_t BasicBloomFilter::Count(const HashKey* key) const return 1; } +broker::expected BasicBloomFilter::DoSerialize() const + { + auto b = bits->Serialize(); + return b; + } + +bool BasicBloomFilter::DoUnserialize(const broker::data& data) + { + auto b = BitVector::Unserialize(data); + if ( ! b ) + return false; + + bits = b.release(); + return true; + } + CountingBloomFilter::CountingBloomFilter() { cells = 0; @@ -232,21 +257,6 @@ string CountingBloomFilter::InternalState() const return fmt("%" PRIu64, cells->Hash()); } -IMPLEMENT_SERIAL(CountingBloomFilter, SER_COUNTINGBLOOMFILTER) - -bool CountingBloomFilter::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_COUNTINGBLOOMFILTER, BloomFilter); - return cells->Serialize(info); - } - -bool CountingBloomFilter::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BloomFilter); - cells = CounterVector::Unserialize(info); - return (cells != 0); - } - // TODO: Use partitioning in add/count to allow for reusing CMS bounds. void CountingBloomFilter::Add(const HashKey* key) { @@ -272,3 +282,19 @@ size_t CountingBloomFilter::Count(const HashKey* key) const return min; } + +broker::expected CountingBloomFilter::DoSerialize() const + { + auto c = cells->Serialize(); + return c; + } + +bool CountingBloomFilter::DoUnserialize(const broker::data& data) + { + auto c = CounterVector::Unserialize(data); + if ( ! c ) + return false; + + cells = c.release(); + return true; + } diff --git a/src/probabilistic/BloomFilter.h b/src/probabilistic/BloomFilter.h index 288a24d416..6f2362de44 100644 --- a/src/probabilistic/BloomFilter.h +++ b/src/probabilistic/BloomFilter.h @@ -4,6 +4,9 @@ #define PROBABILISTIC_BLOOMFILTER_H #include + +#include + #include "BitVector.h" #include "Hasher.h" @@ -11,15 +14,18 @@ namespace probabilistic { class CounterVector; +/** Types of derived BloomFilter classes. */ +enum BloomFilterType { Basic, Counting }; + /** * The abstract base class for Bloom filters. */ -class BloomFilter : public SerialObj { +class BloomFilter { public: /** * Destructor. */ - ~BloomFilter() override; + virtual ~BloomFilter(); /** * Adds an element to the Bloom filter. @@ -71,28 +77,10 @@ public: */ virtual string InternalState() const = 0; - /** - * Serializes the Bloom filter. - * - * @param info The serializaton information to use. - * - * @return True if successful. - */ - bool Serialize(SerialInfo* info) const; - - /** - * Unserializes a Bloom filter. - * - * @param info The serializaton information to use. - * - * @return The unserialized Bloom filter, or null if an error - * occured. - */ - static BloomFilter* Unserialize(UnserialInfo* info); + broker::expected Serialize() const; + static std::unique_ptr Unserialize(const broker::data& data); protected: - DECLARE_ABSTRACT_SERIAL(BloomFilter); - /** * Default constructor. */ @@ -105,6 +93,10 @@ protected: */ explicit BloomFilter(const Hasher* hasher); + virtual broker::expected DoSerialize() const = 0; + virtual bool DoUnserialize(const broker::data& data) = 0; + virtual BloomFilterType Type() const = 0; + const Hasher* hasher; }; @@ -165,7 +157,7 @@ public: string InternalState() const override; protected: - DECLARE_SERIAL(BasicBloomFilter); + friend class BloomFilter; /** * Default constructor. @@ -175,6 +167,10 @@ protected: // Overridden from BloomFilter. void Add(const HashKey* key) override; size_t Count(const HashKey* key) const override; + broker::expected DoSerialize() const override; + bool DoUnserialize(const broker::data& data) override; + BloomFilterType Type() const override + { return BloomFilterType::Basic; } private: BitVector* bits; @@ -210,7 +206,7 @@ public: string InternalState() const override; protected: - DECLARE_SERIAL(CountingBloomFilter); + friend class BloomFilter; /** * Default constructor. @@ -220,6 +216,10 @@ protected: // Overridden from BloomFilter. void Add(const HashKey* key) override; size_t Count(const HashKey* key) const override; + broker::expected DoSerialize() const override; + bool DoUnserialize(const broker::data& data) override; + BloomFilterType Type() const override + { return BloomFilterType::Counting; } private: CounterVector* cells; diff --git a/src/probabilistic/CardinalityCounter.cc b/src/probabilistic/CardinalityCounter.cc index 64715c39fd..748cdc9e48 100644 --- a/src/probabilistic/CardinalityCounter.cc +++ b/src/probabilistic/CardinalityCounter.cc @@ -6,7 +6,6 @@ #include "CardinalityCounter.h" #include "Reporter.h" -#include "Serializer.h" using namespace probabilistic; @@ -197,49 +196,48 @@ uint64_t CardinalityCounter::GetM() const return m; } -bool CardinalityCounter::Serialize(SerialInfo* info) const +broker::expected CardinalityCounter::Serialize() const { - bool valid = true; + broker::vector v = {m, V, alpha_m}; + v.reserve(3 + m); - valid &= SERIALIZE(m); - valid &= SERIALIZE(V); - valid &= SERIALIZE(alpha_m); + for ( size_t i = 0; i < m; ++i ) + v.emplace_back(static_cast(buckets[i])); - for ( unsigned int i = 0; i < m; i++ ) - valid &= SERIALIZE((char)buckets[i]); - - return valid; + return {std::move(v)}; } -CardinalityCounter* CardinalityCounter::Unserialize(UnserialInfo* info) +std::unique_ptr CardinalityCounter::Unserialize(const broker::data& data) { - uint64_t m; - uint64_t V; - double alpha_m; + auto v = caf::get_if(&data); + if ( ! (v && v->size() >= 3) ) + return nullptr; - bool valid = true; - valid &= UNSERIALIZE(&m); - valid &= UNSERIALIZE(&V); - valid &= UNSERIALIZE(&alpha_m); + auto m = caf::get_if(&(*v)[0]); + auto V = caf::get_if(&(*v)[1]); + auto alpha_m = caf::get_if(&(*v)[2]); - CardinalityCounter* c = new CardinalityCounter(m, V, alpha_m); + if ( ! (m && V && alpha_m) ) + return nullptr; + if ( v->size() != 3 + *m ) + return nullptr; - vector& buckets = c->buckets; + auto cc = std::unique_ptr(new CardinalityCounter(*m, *V, *alpha_m)); + if ( *m != cc->m ) + return nullptr; + if ( cc->buckets.size() != * m ) + return nullptr; - for ( unsigned int i = 0; i < m; i++ ) + for ( size_t i = 0; i < *m; ++i ) { - char c; - valid &= UNSERIALIZE(&c); - buckets[i] = (uint8_t)c; + auto x = caf::get_if(&(*v)[3 + i]); + if ( ! x ) + return nullptr; + + cc->buckets[i] = *x; } - if ( ! valid ) - { - delete c; - c = 0; - } - - return c; + return cc; } /** diff --git a/src/probabilistic/CardinalityCounter.h b/src/probabilistic/CardinalityCounter.h index cde2ec402b..a2d69d0809 100644 --- a/src/probabilistic/CardinalityCounter.h +++ b/src/probabilistic/CardinalityCounter.h @@ -84,24 +84,8 @@ public: */ bool Merge(CardinalityCounter* c); - /** - * Serializes the cardinality counter. - * - * @param info The serializaton information to use. - * - * @return True if successful. - */ - bool Serialize(SerialInfo* info) const; - - /** - * Unserializes a cardinality counter. - * - * @param info The serializaton information to use. - * - * @return The unserialized cardinality counter, or null if an error - * occured. - */ - static CardinalityCounter* Unserialize(UnserialInfo* info); + broker::expected Serialize() const; + static std::unique_ptr Unserialize(const broker::data& data); protected: /** diff --git a/src/probabilistic/CounterVector.cc b/src/probabilistic/CounterVector.cc index 8608015422..a847e06ea7 100644 --- a/src/probabilistic/CounterVector.cc +++ b/src/probabilistic/CounterVector.cc @@ -2,9 +2,9 @@ #include "CounterVector.h" +#include #include #include "BitVector.h" -#include "Serializer.h" using namespace probabilistic; @@ -153,46 +153,34 @@ CounterVector operator|(const CounterVector& x, const CounterVector& y) } -uint64 CounterVector::Hash() const +uint64_t CounterVector::Hash() const { return bits->Hash(); } -bool CounterVector::Serialize(SerialInfo* info) const +broker::expected CounterVector::Serialize() const { - return SerialObj::Serialize(info); + auto b = bits->Serialize(); + if ( ! b ) + return broker::ec::invalid_data; // Cannot serialize + + return {broker::vector{static_cast(width), std::move(*b)}}; } -CounterVector* CounterVector::Unserialize(UnserialInfo* info) +std::unique_ptr CounterVector::Unserialize(const broker::data& data) { - return reinterpret_cast(SerialObj::Unserialize(info, SER_COUNTERVECTOR)); + auto v = caf::get_if(&data); + if ( ! (v && v->size() >= 2) ) + return nullptr; + + auto width = caf::get_if(&(*v)[0]); + auto bits = BitVector::Unserialize((*v)[1]); + + auto cv = std::unique_ptr(new CounterVector()); + cv->width = *width; + cv->bits = bits.release(); + return cv; } -IMPLEMENT_SERIAL(CounterVector, SER_COUNTERVECTOR) -bool CounterVector::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_COUNTERVECTOR, SerialObj); - if ( ! bits->Serialize(info) ) - return false; - - return SERIALIZE(static_cast(width)); - } - -bool CounterVector::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(SerialObj); - - bits = BitVector::Unserialize(info); - if ( ! bits ) - return false; - - uint64 w; - if ( ! UNSERIALIZE(&w) ) - return false; - - width = static_cast(w); - - return true; - } diff --git a/src/probabilistic/CounterVector.h b/src/probabilistic/CounterVector.h index 422d172292..41674efd11 100644 --- a/src/probabilistic/CounterVector.h +++ b/src/probabilistic/CounterVector.h @@ -3,7 +3,10 @@ #ifndef PROBABILISTIC_COUNTERVECTOR_H #define PROBABILISTIC_COUNTERVECTOR_H -#include "SerialObj.h" +#include +#include + +#include namespace probabilistic { @@ -12,10 +15,10 @@ class BitVector; /** * A vector of counters, each of which has a fixed number of bits. */ -class CounterVector : public SerialObj { +class CounterVector { public: typedef size_t size_type; - typedef uint64 count_type; + typedef uint64_t count_type; /** * Constructs a counter vector having cells of a given width. @@ -38,7 +41,7 @@ public: /** * Destructor. */ - ~CounterVector() override; + virtual ~CounterVector(); /** * Increments a given cell. @@ -131,26 +134,10 @@ public: * * @return The hash. */ - uint64 Hash() const; + uint64_t Hash() const; - /** - * Serializes the bit vector. - * - * @param info The serializaton information to use. - * - * @return True if successful. - */ - bool Serialize(SerialInfo* info) const; - - /** - * Unserialize the counter vector. - * - * @param info The serializaton information to use. - * - * @return The unserialized counter vector, or null if an error - * occured. - */ - static CounterVector* Unserialize(UnserialInfo* info); + broker::expected Serialize() const; + static std::unique_ptr Unserialize(const broker::data& data); protected: friend CounterVector operator|(const CounterVector& x, @@ -158,8 +145,6 @@ protected: CounterVector() { } - DECLARE_SERIAL(CounterVector); - private: CounterVector& operator=(const CounterVector&); // Disable. diff --git a/src/probabilistic/Hasher.cc b/src/probabilistic/Hasher.cc index d21efbed41..2314166979 100644 --- a/src/probabilistic/Hasher.cc +++ b/src/probabilistic/Hasher.cc @@ -5,7 +5,6 @@ #include "Hasher.h" #include "NetVar.h" -#include "Serializer.h" #include "digest.h" #include "siphash24.h" @@ -41,58 +40,53 @@ Hasher::digest_vector Hasher::Hash(const HashKey* key) const return Hash(key->Key(), key->Size()); } -bool Hasher::Serialize(SerialInfo* info) const - { - return SerialObj::Serialize(info); - } - -Hasher* Hasher::Unserialize(UnserialInfo* info) - { - return reinterpret_cast(SerialObj::Unserialize(info, SER_HASHER)); - } - -bool Hasher::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_HASHER, SerialObj); - - if ( ! SERIALIZE(static_cast(k)) ) - return false; - - if ( ! SERIALIZE(static_cast(seed.h1)) ) - return false; - - return SERIALIZE(static_cast(seed.h2)); - } - -bool Hasher::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(SerialObj); - - uint16 serial_k; - if ( ! UNSERIALIZE(&serial_k) ) - return false; - - k = serial_k; - assert(k > 0); - - seed_t serial_seed; - if ( ! UNSERIALIZE(&serial_seed.h1) ) - return false; - - if ( ! UNSERIALIZE(&serial_seed.h2) ) - return false; - - seed = serial_seed; - - return true; - } - Hasher::Hasher(size_t arg_k, seed_t arg_seed) { k = arg_k; seed = arg_seed; } +broker::expected Hasher::Serialize() const + { + return {broker::vector{ + static_cast(Type()), static_cast(k), + seed.h1, seed.h2 }}; + } + +std::unique_ptr Hasher::Unserialize(const broker::data& data) + { + auto v = caf::get_if(&data); + + if ( ! (v && v->size() == 4) ) + return nullptr; + + auto type = caf::get_if(&(*v)[0]); + auto k = caf::get_if(&(*v)[1]); + auto h1 = caf::get_if(&(*v)[2]); + auto h2 = caf::get_if(&(*v)[3]); + + if ( ! (type && k && h1 && h2) ) + return nullptr; + + std::unique_ptr hasher; + + switch ( *type ) { + case Default: + hasher = std::unique_ptr(new DefaultHasher(*k, {*h1, *h2})); + break; + + case Double: + hasher = std::unique_ptr(new DoubleHasher(*k, {*h1, *h2})); + break; + } + + // Note that the derived classed don't hold any further state of + // their own. They reconstruct all their information from their + // constructors' arguments. + + return hasher; + } + UHF::UHF() { memset(&seed, 0, sizeof(seed)); @@ -167,31 +161,6 @@ bool DefaultHasher::Equals(const Hasher* other) const return hash_functions == o->hash_functions; } -IMPLEMENT_SERIAL(DefaultHasher, SER_DEFAULTHASHER) - -bool DefaultHasher::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_DEFAULTHASHER, Hasher); - - // Nothing to do here, the base class has all we need serialized already. - return true; - } - -bool DefaultHasher::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Hasher); - - hash_functions.clear(); - for ( size_t i = 0; i < K(); ++i ) - { - Hasher::seed_t s = Seed(); - s.h1 += bro_prng(i); - hash_functions.push_back(UHF(s)); - } - - return true; - } - DoubleHasher::DoubleHasher(size_t k, seed_t seed) : Hasher(k, seed), h1(seed + bro_prng(1)), h2(seed + bro_prng(2)) { @@ -223,22 +192,3 @@ bool DoubleHasher::Equals(const Hasher* other) const return h1 == o->h1 && h2 == o->h2; } -IMPLEMENT_SERIAL(DoubleHasher, SER_DOUBLEHASHER) - -bool DoubleHasher::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_DOUBLEHASHER, Hasher); - - // Nothing to do here, the base class has all we need serialized already. - return true; - } - -bool DoubleHasher::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(Hasher); - - h1 = UHF(Seed() + bro_prng(1)); - h2 = UHF(Seed() + bro_prng(2)); - - return true; - } diff --git a/src/probabilistic/Hasher.h b/src/probabilistic/Hasher.h index 7fd2e4fb2f..3218ec4d7a 100644 --- a/src/probabilistic/Hasher.h +++ b/src/probabilistic/Hasher.h @@ -3,16 +3,20 @@ #ifndef PROBABILISTIC_HASHER_H #define PROBABILISTIC_HASHER_H +#include + #include "Hash.h" -#include "SerialObj.h" namespace probabilistic { +/** Types of derived Hasher classes. */ +enum HasherType { Default, Double }; + /** * Abstract base class for hashers. A hasher creates a family of hash * functions to hash an element *k* times. */ -class Hasher : public SerialObj { +class Hasher { public: typedef hash_t digest; typedef std::vector digest_vector; @@ -43,7 +47,7 @@ public: /** * Destructor. */ - ~Hasher() override { } + virtual ~Hasher() { } /** * Computes hash values for an element. @@ -99,12 +103,10 @@ public: */ seed_t Seed() const { return seed; } - bool Serialize(SerialInfo* info) const; - static Hasher* Unserialize(UnserialInfo* info); + broker::expected Serialize() const; + static std::unique_ptr Unserialize(const broker::data& data); protected: - DECLARE_ABSTRACT_SERIAL(Hasher); - Hasher() { } /** @@ -116,6 +118,8 @@ protected: */ Hasher(size_t arg_k, seed_t arg_seed); + virtual HasherType Type() const = 0; + private: size_t k; seed_t seed; @@ -181,6 +185,9 @@ public: return ! (x == y); } + broker::expected Serialize() const; + static UHF Unserialize(const broker::data& data); + private: static size_t compute_seed(Hasher::seed_t seed); @@ -208,11 +215,12 @@ public: DefaultHasher* Clone() const final; bool Equals(const Hasher* other) const final; - DECLARE_SERIAL(DefaultHasher); - private: DefaultHasher() { } + HasherType Type() const override + { return HasherType::Default; } + std::vector hash_functions; }; @@ -236,11 +244,12 @@ public: DoubleHasher* Clone() const final; bool Equals(const Hasher* other) const final; - DECLARE_SERIAL(DoubleHasher); - private: DoubleHasher() { } + HasherType Type() const override + { return HasherType::Double; } + UHF h1; UHF h2; }; diff --git a/src/probabilistic/Topk.cc b/src/probabilistic/Topk.cc index c64357c17f..56b9030f21 100644 --- a/src/probabilistic/Topk.cc +++ b/src/probabilistic/Topk.cc @@ -1,15 +1,13 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include "broker/Data.h" #include "probabilistic/Topk.h" #include "CompHash.h" #include "Reporter.h" -#include "Serializer.h" #include "NetVar.h" namespace probabilistic { -IMPLEMENT_SERIAL(TopkVal, SER_TOPK_VAL); - static void topk_element_hash_delete_func(void* val) { Element* e = (Element*) val; @@ -190,109 +188,6 @@ Val* TopkVal::DoClone(CloneState* state) return state->NewClone(this, clone); } -bool TopkVal::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_TOPK_VAL, OpaqueVal); - - bool v = true; - - v &= SERIALIZE(size); - v &= SERIALIZE(numElements); - v &= SERIALIZE(pruned); - - bool type_present = (type != 0); - v &= SERIALIZE(type_present); - - if ( type_present ) - v &= type->Serialize(info); - else - assert(numElements == 0); - - uint64_t i = 0; - std::list::const_iterator it = buckets.begin(); - while ( it != buckets.end() ) - { - Bucket* b = *it; - uint32_t elements_count = b->elements.size(); - v &= SERIALIZE(elements_count); - v &= SERIALIZE(b->count); - - std::list::const_iterator eit = b->elements.begin(); - while ( eit != b->elements.end() ) - { - Element* element = *eit; - v &= SERIALIZE(element->epsilon); - v &= element->value->Serialize(info); - - eit++; - i++; - } - - it++; - } - - assert(i == numElements); - - return v; - } - -bool TopkVal::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(OpaqueVal); - - bool v = true; - - v &= UNSERIALIZE(&size); - v &= UNSERIALIZE(&numElements); - v &= UNSERIALIZE(&pruned); - - bool type_present = false; - v &= UNSERIALIZE(&type_present); - if ( type_present ) - { - BroType* deserialized_type = BroType::Unserialize(info); - - Typify(deserialized_type); - Unref(deserialized_type); - assert(type); - } - else - assert(numElements == 0); - - uint64_t i = 0; - while ( i < numElements ) - { - Bucket* b = new Bucket(); - uint32_t elements_count; - v &= UNSERIALIZE(&elements_count); - v &= UNSERIALIZE(&b->count); - b->bucketPos = buckets.insert(buckets.end(), b); - - for ( uint64_t j = 0; j < elements_count; j++ ) - { - Element* e = new Element(); - v &= UNSERIALIZE(&e->epsilon); - e->value = Val::Unserialize(info, type); - e->parent = b; - - b->elements.insert(b->elements.end(), e); - - HashKey* key = GetHash(e->value); - assert (elementDict->Lookup(key) == 0); - - elementDict->Insert(key, e); - delete key; - - i++; - } - } - - assert(i == numElements); - - return v; - } - - VectorVal* TopkVal::GetTopK(int k) const // returns vector { if ( numElements == 0 ) @@ -511,4 +406,126 @@ void TopkVal::IncrementCounter(Element* e, unsigned int count) } } -}; +IMPLEMENT_OPAQUE_VALUE(TopkVal) + +broker::expected TopkVal::DoSerialize() const + { + broker::vector d = {size, numElements, pruned}; + + if ( type ) + { + auto t = SerializeType(type); + if ( ! t ) + return broker::ec::invalid_data; + + d.emplace_back(std::move(*t)); + } + else + d.emplace_back(broker::none()); + + uint64_t i = 0; + std::list::const_iterator it = buckets.begin(); + while ( it != buckets.end() ) + { + Bucket* b = *it; + uint32_t elements_count = b->elements.size(); + + d.emplace_back(static_cast(b->elements.size())); + d.emplace_back(b->count); + + std::list::const_iterator eit = b->elements.begin(); + while ( eit != b->elements.end() ) + { + Element* element = *eit; + d.emplace_back(element->epsilon); + auto v = bro_broker::val_to_data(element->value); + if ( ! v ) + return broker::ec::invalid_data; + + d.emplace_back(*v); + + eit++; + i++; + } + + it++; + } + + assert(i == numElements); + return {std::move(d)}; + } + + +bool TopkVal::DoUnserialize(const broker::data& data) + { + auto v = caf::get_if(&data); + + if ( ! (v && v->size() >= 4) ) + return false; + + auto size_ = caf::get_if(&(*v)[0]); + auto numElements_ = caf::get_if(&(*v)[1]); + auto pruned_ = caf::get_if(&(*v)[2]); + + if ( ! (size_ && numElements_ && pruned_) ) + return false; + + size = *size_; + numElements = *numElements_; + pruned = *pruned_; + + auto no_type = caf::get_if(&(*v)[3]); + if ( ! no_type ) + { + BroType* t = UnserializeType((*v)[3]); + if ( ! t ) + return false; + + Typify(t); + Unref(t); + } + + uint64_t i = 0; + uint64_t idx = 4; + + while ( i < numElements ) + { + Bucket* b = new Bucket(); + auto elements_count = caf::get_if(&(*v)[idx++]); + auto count = caf::get_if(&(*v)[idx++]); + + if ( ! (elements_count && count) ) + return false; + + b->count = *count; + b->bucketPos = buckets.insert(buckets.end(), b); + + for ( uint64_t j = 0; j < *elements_count; j++ ) + { + Element* e = new Element(); + auto epsilon = caf::get_if(&(*v)[idx++]); + Val* val = bro_broker::data_to_val((*v)[idx++], type); + + if ( ! (epsilon && val) ) + return false; + + e->epsilon = *epsilon; + e->value = val; + e->parent = b; + + b->elements.insert(b->elements.end(), e); + + HashKey* key = GetHash(e->value); + assert (elementDict->Lookup(key) == 0); + + elementDict->Insert(key, e); + delete key; + + i++; + } + } + + assert(i == numElements); + return true; + } +} diff --git a/src/probabilistic/Topk.h b/src/probabilistic/Topk.h index 22c7aef1e1..24d05e12af 100644 --- a/src/probabilistic/Topk.h +++ b/src/probabilistic/Topk.h @@ -131,6 +131,8 @@ public: */ Val* DoClone(CloneState* state) override; + DECLARE_OPAQUE_VALUE(TopkVal) + protected: /** * Construct an empty TopkVal. Only used for deserialization @@ -170,8 +172,6 @@ private: uint64 size; // how many elements are we tracking? uint64 numElements; // how many elements do we have at the moment bool pruned; // was this data structure pruned? - - DECLARE_SERIAL(TopkVal); }; }; diff --git a/src/probabilistic/cardinality-counter.bif b/src/probabilistic/cardinality-counter.bif index 2fa7953c9e..1e12765b57 100644 --- a/src/probabilistic/cardinality-counter.bif +++ b/src/probabilistic/cardinality-counter.bif @@ -90,7 +90,7 @@ function hll_cardinality_merge_into%(handle1: opaque of cardinality, handle2: op bool res = h1->Merge(h2); if ( ! res ) { - reporter->Error("Carinality counters with different parameters cannot be merged"); + reporter->Error("Cardinality counters with different parameters cannot be merged"); return val_mgr->GetBool(0); } diff --git a/src/zeek.bif b/src/zeek.bif index 10f9bfa4bd..139c77f955 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -4912,44 +4912,6 @@ function uninstall_dst_net_filter%(snet: subnet%) : bool return val_mgr->GetBool(sessions->GetPacketFilter()->RemoveDst(snet)); %} -## Writes the binary event stream generated by the core to a given file. -## Use the ``-x `` command line switch to replay saved events. -## -## filename: The name of the file which stores the events. -## -## Returns: True if opening the target file succeeds. -## -## .. zeek:see:: capture_state_updates -function capture_events%(filename: string%) : bool - %{ - if ( ! event_serializer ) - event_serializer = new FileSerializer(); - else - event_serializer->Close(); - - return val_mgr->GetBool(event_serializer->Open( - (const char*) filename->CheckString())); - %} - -## Writes state updates generated by synchronized variables to a -## file. -## -## filename: The name of the file which stores the state updates. -## -## Returns: True if opening the target file succeeds. -## -## .. zeek:see:: capture_events -function capture_state_updates%(filename: string%) : bool - %{ - if ( ! state_serializer ) - state_serializer = new FileSerializer(); - else - state_serializer->Close(); - - return val_mgr->GetBool(state_serializer->Open( - (const char*) filename->CheckString())); - %} - ## Checks whether the last raised event came from a remote peer. ## ## Returns: True if the last raised event came from a remote peer. diff --git a/testing/btest/Baseline/bifs.capture_state_updates/out b/testing/btest/Baseline/bifs.capture_state_updates/out deleted file mode 100644 index 62a6e3c9df..0000000000 --- a/testing/btest/Baseline/bifs.capture_state_updates/out +++ /dev/null @@ -1 +0,0 @@ -T diff --git a/testing/btest/Baseline/broker.opaque/.stderr b/testing/btest/Baseline/broker.opaque/.stderr new file mode 100644 index 0000000000..bf07a71a21 --- /dev/null +++ b/testing/btest/Baseline/broker.opaque/.stderr @@ -0,0 +1 @@ +error: incompatible Bloom filter types diff --git a/testing/btest/Baseline/broker.opaque/out b/testing/btest/Baseline/broker.opaque/out new file mode 100644 index 0000000000..35bf821c47 --- /dev/null +++ b/testing/btest/Baseline/broker.opaque/out @@ -0,0 +1,53 @@ +============ Topk +[b, a, c] +[b, a, c] +============ HLL +3.000069 +3.000069 +3.000069 +============ Bloom +0 +1 +0 +1 +============ Hashes +5b9164ad6f496d9dee12ec7634ce253f +5b9164ad6f496d9dee12ec7634ce253f +30ae97492ce1da88d0e7117ace0a60a6f9e1e0bc +30ae97492ce1da88d0e7117ace0a60a6f9e1e0bc +25b6746d5172ed6352966a013d93ac846e1110d5a25e8f183b5931f4688842a1 +25b6746d5172ed6352966a013d93ac846e1110d5a25e8f183b5931f4688842a1 +============ X509 +[version=3, serial=040000000001154B5AC394, subject=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, issuer=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, cn=GlobalSign Root CA, not_valid_before=904651200.0, not_valid_after=1832673600.0, key_alg=rsaEncryption, sig_alg=sha1WithRSAEncryption, key_type=rsa, key_length=2048, exponent=65537, curve=] +[version=3, serial=040000000001154B5AC394, subject=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, issuer=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, cn=GlobalSign Root CA, not_valid_before=904651200.0, not_valid_after=1832673600.0, key_alg=rsaEncryption, sig_alg=sha1WithRSAEncryption, key_type=rsa, key_length=2048, exponent=65537, curve=] +============ Entropy +[entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027] +[entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027] +============ broker::Data +broker::data{{hi, there}} +broker::data{{hi, there}} +T +============ broker::Set +| [data=broker::data{!}] | [data=broker::data{!}] + > [data=broker::data{hi}] | [data=broker::data{hi}] +| [data=broker::data{hi}] | [data=broker::data{hi}] + > [data=broker::data{there}] | [data=broker::data{there}] +| [data=broker::data{there}] | [data=broker::data{there}] +============ broker::Table +| [key=[data=broker::data{!}], val=[data=broker::data{30}]] | [key=[data=broker::data{!}], val=[data=broker::data{30}]] + > [key=[data=broker::data{hi}], val=[data=broker::data{10}]] | [key=[data=broker::data{hi}], val=[data=broker::data{10}]] +| [key=[data=broker::data{hi}], val=[data=broker::data{10}]] | [key=[data=broker::data{hi}], val=[data=broker::data{10}]] + > [key=[data=broker::data{there}], val=[data=broker::data{20}]] | [key=[data=broker::data{there}], val=[data=broker::data{20}]] +| [key=[data=broker::data{there}], val=[data=broker::data{20}]] | [key=[data=broker::data{there}], val=[data=broker::data{20}]] +============ broker::Vector +| [data=broker::data{hi}] | [data=broker::data{hi}] + > [data=broker::data{there}] | [data=broker::data{there}] +| [data=broker::data{there}] | [data=broker::data{there}] + > [data=broker::data{!}] | [data=broker::data{!}] +| [data=broker::data{!}] | [data=broker::data{!}] +============ broker::Record +| [data=broker::data{hi}] | [data=broker::data{hi}] + > [data=broker::data{there}] | [data=broker::data{there}] +| [data=broker::data{there}] | [data=broker::data{there}] + > [data=broker::data{!}] | [data=broker::data{!}] +| [data=broker::data{!}] | [data=broker::data{!}] diff --git a/testing/btest/Baseline/core.leaks.broker.data/zeek..stdout b/testing/btest/Baseline/core.leaks.broker.data/zeek..stdout index a58dc4a480..033e8af5f2 100644 --- a/testing/btest/Baseline/core.leaks.broker.data/zeek..stdout +++ b/testing/btest/Baseline/core.leaks.broker.data/zeek..stdout @@ -116,3 +116,5 @@ T T 3 [data=broker::data{goodbye}] +*************************** +T diff --git a/testing/btest/Baseline/language.copy-all-opaques/out b/testing/btest/Baseline/language.copy-all-opaques/out index ad38ca1a8d..68b12cecac 100644 --- a/testing/btest/Baseline/language.copy-all-opaques/out +++ b/testing/btest/Baseline/language.copy-all-opaques/out @@ -4,6 +4,7 @@ ============ HLL 3.000069 3.000069 +3.000069 ============ Bloom 0 1 diff --git a/testing/btest/bifs/capture_state_updates.zeek b/testing/btest/bifs/capture_state_updates.zeek deleted file mode 100644 index b9a802a53d..0000000000 --- a/testing/btest/bifs/capture_state_updates.zeek +++ /dev/null @@ -1,9 +0,0 @@ -# -# @TEST-EXEC: zeek -b %INPUT >out -# @TEST-EXEC: btest-diff out -# @TEST-EXEC: test -f testfile - -event zeek_init() - { - print capture_state_updates("testfile"); - } diff --git a/testing/btest/broker/opaque.zeek b/testing/btest/broker/opaque.zeek new file mode 100644 index 0000000000..e0a3bef6c7 --- /dev/null +++ b/testing/btest/broker/opaque.zeek @@ -0,0 +1,162 @@ +# @TEST-EXEC: zeek -b %INPUT >out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff .stderr + +event zeek_init() + { + print "============ Topk"; + local k1: opaque of topk = topk_init(4); + topk_add(k1, "a"); + topk_add(k1, "b"); + topk_add(k1, "b"); + topk_add(k1, "c"); + local k2 = Broker::__opaque_clone_through_serialization(k1); + print topk_get_top(k1, 5); + topk_add(k1, "shoulnotshowup"); + print topk_get_top(k2, 5); + + print "============ HLL"; + local c1 = hll_cardinality_init(0.01, 0.95); + hll_cardinality_add(c1, 2001); + hll_cardinality_add(c1, 2002); + hll_cardinality_add(c1, 2003); + + print hll_cardinality_estimate(c1); + local c2 = Broker::__opaque_clone_through_serialization(c1); + hll_cardinality_add(c1, 2004); + print hll_cardinality_estimate(c2); + + local c3 = hll_cardinality_init(0.01, 0.95); + hll_cardinality_merge_into(c3, c2); + print hll_cardinality_estimate(c3); + + print "============ Bloom"; + local bf_cnt = bloomfilter_basic_init(0.1, 1000); + bloomfilter_add(bf_cnt, 42); + bloomfilter_add(bf_cnt, 84); + bloomfilter_add(bf_cnt, 168); + print bloomfilter_lookup(bf_cnt, 0); + print bloomfilter_lookup(bf_cnt, 42); + local bf_copy = Broker::__opaque_clone_through_serialization(bf_cnt); + bloomfilter_add(bf_cnt, 0); + print bloomfilter_lookup(bf_copy, 0); + print bloomfilter_lookup(bf_copy, 42); + # check that typefication transfered. + bloomfilter_add(bf_copy, 0.5); # causes stderr output "error: incompatible Bloom filter types" + + print "============ Hashes"; + local md5a = md5_hash_init(); + md5_hash_update(md5a, "one"); + local md5b = Broker::__opaque_clone_through_serialization(md5a); + md5_hash_update(md5a, "two"); + md5_hash_update(md5b, "two"); + print md5_hash_finish(md5a); + print md5_hash_finish(md5b); + + local sha1a = sha1_hash_init(); + sha1_hash_update(sha1a, "one"); + local sha1b = Broker::__opaque_clone_through_serialization(sha1a); + sha1_hash_update(sha1a, "two"); + sha1_hash_update(sha1b, "two"); + print sha1_hash_finish(sha1a); + print sha1_hash_finish(sha1b); + + local sha256a = sha256_hash_init(); + sha256_hash_update(sha256a, "one"); + local sha256b = Broker::__opaque_clone_through_serialization(sha256a); + sha256_hash_update(sha256a, "two"); + sha256_hash_update(sha256b, "two"); + print sha256_hash_finish(sha256a); + print sha256_hash_finish(sha256b); + + print "============ X509"; + local x509 = x509_from_der("\x30\x82\x03\x75\x30\x82\x02\x5D\xA0\x03\x02\x01\x02\x02\x0B\x04\x00\x00\x00\x00\x01\x15\x4B\x5A\xC3\x94\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x57\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x42\x45\x31\x19\x30\x17\x06\x03\x55\x04\x0A\x13\x10\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x6E\x76\x2D\x73\x61\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x13\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x1E\x17\x0D\x39\x38\x30\x39\x30\x31\x31\x32\x30\x30\x30\x30\x5A\x17\x0D\x32\x38\x30\x31\x32\x38\x31\x32\x30\x30\x30\x30\x5A\x30\x57\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x42\x45\x31\x19\x30\x17\x06\x03\x55\x04\x0A\x13\x10\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x6E\x76\x2D\x73\x61\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x13\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xDA\x0E\xE6\x99\x8D\xCE\xA3\xE3\x4F\x8A\x7E\xFB\xF1\x8B\x83\x25\x6B\xEA\x48\x1F\xF1\x2A\xB0\xB9\x95\x11\x04\xBD\xF0\x63\xD1\xE2\x67\x66\xCF\x1C\xDD\xCF\x1B\x48\x2B\xEE\x8D\x89\x8E\x9A\xAF\x29\x80\x65\xAB\xE9\xC7\x2D\x12\xCB\xAB\x1C\x4C\x70\x07\xA1\x3D\x0A\x30\xCD\x15\x8D\x4F\xF8\xDD\xD4\x8C\x50\x15\x1C\xEF\x50\xEE\xC4\x2E\xF7\xFC\xE9\x52\xF2\x91\x7D\xE0\x6D\xD5\x35\x30\x8E\x5E\x43\x73\xF2\x41\xE9\xD5\x6A\xE3\xB2\x89\x3A\x56\x39\x38\x6F\x06\x3C\x88\x69\x5B\x2A\x4D\xC5\xA7\x54\xB8\x6C\x89\xCC\x9B\xF9\x3C\xCA\xE5\xFD\x89\xF5\x12\x3C\x92\x78\x96\xD6\xDC\x74\x6E\x93\x44\x61\xD1\x8D\xC7\x46\xB2\x75\x0E\x86\xE8\x19\x8A\xD5\x6D\x6C\xD5\x78\x16\x95\xA2\xE9\xC8\x0A\x38\xEB\xF2\x24\x13\x4F\x73\x54\x93\x13\x85\x3A\x1B\xBC\x1E\x34\xB5\x8B\x05\x8C\xB9\x77\x8B\xB1\xDB\x1F\x20\x91\xAB\x09\x53\x6E\x90\xCE\x7B\x37\x74\xB9\x70\x47\x91\x22\x51\x63\x16\x79\xAE\xB1\xAE\x41\x26\x08\xC8\x19\x2B\xD1\x46\xAA\x48\xD6\x64\x2A\xD7\x83\x34\xFF\x2C\x2A\xC1\x6C\x19\x43\x4A\x07\x85\xE7\xD3\x7C\xF6\x21\x68\xEF\xEA\xF2\x52\x9F\x7F\x93\x90\xCF\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x60\x7B\x66\x1A\x45\x0D\x97\xCA\x89\x50\x2F\x7D\x04\xCD\x34\xA8\xFF\xFC\xFD\x4B\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\xD6\x73\xE7\x7C\x4F\x76\xD0\x8D\xBF\xEC\xBA\xA2\xBE\x34\xC5\x28\x32\xB5\x7C\xFC\x6C\x9C\x2C\x2B\xBD\x09\x9E\x53\xBF\x6B\x5E\xAA\x11\x48\xB6\xE5\x08\xA3\xB3\xCA\x3D\x61\x4D\xD3\x46\x09\xB3\x3E\xC3\xA0\xE3\x63\x55\x1B\xF2\xBA\xEF\xAD\x39\xE1\x43\xB9\x38\xA3\xE6\x2F\x8A\x26\x3B\xEF\xA0\x50\x56\xF9\xC6\x0A\xFD\x38\xCD\xC4\x0B\x70\x51\x94\x97\x98\x04\xDF\xC3\x5F\x94\xD5\x15\xC9\x14\x41\x9C\xC4\x5D\x75\x64\x15\x0D\xFF\x55\x30\xEC\x86\x8F\xFF\x0D\xEF\x2C\xB9\x63\x46\xF6\xAA\xFC\xDF\xBC\x69\xFD\x2E\x12\x48\x64\x9A\xE0\x95\xF0\xA6\xEF\x29\x8F\x01\xB1\x15\xB5\x0C\x1D\xA5\xFE\x69\x2C\x69\x24\x78\x1E\xB3\xA7\x1C\x71\x62\xEE\xCA\xC8\x97\xAC\x17\x5D\x8A\xC2\xF8\x47\x86\x6E\x2A\xC4\x56\x31\x95\xD0\x67\x89\x85\x2B\xF9\x6C\xA6\x5D\x46\x9D\x0C\xAA\x82\xE4\x99\x51\xDD\x70\xB7\xDB\x56\x3D\x61\xE4\x6A\xE1\x5C\xD6\xF6\xFE\x3D\xDE\x41\xCC\x07\xAE\x63\x52\xBF\x53\x53\xF4\x2B\xE9\xC7\xFD\xB6\xF7\x82\x5F\x85\xD2\x41\x18\xDB\x81\xB3\x04\x1C\xC5\x1F\xA4\x80\x6F\x15\x20\xC9\xDE\x0C\x88\x0A\x1D\xD6\x66\x55\xE2\xFC\x48\xC9\x29\x26\x69\xE0"); + local x5092 = Broker::__opaque_clone_through_serialization(x509); + print x509_parse(x509); + print x509_parse(x5092); + + print "============ Entropy"; + local handle = entropy_test_init(); + entropy_test_add(handle, "dh3Hie02uh^s#Sdf9L3frd243h$d78r2G4cM6*Q05d(7rh46f!0|4-f"); + local handle2 = Broker::__opaque_clone_through_serialization(handle); + print entropy_test_finish(handle); + print entropy_test_finish(handle2); + + print "============ broker::Data"; + local s1: Broker::Data = Broker::set_create(); + Broker::set_insert(s1, "hi"); + Broker::set_insert(s1, "there"); + local d2 = Broker::__opaque_clone_through_serialization(s1$data); + print s1$data; + print d2; + print same_object(s1$data, d2) == F; + + print "============ broker::Set"; + local cs = Broker::set_create(); + Broker::set_insert(cs, "hi"); + Broker::set_insert(cs, "there"); + Broker::set_insert(cs, "!"); + + local i = Broker::set_iterator(cs); + while ( ! Broker::set_iterator_last(i) ) + { + local ci = Broker::__opaque_clone_through_serialization(i); + print fmt("| %s | %s", Broker::set_iterator_value(i), Broker::set_iterator_value(ci)); + Broker::set_iterator_next(i); + Broker::set_iterator_next(ci); + if ( ! Broker::set_iterator_last(i) ) + print fmt(" > %s | %s", Broker::set_iterator_value(i), Broker::set_iterator_value(ci)); + } + + print "============ broker::Table"; + local ct = Broker::table_create(); + Broker::table_insert(ct, "hi", 10); + Broker::table_insert(ct, "there", 20); + Broker::table_insert(ct, "!", 30); + + local j = Broker::table_iterator(ct); + while ( ! Broker::table_iterator_last(j) ) + { + local cj = Broker::__opaque_clone_through_serialization(j); + print fmt("| %s | %s", Broker::table_iterator_value(j), Broker::table_iterator_value(cj)); + Broker::table_iterator_next(j); + Broker::table_iterator_next(cj); + if ( ! Broker::table_iterator_last(j) ) + print fmt(" > %s | %s", Broker::table_iterator_value(j), Broker::table_iterator_value(cj)); + } + + print "============ broker::Vector"; + local cv = Broker::vector_create(); + Broker::vector_insert(cv, 0, "hi"); + Broker::vector_insert(cv, 1, "there"); + Broker::vector_insert(cv, 2, "!"); + + local k = Broker::vector_iterator(cv); + while ( ! Broker::vector_iterator_last(k) ) + { + local ck = Broker::__opaque_clone_through_serialization(k); + print fmt("| %s | %s", Broker::vector_iterator_value(k), Broker::vector_iterator_value(ck)); + Broker::vector_iterator_next(k); + Broker::vector_iterator_next(ck); + if ( ! Broker::vector_iterator_last(k) ) + print fmt(" > %s | %s", Broker::vector_iterator_value(k), Broker::vector_iterator_value(ck)); + } + + print "============ broker::Record"; + local cr = Broker::record_create(3); + Broker::record_assign(cr, 0, "hi"); + Broker::record_assign(cr, 1, "there"); + Broker::record_assign(cr, 2, "!"); + + local l = Broker::record_iterator(cr); + while ( ! Broker::record_iterator_last(l) ) + { + local cl = Broker::__opaque_clone_through_serialization(l); + print fmt("| %s | %s", Broker::record_iterator_value(l), Broker::record_iterator_value(cl)); + Broker::record_iterator_next(l); + Broker::record_iterator_next(cl); + if ( ! Broker::record_iterator_last(l) ) + print fmt(" > %s | %s", Broker::record_iterator_value(l), Broker::record_iterator_value(cl)); + } + + } diff --git a/testing/btest/core/leaks/broker/data.zeek b/testing/btest/core/leaks/broker/data.zeek index 9f9daadee0..c70120b360 100644 --- a/testing/btest/core/leaks/broker/data.zeek +++ b/testing/btest/core/leaks/broker/data.zeek @@ -258,4 +258,13 @@ print Broker::record_size(cr); print Broker::record_assign(cr, 1, ("goodbye")); print Broker::record_size(cr); print Broker::record_lookup(cr, 1); + +print "***************************"; + +### Test an opaque value + +local k1: opaque of topk = topk_init(4); +topk_add(k1, "a"); +print Broker::data(k1) is opaque of topk; +Broker::data(k1) as opaque of topk; } diff --git a/testing/btest/core/leaks/copy-all-opaques.zeek b/testing/btest/core/leaks/copy-all-opaques.zeek index c70238dd07..259d971bb2 100644 --- a/testing/btest/core/leaks/copy-all-opaques.zeek +++ b/testing/btest/core/leaks/copy-all-opaques.zeek @@ -1,11 +1,18 @@ # @TEST-GROUP: leaks # @TEST-REQUIRES: zeek --help 2>&1 | grep -q mem-leaks -# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run zeek zeek -m -b %INPUT +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run zeek zeek -m -b -r $TRACES/wikipedia.trace %INPUT # @TEST-EXEC: btest-bg-wait 60 -event zeek_init() +global did_it = F; + +event new_connection(c: connection) { + if ( did_it ) + return; + + did_it = T; + print "============ Topk"; local k1: opaque of topk = topk_init(4); topk_add(k1, "a"); diff --git a/testing/btest/core/leaks/copy-all-types.zeek b/testing/btest/core/leaks/copy-all-types.zeek index f83b4ee53e..3d5d9b958f 100644 --- a/testing/btest/core/leaks/copy-all-types.zeek +++ b/testing/btest/core/leaks/copy-all-types.zeek @@ -2,7 +2,7 @@ # @TEST-GROUP: leaks # @TEST-REQUIRES: zeek --help 2>&1 | grep -q mem-leaks -# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run zeek zeek -m -b %INPUT +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run zeek zeek -m -b -r $TRACES/wikipedia.trace %INPUT # @TEST-EXEC: btest-bg-wait 60 type MyEnum: enum { ENUMME }; @@ -125,9 +125,15 @@ function compare_otr(a: TestRecord, b: TestRecord): bool return T; } +global did_it = F; -event zeek_init() +event new_connection(c: connection) { + if ( did_it ) + return; + + did_it = T; + local i1 = -42; local i2 = copy(i1); check(i1, i2, i1 == i2, T); diff --git a/testing/btest/language/copy-all-opaques.zeek b/testing/btest/language/copy-all-opaques.zeek index 25ca89fd80..06b4a07471 100644 --- a/testing/btest/language/copy-all-opaques.zeek +++ b/testing/btest/language/copy-all-opaques.zeek @@ -27,6 +27,10 @@ event zeek_init() hll_cardinality_add(c1, 2004); print hll_cardinality_estimate(c2); + local c3 = hll_cardinality_init(0.01, 0.95); + hll_cardinality_merge_into(c3, c2); + print hll_cardinality_estimate(c3); + print "============ Bloom"; local bf_cnt = bloomfilter_basic_init(0.1, 1000); bloomfilter_add(bf_cnt, 42); diff --git a/testing/btest/scripts/base/protocols/ssl/ocsp-http-get.test b/testing/btest/scripts/base/protocols/ssl/ocsp-http-get.test index 747c1a667c..6b4b034c69 100644 --- a/testing/btest/scripts/base/protocols/ssl/ocsp-http-get.test +++ b/testing/btest/scripts/base/protocols/ssl/ocsp-http-get.test @@ -32,7 +32,7 @@ event ocsp_response_status(f: fa_file, status: string) print "ocsp_response_status", status; } -event ocsp_response_bytes(f: fa_file, resp_ref: opaque of ocsp_resp, status: string, version: count, responderId: string, producedAt: time, signatureAlgorithm: string, certs: x509_opaque_vector) +event ocsp_response_bytes(f: fa_file, status: string, version: count, responderId: string, producedAt: time, signatureAlgorithm: string, certs: x509_opaque_vector) { print "ocsp_response_bytes", status, version, responderId, producedAt, signatureAlgorithm; } diff --git a/testing/btest/scripts/base/protocols/ssl/ocsp-request-only.test b/testing/btest/scripts/base/protocols/ssl/ocsp-request-only.test index 348da52f96..5106a17c75 100644 --- a/testing/btest/scripts/base/protocols/ssl/ocsp-request-only.test +++ b/testing/btest/scripts/base/protocols/ssl/ocsp-request-only.test @@ -31,7 +31,7 @@ event ocsp_response_status(f: fa_file, status: string) print "ocsp_response_status", status; } -event ocsp_response_bytes(f: fa_file, resp_ref: opaque of ocsp_resp, status: string, version: count, responderId: string, producedAt: time, signatureAlgorithm: string, certs: x509_opaque_vector) +event ocsp_response_bytes(f: fa_file, status: string, version: count, responderId: string, producedAt: time, signatureAlgorithm: string, certs: x509_opaque_vector) { print "ocsp_response_bytes", status, version, responderId, producedAt, signatureAlgorithm; } diff --git a/testing/btest/scripts/base/protocols/ssl/ocsp-request-response.test b/testing/btest/scripts/base/protocols/ssl/ocsp-request-response.test index 1942b57bad..67f62e451d 100644 --- a/testing/btest/scripts/base/protocols/ssl/ocsp-request-response.test +++ b/testing/btest/scripts/base/protocols/ssl/ocsp-request-response.test @@ -32,7 +32,7 @@ event ocsp_response_status(f: fa_file, status: string) print "ocsp_response_status", status; } -event ocsp_response_bytes(f: fa_file, resp_ref: opaque of ocsp_resp, status: string, version: count, responderId: string, producedAt: time, signatureAlgorithm: string, certs: x509_opaque_vector) +event ocsp_response_bytes(f: fa_file, status: string, version: count, responderId: string, producedAt: time, signatureAlgorithm: string, certs: x509_opaque_vector) { print "ocsp_response_bytes", status, version, responderId, producedAt, signatureAlgorithm; } diff --git a/testing/btest/scripts/base/protocols/ssl/ocsp-response-only.test b/testing/btest/scripts/base/protocols/ssl/ocsp-response-only.test index 871ac59a34..568915d7aa 100644 --- a/testing/btest/scripts/base/protocols/ssl/ocsp-response-only.test +++ b/testing/btest/scripts/base/protocols/ssl/ocsp-response-only.test @@ -32,7 +32,7 @@ event ocsp_response_status(f: fa_file, status: string) print "ocsp_response_status", status; } -event ocsp_response_bytes(f: fa_file, resp_ref: opaque of ocsp_resp, status: string, version: count, responderId: string, producedAt: time, signatureAlgorithm: string, certs: x509_opaque_vector) +event ocsp_response_bytes(f: fa_file, status: string, version: count, responderId: string, producedAt: time, signatureAlgorithm: string, certs: x509_opaque_vector) { print "ocsp_response_bytes", status, version, responderId, producedAt, signatureAlgorithm; } diff --git a/testing/btest/scripts/base/protocols/ssl/ocsp-revoked.test b/testing/btest/scripts/base/protocols/ssl/ocsp-revoked.test index 5f5f1486ea..e26bae59a5 100644 --- a/testing/btest/scripts/base/protocols/ssl/ocsp-revoked.test +++ b/testing/btest/scripts/base/protocols/ssl/ocsp-revoked.test @@ -32,7 +32,7 @@ event ocsp_response_status(f: fa_file, status: string) print "ocsp_response_status", status; } -event ocsp_response_bytes(f: fa_file, resp_ref: opaque of ocsp_resp, status: string, version: count, responderId: string, producedAt: time, signatureAlgorithm: string, certs: x509_opaque_vector) +event ocsp_response_bytes(f: fa_file, status: string, version: count, responderId: string, producedAt: time, signatureAlgorithm: string, certs: x509_opaque_vector) { print "ocsp_response_bytes", status, version, responderId, producedAt, signatureAlgorithm; }