Initial import of svn+ssh:://svn.icir.org/bro/trunk/bro as of r7088

src/Serializer.h

@@ -0,0 +1,437 @@
+// $Id: Serializer.h 6752 2009-06-14 04:24:52Z vern $
+
+#ifndef SERIALIZER_H
+#define SERIALIZER_H
+
+#include <map>
+#include <list>
+#include <pcap.h>
+
+#include "ID.h"
+#include "List.h"
+#include "Expr.h"
+#include "ChunkedIO.h"
+#include "SerializationFormat.h"
+#include "StateAccess.h"
+#include "PriorityQueue.h"
+#include "SerialInfo.h"
+#include "IP.h"
+#include "Timer.h"
+#include "IOSource.h"
+
+class SerializationCache;
+class SerialInfo;
+
+class Connection;
+class Timer;
+class Packet;
+
+class Serializer {
+public:
+	// Currently ID serialization is the only method which may suspend.
+	bool Serialize(SerialInfo* info, const ID& id);
+	bool Serialize(SerialInfo* info, const char* func, val_list* args);
+	bool Serialize(SerialInfo* info, const StateAccess& s);
+	bool Serialize(SerialInfo* info, const Connection& c);
+	bool Serialize(SerialInfo* info, const Timer& t);
+	bool Serialize(SerialInfo* info, const Packet& p);
+
+	// Access to the current cache.
+	SerializationCache* Cache()	{ return current_cache; }
+	void SetCache(SerializationCache* cache)
+		{ current_cache = cache; }
+
+	// Input/output methods.
+
+#define DECLARE_READ(type) \
+	bool Read(type* v, const char* tag)	{ return format->Read(v, tag); }
+
+#define DECLARE_WRITE(type) \
+	bool Write(type v, const char* tag)	\
+		{ return format->Write(v, tag); }
+
+#define DECLARE_IO(type)	\
+	DECLARE_READ(type)	\
+	DECLARE_WRITE(type)
+
+	DECLARE_IO(int)
+	DECLARE_IO(uint16)
+	DECLARE_IO(uint32)
+	DECLARE_IO(int64)
+	DECLARE_IO(uint64)
+	DECLARE_IO(char)
+	DECLARE_IO(bool)
+	DECLARE_IO(double)
+
+	bool Read(char** str, int* len, const char* tag)
+		{ return format->Read(str, len, tag); }
+	bool Read(const char** str, int* len, const char* tag)
+		// This cast is ok.
+		{ return format->Read(const_cast<char**>(str), len, tag); }
+
+	bool Read(string* s, const char* tag);
+
+	bool Write(const char* s, const char* tag)
+		{ return format->Write(s, tag); }
+	bool Write(const char* buf, int len, const char* tag)
+		{ return format->Write(buf, len, tag); }
+	bool Write(const string& s, const char* tag)
+		{ return format->Write(s.data(), s.size(), tag); }
+
+	bool WriteOpenTag(const char* tag)
+		{ return format->WriteOpenTag(tag); }
+	bool WriteCloseTag(const char* tag)
+		{ return format->WriteCloseTag(tag); }
+
+	bool WriteSeparator()	{ return format->WriteSeparator(); }
+
+	void Error(const char* msg);
+	void Warning(const char* msg);
+
+	void SetErrorDescr(const char* descr)
+		{ delete [] error_descr; error_descr = copy_string(descr); }
+
+protected:
+	// Format defaults to binary serialization.
+	Serializer(SerializationFormat* format = 0);
+	virtual ~Serializer();
+
+	// Reads next object.
+	// If 'block' is true, wait until an object can be read.
+	// Returns 0 if no more object available, -1 on error.
+	int Unserialize(UnserialInfo* info, bool block = false);
+
+	// Callback for error messages.
+	virtual void ReportError(const char* msg) = 0;
+
+	// Callbacks for unserialized objects.
+
+	// id points to ID in global scope, val is unserialized value.
+	virtual void GotID(ID* id, Val* val) = 0;
+	virtual void GotEvent(const char* name, double time,
+				EventHandlerPtr event, val_list* args) = 0;
+	virtual void GotFunctionCall(const char* name, double time,
+				Func* func, val_list* args) = 0;
+	virtual void GotStateAccess(StateAccess* s) = 0;
+	virtual void GotTimer(Timer* t) = 0;
+	virtual void GotConnection(Connection* c) = 0;
+	virtual void GotPacket(Packet* packet) = 0;
+
+	// Magic to recognize state files.
+	static const uint32 MAGIC = 0x42525354;
+
+	// This will be increased whenever there is an incompatible change
+	// in the data format.
+	static const uint32 DATA_FORMAT_VERSION = 18;
+
+	ChunkedIO* io;
+
+private:
+	bool StartSerialization(SerialInfo* info, const char* descr, char tag);
+	bool EndSerialization(SerialInfo* info);
+
+	bool UnserializeID(UnserialInfo* info);
+	bool UnserializeCall(UnserialInfo* info);
+	bool UnserializeStateAccess(UnserialInfo* info);
+	bool UnserializeTimer(UnserialInfo* info);
+	bool UnserializeConnection(UnserialInfo* info);
+	bool UnserializePacket(UnserialInfo* info);
+
+	SerializationFormat* format;
+	SerializationCache* current_cache;
+	const char* error_descr;	// used in error messages
+};
+
+
+
+// We maintain an LRU-cache for some of the objects which have already been
+// serialized. For the cache, we need two types of IDs: TransientIDs (defined
+// in SerialObj.cc) uniquely reference an object during the lifetime of a
+// process.  PermanentIDs uniquely reference an object within a serialization.
+
+class SerializationCache {
+public:
+	typedef uint64 PermanentID;
+	static const PermanentID NONE = 0;
+
+	// If max_cache_size is greater than zero, we'll remove old entries
+	// automatically if limit is reached (LRU expiration).
+	SerializationCache(unsigned int max_cache_size = 0);
+	~SerializationCache();
+
+	PermanentID Register(const SerialObj* obj, PermanentID pid,
+				bool new_cache_strategy);
+
+	const SerialObj* Lookup(PermanentID pid)
+		{
+		PIDMap::const_iterator i = pid_map.find(pid);
+		if ( i == pid_map.end() )
+			return 0;
+
+		assert(i->second);
+		MoveEntryToTail(i->second);
+		return i->second->obj.serial;
+		}
+
+	PermanentID Lookup(const TransientID& tid)
+		{
+		TIDMap::const_iterator i = tid_map.find(tid.Value());
+		if ( i == tid_map.end() )
+			return 0;
+
+		uint64 modified = i->second->obj.serial->LastModified();
+		if ( modified == SerialObj::ALWAYS || modified > i->second->time )
+			return 0;
+
+		assert(i->second);
+		MoveEntryToTail(i->second);
+		return i->second->pid;
+		}
+
+	unsigned int GetMaxCacheSize() const	{ return max_cache_size; }
+	void SetMaxCacheSize(unsigned int size)	{ max_cache_size = size; }
+
+	// These methods have to be called at the start/end of the
+	// serialization of an entity. The cache guarentees that objects
+	// registered after Begin() remain valid until End() is called.
+	// After End(), objects which are not derived from BroObj are
+	// discarded; others *may* remain valid.
+	void Begin(bool can_keep_in_cache)	{ End(can_keep_in_cache); }
+	void End(bool can_keep_in_cache);
+
+	void Clear();
+
+private:
+
+	struct CacheList;
+
+	struct CacheEntry {
+		union {
+			const SerialObj* serial;
+			const BroObj* bro;
+		} obj;
+
+		bool is_bro_obj;
+		PermanentID pid;
+		TransientID::ID tid;
+		uint64 time;
+		struct CacheList* cache;
+		CacheEntry* prev;
+		CacheEntry* next;
+
+		SerialType stype;	// primarily for debugging
+	};
+
+	// We maintain two LRU-sorted lists, one for often-changing objects and
+	// one for only rarely changing objects;
+	struct CacheList {
+		CacheEntry* head;
+		CacheEntry* tail;
+		unsigned int size;
+	};
+
+	void RemoveEntry(CacheEntry* e);
+	void UnlinkEntry(CacheEntry* e);
+	void MoveEntryToTail(CacheEntry* e);
+
+	unsigned int max_cache_size;
+
+	typedef map<PermanentID, CacheEntry*> PIDMap;
+	typedef map<TransientID::ID, CacheEntry*> TIDMap;
+
+	TIDMap tid_map;
+	PIDMap pid_map;
+
+	CacheList cache_stable;
+	CacheList cache_unstable;
+
+	// Objects in the cache which aren't derived from BroObj. These are
+	// always stored in the unstable cache.
+	typedef list<CacheEntry*> VolatileList;
+	VolatileList volatiles;
+
+	PermanentID next_id;
+};
+
+// A serializer for cloning objects.  Objects can be serialized into
+// the serializer and unserialized into new objects.  An absolutely
+// minimal implementation of Serializer!
+class CloneSerializer : public Serializer {
+public:
+	CloneSerializer(SerializationFormat* format = 0) : Serializer(format) { }
+	virtual ~CloneSerializer()	{ }
+
+protected:
+	virtual void ReportError(const char* msg)	{ run_time(msg); }
+	virtual void GotID(ID* id, Val* val)	{ }
+	virtual void GotEvent(const char* name, double time,
+				EventHandlerPtr event, val_list* args)	{ }
+	virtual void GotFunctionCall(const char* name, double time,
+				Func* func, val_list* args)	{ }
+	virtual void GotStateAccess(StateAccess* s)	{ delete s; }
+	virtual void GotTimer(Timer* t)	{ }
+	virtual void GotConnection(Connection* c)	{ }
+	virtual void GotPacket(Packet* packet)	{ }
+};
+
+// Write values/events to file or fd.
+class FileSerializer : public Serializer {
+public:
+	FileSerializer(SerializationFormat* format = 0);
+	virtual ~FileSerializer();
+
+	// Opens the file for serialization.
+	bool Open(const char* file, bool pure = false);
+	bool Close();
+
+	// Reads the file.
+	bool Read(UnserialInfo* info, const char* file, bool header = true);
+
+protected:
+	virtual void ReportError(const char* msg);
+	virtual void GotID(ID* id, Val* val);
+	virtual void GotEvent(const char* name, double time,
+				EventHandlerPtr event, val_list* args);
+	virtual void GotFunctionCall(const char* name, double time,
+				Func* func, val_list* args);
+	virtual void GotStateAccess(StateAccess* s);
+	virtual void GotTimer(Timer* t);
+	virtual void GotConnection(Connection* c);
+	virtual void GotPacket(Packet* packet);
+
+	bool OpenFile(const char* file, bool readonly, bool should_exist = false);
+	void CloseFile();
+	bool ReadFile(const char* file);
+	bool PrepareForWriting();
+	bool ReadHeader(UnserialInfo* info = 0);
+
+	SerializationCache cache;
+	const char* file;
+	int fd;
+};
+
+// Converts from one serialization format into another.
+class ConversionSerializer:public FileSerializer {
+public:
+	ConversionSerializer(SerializationFormat* in, SerializationFormat* out);
+	virtual ~ConversionSerializer();
+
+	bool Convert(const char* file_in, const char* file_out);
+
+protected:
+	virtual void GotID(ID* id, Val* val);
+	virtual void GotEvent(const char* name, double time,
+				EventHandlerPtr event, val_list* args);
+	virtual void GotFunctionCall(const char* name, double time,
+				Func* func, val_list* args);
+	virtual void GotStateAccess(StateAccess* s);
+	virtual void GotPacket(Packet* packet);
+
+	FileSerializer* serout;
+};
+
+
+// Abstract interface class for external sources providing a stream of events.
+class EventSource {
+public:
+	virtual ~EventSource() { }
+
+	// Returns time of the oldest event (0 if none available).
+	virtual double NextTimestamp(double* local_network_time) = 0;
+
+	// Dispatches the oldest event and removes it.
+	virtual void DispatchNextEvent() = 0;
+
+	// Returns true if there are more events to expect from this source.
+	virtual bool IsActive() = 0;
+};
+
+// Plays a file of events back.
+class EventPlayer : public FileSerializer, public IOSource {
+public:
+	EventPlayer(const char* file);
+	virtual ~EventPlayer();
+
+	virtual void GetFds(int* read, int* write, int* except);
+	virtual double NextTimestamp(double* local_network_time);
+	virtual void Process();
+	virtual const char* Tag()	{ return "EventPlayer"; }
+
+protected:
+	virtual void GotID(ID* id, Val* val)	{}
+	virtual void GotEvent(const char* name, double time,
+				EventHandlerPtr event, val_list* args);
+	virtual void GotFunctionCall(const char* name, double time,
+				Func* func, val_list* args);
+
+	double stream_time;	// time of first captured event
+	double replay_time;	// network time of replay start
+
+	// Next event waiting to be dispatched.
+	double ne_time;
+	EventHandlerPtr ne_handler;
+	val_list* ne_args;
+
+};
+
+
+// A link-layer packet.
+//
+// Eventually we should use something like this consistently throughout Bro,
+// replacing the current packet arguments in functions like *::NextPacket().
+// Before doing this, though, we should consider provisioning for packet
+// formats other than just libpcap by designing a more abstract interface.
+//
+// Note that for serialization we don't use much of the support provided by
+// the serialization framework. Serialize/Unserialize do all the work by
+// themselves. In particular, Packets aren't derived from SerialObj. They are
+// completely seperate and self-contained entities, and we don't need any of
+// the sophisticated features like object caching.
+
+class Packet {
+public:
+	// Argument is whether we should delete associatd memory upon
+	// destruction.
+	Packet(TimerMgr::Tag arg_tag, bool arg_free = false)
+		{
+		time = 0.0;
+		hdr = 0;
+		pkt = 0;
+		hdr_size = 0;
+		free = arg_free;
+		tag = arg_tag;
+		}
+
+	~Packet()
+		{
+		if ( free )
+			{
+			delete hdr;
+			delete [] pkt;
+			}
+		}
+
+	const IP_Hdr IP() const
+		{ return IP_Hdr((struct ip *) (pkt + hdr_size)); }
+
+	void Describe(ODesc* d) const;
+
+	bool Serialize(SerialInfo* info) const;
+	static Packet* Unserialize(UnserialInfo* info);
+
+	const struct pcap_pkthdr* hdr;
+	const u_char* pkt;
+	TimerMgr::Tag tag;
+	uint32 link_type;
+
+	double time;
+	int hdr_size;
+
+private:
+	bool free;
+};
+
+extern FileSerializer* event_serializer;
+extern FileSerializer* state_serializer;
+
+#endif