Increase UIDs to 96 bits w/ C/F prefix - BIT-1016

- The bit-length is adjustable via redef'ing bits_per_uid.

- Prefix 'C' is used for connection UIDS (including IP tunnels) and
  'F' for files.

src/CMakeLists.txt

@@ -325,6 +325,7 @@ set(bro_SRCS
     Trigger.cc
     TunnelEncapsulation.cc
     Type.cc
+    UID.cc
     Val.cc
     Var.cc
     bsd-getopt-long.c

src/Conn.cc

@@ -160,8 +160,6 @@ Connection::Connection(NetSessions* s, HashKey* k, double t, const ConnID* id,
 	TimerMgr::Tag* tag = current_iosrc->GetCurrentTag();
 	conn_timer_mgr = tag ? new TimerMgr::Tag(*tag) : 0;
 
-	uid = 0; // Will set later.
-
 	if ( arg_encap )
 		encapsulation = new EncapsulationStack(*arg_encap);
 	else
@@ -380,10 +378,9 @@ RecordVal* Connection::BuildConnVal()
 		conn_val->Assign(8, new StringVal(""));	// history
 
 		if ( ! uid )
-			uid = calculate_unique_id();
+			uid = Bro::UID(bits_per_uid);
 
-		char tmp[20];
-		conn_val->Assign(9, new StringVal(uitoa_n(uid, tmp, sizeof(tmp), 62)));
+		conn_val->Assign(9, new StringVal(uid.Base62("C").c_str()));
 
 		if ( encapsulation && encapsulation->Depth() > 0 )
 			conn_val->Assign(10, encapsulation->GetVectorVal());

src/Conn.h

@@ -13,6 +13,7 @@
 #include "RuleMatcher.h"
 #include "IPAddr.h"
 #include "TunnelEncapsulation.h"
+#include "UID.h"
 
 #include "analyzer/Tag.h"
 #include "analyzer/Analyzer.h"
@@ -252,9 +253,9 @@ public:
 	// Sets the transport protocol in use.
 	void SetTransport(TransportProto arg_proto)	{ proto = arg_proto; }
 
-	void SetUID(uint64 arg_uid)	 { uid = arg_uid; }
+	void SetUID(Bro::UID arg_uid)	 { uid = arg_uid; }
 
-	uint64 GetUID() const { return uid; }
+	Bro::UID GetUID() const { return uid; }
 
 	const EncapsulationStack* GetEncapsulation() const
 		{ return encapsulation; }
@@ -321,7 +322,7 @@ protected:
 	analyzer::TransportLayerAnalyzer* root_analyzer;
 	analyzer::pia::PIA* primary_PIA;
 
-	uint64 uid;	// Globally unique connection ID.
+	Bro::UID uid;	// Globally unique connection ID.
 };
 
 class ConnectionTimer : public Timer {

src/NetVar.cc

@@ -240,6 +240,8 @@ StringVal* cmd_line_bpf_filter;
 
 StringVal* global_hash_seed;
 
+bro_uint_t bits_per_uid;
+
 OpaqueType* md5_type;
 OpaqueType* sha1_type;
 OpaqueType* sha256_type;
@@ -309,6 +311,8 @@ void init_general_global_var()
 
 	global_hash_seed = opt_internal_string("global_hash_seed");
 
+	bits_per_uid = opt_internal_unsigned("bits_per_uid");
+
 	md5_type = new OpaqueType("md5");
 	sha1_type = new OpaqueType("sha1");
 	sha256_type = new OpaqueType("sha256");

src/NetVar.h

@@ -244,6 +244,8 @@ extern StringVal* cmd_line_bpf_filter;
 
 extern StringVal* global_hash_seed;
 
+extern bro_uint_t bits_per_uid;
+
 class OpaqueType;
 extern OpaqueType* md5_type;
 extern OpaqueType* sha1_type;

src/TunnelEncapsulation.cc

@@ -11,7 +11,7 @@ EncapsulatingConn::EncapsulatingConn(Connection* c, BifEnum::Tunnel::Type t)
 	{
 	if ( ! uid )
 		{
-		uid = calculate_unique_id();
+		uid = Bro::UID(bits_per_uid);
 		c->SetUID(uid);
 		}
 	}
@@ -28,8 +28,7 @@ RecordVal* EncapsulatingConn::GetRecordVal() const
 	rv->Assign(0, id_val);
 	rv->Assign(1, new EnumVal(type, BifType::Enum::Tunnel::Type));
 
-	char tmp[20];
-	rv->Assign(2, new StringVal(uitoa_n(uid, tmp, sizeof(tmp), 62)));
+	rv->Assign(2, new StringVal(uid.Base62("C").c_str()));
 
 	return rv;
 	}

src/TunnelEncapsulation.h

@@ -7,6 +7,7 @@
 #include "NetVar.h"
 #include "IPAddr.h"
 #include "Val.h"
+#include "UID.h"
 #include <vector>
 
 class Connection;
@@ -25,7 +26,7 @@ public:
 	 */
 	EncapsulatingConn()
 		: src_port(0), dst_port(0), proto(TRANSPORT_UNKNOWN),
-		  type(BifEnum::Tunnel::NONE), uid(0)
+		  type(BifEnum::Tunnel::NONE), uid()
 		{}
 
 	/**
@@ -39,9 +40,9 @@ public:
 	 */
 	EncapsulatingConn(const IPAddr& s, const IPAddr& d)
 		: src_addr(s), dst_addr(d), src_port(0), dst_port(0),
-		  proto(TRANSPORT_UNKNOWN), type(BifEnum::Tunnel::IP)
+		  proto(TRANSPORT_UNKNOWN), type(BifEnum::Tunnel::IP),
+		  uid(Bro::UID(bits_per_uid))
 		{
-		uid = calculate_unique_id();
 		}
 
 	/**
@@ -108,7 +109,7 @@ protected:
 	uint16 dst_port;
 	TransportProto proto;
 	BifEnum::Tunnel::Type type;
-	uint64 uid;
+	Bro::UID uid;
 };
 
 /**

src/UID.cc

@@ -0,0 +1,43 @@
+#include <cstdlib>
+
+#include "UID.h"
+
+using namespace Bro;
+using namespace std;
+
+void UID::Set(bro_uint_t bits, const std::vector<uint64>& v)
+	{
+	uid.clear();
+
+	div_t res = div(bits, 64);
+	int size = res.rem ? res.quot + 1 : res.quot;
+
+	for ( int i = 0; i < size; ++i )
+		uid.push_back(i < v.size() ? v[i] : calculate_unique_id());
+
+	if ( res.rem )
+		uid[0] >>= 64 - res.rem;
+	}
+
+string UID::Base62(const std::string& prefix) const
+	{
+	char tmp[64]; // technically, this should dynamically scale based on size
+	string rval(prefix);
+
+	for ( size_t i = 0; i < uid.size(); ++i )
+		rval.append(uitoa_n(uid[i], tmp, sizeof(tmp), 62));
+
+	return rval;
+	}
+
+bool Bro::operator==(const UID& u1, const UID& u2)
+	{
+	if ( u1.uid.size() != u2.uid.size() )
+		return false;
+
+	for ( size_t i = 0; i < u1.uid.size(); ++i )
+		if ( u1.uid[i] != u2.uid[i] )
+			return false;
+
+	return true;
+	}

src/UID.h

@@ -0,0 +1,85 @@
+#ifndef BRO_UID_H
+#define BRO_UID_H
+
+#include <string>
+#include <vector>
+
+#include "util.h"
+
+namespace Bro {
+
+/**
+ * A class for creating/manager UIDs of arbitrary bit-length and converting
+ * them to human-readable strings in Base62 format.
+ */
+class UID {
+public:
+
+	/**
+	 * Default ctor.  The UID is uninitialized and in string format is
+	 * represented by an empty string.
+	 */
+	UID() {}
+
+	/**
+	 * Construct a UID of a given bit-length, optionally from given values.
+	 * @see UID::Set
+	 */
+	UID(bro_uint_t bits, const std::vector<uint64>& v = std::vector<uint64>())
+		{ Set(bits, v); }
+
+	/**
+	 * Copy constructor.
+	 */
+	UID(const UID& other) { uid = other.uid; }
+
+	/**
+	 * Inititialize a UID of a given bit-length, optionally from given values.
+	 * @param bits The desired length in bits of the UID.
+	 * @param v A vector of values with which to initialize the UID.
+	 *          If empty or doesn't contain enough values to satisfy \a bits,
+	 *          then values are automatically generated using
+	 *          calculate_unique_id().  If \a bits isn't evenly divisible by
+	 *          64, then a value is truncated to bit in desired bit-length.
+	 */
+	void Set(bro_uint_t bits,
+	         const std::vector<uint64>& v = std::vector<uint64>());
+
+	/**
+	 * Returns a base62 (characters 0-9, A-Z, a-z) representation of the UID.
+	 * @param prefix An optional string prefix.
+	 * @return a base62 string representing the UID.
+	 */
+	std::string Base62(const std::string& prefix = "") const;
+
+	/**
+	 * @return false if the UID instance was create via the default ctor
+	 *         and not yet initialized w/ Set().
+	 * TODO: this would be better as an "explicit" conversion operator (C++11)
+	 */
+	operator bool() const { return ( ! uid.empty() ); }
+
+	/**
+	 * Assignment operator.
+	 */
+	UID& operator=(const UID& other) { uid = other.uid; return *this; }
+
+	/**
+	 * UID equality operator.
+	 */
+	friend bool operator==(const UID& u1, const UID& u2);
+
+	/**
+	 * UID inequality operator.
+	 */
+	friend bool operator!=(const UID& u1, const UID& u2)
+		{ return ! ( u1 == u2 ); }
+
+private:
+
+	std::vector<uint64> uid;
+};
+
+} // namespace Bro
+
+#endif

src/file_analysis/Manager.cc

@@ -9,6 +9,7 @@
 #include "Analyzer.h"
 #include "Var.h"
 #include "Event.h"
+#include "UID.h"
 
 #include "plugin/Manager.h"
 
@@ -57,15 +58,16 @@ string Manager::HashHandle(const string& handle) const
 	if ( salt.empty() )
 		salt = BifConst::Files::salt->CheckString();
 
-	char tmp[20];
 	uint64 hash[2];
 	string msg(handle + salt);
 
 	MD5(reinterpret_cast<const u_char*>(msg.data()), msg.size(),
 	    reinterpret_cast<u_char*>(hash));
-	uitoa_n(hash[0], tmp, sizeof(tmp), 62);
 
-	return tmp;
+	vector<uint64> v;
+	v.push_back(hash[0]);
+	v.push_back(hash[1]);
+	return Bro::UID(bits_per_uid, v).Base62("F");
 	}
 
 void Manager::SetHandle(const string& handle)

src/file_analysis/Manager.h

@@ -62,7 +62,7 @@ public:
 	/**
 	 * Creates a file identifier from a unique file handle string.
 	 * @param handle a unique string which identifies a single file.
-	 * @return a prettified MD5 hash of \a handle, truncated to 64-bits.
+	 * @return a prettified MD5 hash of \a handle, truncated to 96-bits.
 	 */
 	string HashHandle(const string& handle) const;