diff --git a/src/script_opt/FuncInfo.cc b/src/script_opt/FuncInfo.cc index 8b6314dd4d..e8a073c28f 100644 --- a/src/script_opt/FuncInfo.cc +++ b/src/script_opt/FuncInfo.cc @@ -252,6 +252,7 @@ static std::unordered_map func_attrs = { {"fmt", ATTR_FOLDABLE}, {"fmt_ftp_port", ATTR_IDEMPOTENT}, // can error {"fnv1a32", ATTR_FOLDABLE}, + {"fnv1a64", ATTR_FOLDABLE}, {"generate_all_events", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"get_broker_stats", ATTR_NO_ZEEK_SIDE_EFFECTS}, {"get_conn_stats", ATTR_NO_ZEEK_SIDE_EFFECTS}, diff --git a/src/zeek.bif b/src/zeek.bif index eccc7f0292..b164395fa6 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -922,6 +922,37 @@ function paraglob_equals%(p_one: opaque of paraglob, p_two: opaque of paraglob%) ); %} +%%{ +template +IntType fnva(zeek::Val* input, IntType offset, IntType prime) { + zeek::ODesc desc(zeek::DESC_BINARY); + + auto length = 0; + const u_char* bytes = NULL; + + if (input->GetType()->Tag() == zeek::TYPE_STRING) + { + length = ((zeek::StringVal*) input) -> Len(); + bytes = ((zeek::StringVal*) input)-> Bytes(); + } + else + { + input->Describe(&desc); + bytes = desc.Bytes(); + length = desc.Len(); + } + + IntType rval = offset; + for ( auto i = 0; i < length; ++i ) + { + rval ^= bytes[i]; + rval *= prime; + } + + return rval; +} +%%} + ## Returns 32-bit digest of arbitrary input values using FNV-1a hash algorithm. ## See ``_. ## @@ -932,21 +963,26 @@ function paraglob_equals%(p_one: opaque of paraglob, p_two: opaque of paraglob%) ## .. zeek:see:: hrw_weight function fnv1a32%(input: any%): count %{ - zeek::ODesc desc(DESC_BINARY); - input->Describe(&desc); - auto bytes = desc.Bytes(); + uint32_t offset = 2166136261; + uint32_t prime = 16777619; - uint32_t offset32 = 2166136261; - uint32_t prime32 = 16777619; - uint32_t rval = offset32; + auto hash = fnva(input, offset, prime); + return zeek::val_mgr->Count(hash); + %} - for ( auto i = 0; i < desc.Len(); ++i ) - { - rval ^= (uint32_t) bytes[i]; - rval *= prime32; - } +## Returns 64-bit digest of arbitrary input values using FNV-1a hash algorithm. +## See ``_. +## +## input: The desired input value to hash. +## +## Returns: The hashed value. +function fnv1a64%(input: any%): count + %{ + uint64_t offset = 0xCBF29CE484222325; + uint64_t prime = 0x100000001B3; - return zeek::val_mgr->Count(rval); + uint64_t hash = fnva(input, offset, prime); + return zeek::val_mgr->Count(hash); %} ## Calculates a weight value for use in a Rendezvous Hashing algorithm. diff --git a/testing/btest/Baseline/bifs.fnv1a_hash/out b/testing/btest/Baseline/bifs.fnv1a_hash/out new file mode 100644 index 0000000000..a502e77979 --- /dev/null +++ b/testing/btest/Baseline/bifs.fnv1a_hash/out @@ -0,0 +1,34 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Test vector: a (string) +fnv1a32: 0xe40c292c +fnv1a64: 0xaf63dc4c8601ec8c +Test vector: foobar (string) +fnv1a32: 0xbf9cf968 +fnv1a64: 0x85944171f73967e8 +Test vector: a very very long input sort of (string) +fnv1a32: 0xd14bf21b +fnv1a64: 0xb2c1be81250455fb +Test vector: 123 (string) +fnv1a32: 0x7238631b +fnv1a64: 0x456fc2181822c4db +Test vector: 123 (count) +fnv1a32: 0xc5b442fd +fnv1a64: 0xb03db9ae8d1a129d +Test vector: 123.0 (double) +fnv1a32: 0x15d2016f +fnv1a64: 0xaac1b659ecb8cfcf +Test vector: T (string) +fnv1a32: 0xd10c0b43 +fnv1a64: 0xaf64094c86023903 +Test vector: T (bool) +fnv1a32: 0x6c1151a5 +fnv1a64: 0x7e1ef442cd317605 +Test vector: F (bool) +fnv1a32: 0xeaff284 +fnv1a64: 0x5f242d39c2422be4 +Test vector: foobar (string) +fnv1a32: 0xbf9cf968 +fnv1a64: 0x85944171f73967e8 +Test vector: \x01\x02\x03\x04 (string) +fnv1a32: 0x5734a87d +fnv1a64: 0xbe7a5e775165785d diff --git a/testing/btest/Baseline/scripts.base.frameworks.cluster.custom_pool_limits/manager-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.cluster.custom_pool_limits/manager-1..stdout index a07dae7748..ffee276d0d 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.cluster.custom_pool_limits/manager-1..stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.cluster.custom_pool_limits/manager-1..stdout @@ -3,19 +3,19 @@ hrw, 0, zeek/cluster/node/proxy-1/ hrw (custom pool), 0, zeek/cluster/node/proxy-2/ hrw, 1, zeek/cluster/node/proxy-1/ -hrw (custom pool), 1, zeek/cluster/node/proxy-2/ +hrw (custom pool), 1, zeek/cluster/node/proxy-1/ hrw, 2, zeek/cluster/node/proxy-1/ -hrw (custom pool), 2, zeek/cluster/node/proxy-1/ +hrw (custom pool), 2, zeek/cluster/node/proxy-2/ hrw, 3, zeek/cluster/node/proxy-1/ hrw (custom pool), 3, zeek/cluster/node/proxy-2/ hrw, 13, zeek/cluster/node/proxy-1/ -hrw (custom pool), 13, zeek/cluster/node/proxy-2/ +hrw (custom pool), 13, zeek/cluster/node/proxy-1/ hrw, 37, zeek/cluster/node/proxy-1/ -hrw (custom pool), 37, zeek/cluster/node/proxy-2/ +hrw (custom pool), 37, zeek/cluster/node/proxy-1/ hrw, 42, zeek/cluster/node/proxy-1/ hrw (custom pool), 42, zeek/cluster/node/proxy-1/ hrw, 101, zeek/cluster/node/proxy-1/ -hrw (custom pool), 101, zeek/cluster/node/proxy-2/ +hrw (custom pool), 101, zeek/cluster/node/proxy-1/ rr, zeek/cluster/node/proxy-1/ rr (custom pool), zeek/cluster/node/proxy-1/ rr, zeek/cluster/node/proxy-1/ @@ -35,19 +35,19 @@ rr (custom pool), zeek/cluster/node/proxy-2/ hrw, 0, zeek/cluster/node/proxy-1/ hrw (custom pool), 0, zeek/cluster/node/proxy-2/ hrw, 1, zeek/cluster/node/proxy-1/ -hrw (custom pool), 1, zeek/cluster/node/proxy-2/ +hrw (custom pool), 1, zeek/cluster/node/proxy-1/ hrw, 2, zeek/cluster/node/proxy-1/ -hrw (custom pool), 2, zeek/cluster/node/proxy-1/ +hrw (custom pool), 2, zeek/cluster/node/proxy-2/ hrw, 3, zeek/cluster/node/proxy-1/ hrw (custom pool), 3, zeek/cluster/node/proxy-2/ hrw, 13, zeek/cluster/node/proxy-1/ -hrw (custom pool), 13, zeek/cluster/node/proxy-2/ +hrw (custom pool), 13, zeek/cluster/node/proxy-1/ hrw, 37, zeek/cluster/node/proxy-1/ -hrw (custom pool), 37, zeek/cluster/node/proxy-2/ +hrw (custom pool), 37, zeek/cluster/node/proxy-1/ hrw, 42, zeek/cluster/node/proxy-1/ hrw (custom pool), 42, zeek/cluster/node/proxy-1/ hrw, 101, zeek/cluster/node/proxy-1/ -hrw (custom pool), 101, zeek/cluster/node/proxy-2/ +hrw (custom pool), 101, zeek/cluster/node/proxy-1/ 2nd stuff hrw, 0, hrw (custom pool), 0, zeek/cluster/node/proxy-2/ diff --git a/testing/btest/Baseline/scripts.base.frameworks.cluster.topic_distribution/manager-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.cluster.topic_distribution/manager-1..stdout index 46f0ead64c..a32b404129 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.cluster.topic_distribution/manager-1..stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.cluster.topic_distribution/manager-1..stdout @@ -1,13 +1,13 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. 1st stuff hrw, 0, zeek/cluster/node/proxy-2/ -hrw, 1, zeek/cluster/node/proxy-2/ -hrw, 2, zeek/cluster/node/proxy-1/ +hrw, 1, zeek/cluster/node/proxy-1/ +hrw, 2, zeek/cluster/node/proxy-2/ hrw, 3, zeek/cluster/node/proxy-2/ -hrw, 13, zeek/cluster/node/proxy-2/ -hrw, 37, zeek/cluster/node/proxy-2/ +hrw, 13, zeek/cluster/node/proxy-1/ +hrw, 37, zeek/cluster/node/proxy-1/ hrw, 42, zeek/cluster/node/proxy-1/ -hrw, 101, zeek/cluster/node/proxy-2/ +hrw, 101, zeek/cluster/node/proxy-1/ rr, zeek/cluster/node/proxy-1/ rr, zeek/cluster/node/proxy-2/ rr, zeek/cluster/node/proxy-1/ @@ -17,13 +17,13 @@ rr, zeek/cluster/node/proxy-2/ rr, zeek/cluster/node/proxy-1/ rr, zeek/cluster/node/proxy-2/ hrw, 0, zeek/cluster/node/proxy-2/ -hrw, 1, zeek/cluster/node/proxy-2/ -hrw, 2, zeek/cluster/node/proxy-1/ +hrw, 1, zeek/cluster/node/proxy-1/ +hrw, 2, zeek/cluster/node/proxy-2/ hrw, 3, zeek/cluster/node/proxy-2/ -hrw, 13, zeek/cluster/node/proxy-2/ -hrw, 37, zeek/cluster/node/proxy-2/ +hrw, 13, zeek/cluster/node/proxy-1/ +hrw, 37, zeek/cluster/node/proxy-1/ hrw, 42, zeek/cluster/node/proxy-1/ -hrw, 101, zeek/cluster/node/proxy-2/ +hrw, 101, zeek/cluster/node/proxy-1/ 2nd stuff hrw, 0, zeek/cluster/node/proxy-2/ hrw, 1, zeek/cluster/node/proxy-2/ diff --git a/testing/btest/Baseline/scripts.base.frameworks.cluster.topic_distribution_bifs/proxy-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.cluster.topic_distribution_bifs/proxy-1..stdout index a89373f442..c4e16958ba 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.cluster.topic_distribution_bifs/proxy-1..stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.cluster.topic_distribution_bifs/proxy-1..stdout @@ -1,6 +1,9 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -got distributed event hrw, 2 +got distributed event hrw, 1 +got distributed event hrw, 13 +got distributed event hrw, 37 got distributed event hrw, 42 +got distributed event hrw, 101 got distributed event rr, 0 got distributed event rr, 2 got distributed event rr, 13 diff --git a/testing/btest/Baseline/scripts.base.frameworks.cluster.topic_distribution_bifs/proxy-2..stdout b/testing/btest/Baseline/scripts.base.frameworks.cluster.topic_distribution_bifs/proxy-2..stdout index e6e28472a9..fec1ae9fe4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.cluster.topic_distribution_bifs/proxy-2..stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.cluster.topic_distribution_bifs/proxy-2..stdout @@ -1,10 +1,7 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. got distributed event hrw, 0 -got distributed event hrw, 1 +got distributed event hrw, 2 got distributed event hrw, 3 -got distributed event hrw, 13 -got distributed event hrw, 37 -got distributed event hrw, 101 got distributed event rr, 1 got distributed event rr, 3 got distributed event rr, 37 diff --git a/testing/btest/Baseline/scripts.base.utils.hash_hrw/output b/testing/btest/Baseline/scripts.base.utils.hash_hrw/output index da11e37b36..280e114f5a 100644 --- a/testing/btest/Baseline/scripts.base.utils.hash_hrw/output +++ b/testing/btest/Baseline/scripts.base.utils.hash_hrw/output @@ -9,21 +9,21 @@ T F [id=0, user_data=alice] [id=3, user_data=dave] -[id=4, user_data=eve] -[id=4, user_data=eve] -[id=4, user_data=eve] -[id=0, user_data=alice] +[id=1, user_data=bob] +[id=1, user_data=bob] +[id=1, user_data=bob] +[id=3, user_data=dave] [id=3, user_data=dave] [id=1, user_data=bob] [id=0, user_data=alice] [id=1, user_data=bob] T -[id=4, user_data=eve] +[id=1, user_data=bob] +[id=3, user_data=dave] +[id=1, user_data=bob] +[id=1, user_data=bob] +[id=1, user_data=bob] [id=3, user_data=dave] -[id=4, user_data=eve] -[id=4, user_data=eve] -[id=4, user_data=eve] -[id=4, user_data=eve] [id=3, user_data=dave] [id=1, user_data=bob] [id=4, user_data=eve] @@ -31,10 +31,10 @@ T T [id=0, user_data=alice] [id=3, user_data=dave] -[id=4, user_data=eve] -[id=4, user_data=eve] -[id=4, user_data=eve] -[id=0, user_data=alice] +[id=1, user_data=bob] +[id=1, user_data=bob] +[id=1, user_data=bob] +[id=3, user_data=dave] [id=3, user_data=dave] [id=1, user_data=bob] [id=0, user_data=alice] diff --git a/testing/btest/bifs/fnv1a_hash.zeek b/testing/btest/bifs/fnv1a_hash.zeek new file mode 100644 index 0000000000..75a84da84b --- /dev/null +++ b/testing/btest/bifs/fnv1a_hash.zeek @@ -0,0 +1,33 @@ +# @TEST-EXEC: zeek -b %INPUT >out 2>&1 +# @TEST-EXEC: btest-diff out + +function output_hashes(val: any) + { + print(fmt("Test vector: %s (%s)", val, type_name(val))); + print(fmt("fnv1a32: 0x%x", fnv1a32(val))); + print(fmt("fnv1a64: 0x%x", fnv1a64(val))); + } + +event zeek_init() + { + output_hashes("a"); + output_hashes("foobar"); + output_hashes("a very very long input sort of"); + output_hashes("123"); + output_hashes(123); + output_hashes(123.0); + output_hashes("T"); + output_hashes(T); + output_hashes(F); + + local s: string; + local bytes_from_hex: string; + + s = "666f6f626172"; # hex representation of foobar + bytes_from_hex = hexstr_to_bytestring(s); + output_hashes(bytes_from_hex); + + s = "01020304"; # hex representation of 4 bytes + bytes_from_hex = hexstr_to_bytestring(s); + output_hashes(bytes_from_hex); + }