mirror of
https://github.com/zeek/zeek.git
synced 2025-10-02 06:38:20 +00:00
Merge remote-tracking branch 'origin/topic/timw/faster-hashing'
- Minor cleanups in siphash24.h (code style, header include) - Updated COPYING.3rdparty with new license info * origin/topic/timw/faster-hashing: Add a faster siphash24 implementation than the reference one
This commit is contained in:
Jon Siwek
commit
84e3e6c619
5 changed files with 132 additions and 164 deletions
12
CHANGES
12
CHANGES
|
|
@ -1,4 +1,16 @@
|
||||||
|
|
||||||
|
3.2.0-dev.144 | 2020-02-25 19:52:57 -0800
|
||||||
|
|
||||||
|
* Add a faster siphash24 implementation than the reference one (Tim Wojtulewicz, Corelight)
|
||||||
|
|
||||||
|
- Average of 10 runs of 2009-M57-day11-18.trace (release build at -O3):
|
||||||
|
- Master: 6.027s 93650 bytes max RSS
|
||||||
|
- Commit: 5.950s 93271 bytes max RSS
|
||||||
|
|
||||||
|
- Hashing a fixed 32-byte payload 10 million times with a fixed key:
|
||||||
|
- Master: 1.397411s
|
||||||
|
- Commit: 0.998211s
|
||||||
|
|
||||||
3.2.0-dev.142 | 2020-02-25 19:27:28 -0800
|
3.2.0-dev.142 | 2020-02-25 19:27:28 -0800
|
||||||
|
|
||||||
* Updates to Broker to build CAF as sub-project (Dominik Charousset, Corelight)
|
* Updates to Broker to build CAF as sub-project (Dominik Charousset, Corelight)
|
||||||
|
|
|
||||||
|
|
@ -504,19 +504,27 @@ POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
|
||||||
==============================================================================
|
==============================================================================
|
||||||
|
|
||||||
SipHash reference C implementation
|
Taken from https://github.com/majek/csiphash with MIT License:
|
||||||
|
|
||||||
Copyright (c) 2012-2014 Jean-Philippe Aumasson
|
Copyright (c) 2013 Marek Majkowski <marek@popcount.org>
|
||||||
<jeanphilippe.aumasson@gmail.com>
|
|
||||||
Copyright (c) 2012-2014 Daniel J. Bernstein <djb@cr.yp.to>
|
|
||||||
|
|
||||||
To the extent possible under law, the author(s) have dedicated all copyright
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
and related and neighboring rights to this software to the public domain
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
worldwide. This software is distributed without any warranty.
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
You should have received a copy of the CC0 Public Domain Dedication along
|
The above copyright notice and this permission notice shall be included in
|
||||||
with this software. If not, see
|
all copies or substantial portions of the Software.
|
||||||
<http://creativecommons.org/publicdomain/zero/1.0/>.
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||||
|
THE SOFTWARE.
|
||||||
|
|
||||||
==============================================================================
|
==============================================================================
|
||||||
|
|
||||||
|
|
|
||||||
2
VERSION
2
VERSION
|
|
@ -1 +1 @@
|
||||||
3.2.0-dev.142
|
3.2.0-dev.144
|
||||||
|
|
|
||||||
244
src/siphash24.c
244
src/siphash24.c
|
|
@ -1,166 +1,114 @@
|
||||||
/*
|
/* <MIT License>
|
||||||
SipHash reference C implementation
|
Copyright (c) 2013 Marek Majkowski <marek@popcount.org>
|
||||||
|
|
||||||
Copyright (c) 2012-2014 Jean-Philippe Aumasson
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
<jeanphilippe.aumasson@gmail.com>
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
Copyright (c) 2012-2014 Daniel J. Bernstein <djb@cr.yp.to>
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
To the extent possible under law, the author(s) have dedicated all copyright
|
The above copyright notice and this permission notice shall be included in
|
||||||
and related and neighboring rights to this software to the public domain
|
all copies or substantial portions of the Software.
|
||||||
worldwide. This software is distributed without any warranty.
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||||
|
THE SOFTWARE.
|
||||||
|
</MIT License>
|
||||||
|
|
||||||
|
Original location:
|
||||||
|
https://github.com/majek/csiphash/
|
||||||
|
|
||||||
|
Solution inspired by code from:
|
||||||
|
Samuel Neves (supercop/crypto_auth/siphash24/little)
|
||||||
|
djb (supercop/crypto_auth/siphash24/little2)
|
||||||
|
Jean-Philippe Aumasson (https://131002.net/siphash/siphash24.c)
|
||||||
|
*/
|
||||||
|
|
||||||
You should have received a copy of the CC0 Public Domain Dedication along
|
|
||||||
with
|
|
||||||
this software. If not, see
|
|
||||||
<http://creativecommons.org/publicdomain/zero/1.0/>.
|
|
||||||
*/
|
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <stdio.h>
|
|
||||||
#include <string.h>
|
|
||||||
|
|
||||||
/* default: SipHash-2-4 */
|
#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \
|
||||||
#define cROUNDS 2
|
__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
|
||||||
#define dROUNDS 4
|
# define _le64toh(x) ((uint64_t)(x))
|
||||||
|
#elif defined(_WIN32)
|
||||||
#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b))))
|
/* Windows is always little endian, unless you're on xbox360
|
||||||
|
http://msdn.microsoft.com/en-us/library/b0084kay(v=vs.80).aspx */
|
||||||
#define U32TO8_LE(p, v) \
|
# define _le64toh(x) ((uint64_t)(x))
|
||||||
(p)[0] = (uint8_t)((v)); \
|
#elif defined(__APPLE__)
|
||||||
(p)[1] = (uint8_t)((v) >> 8); \
|
# include <libkern/OSByteOrder.h>
|
||||||
(p)[2] = (uint8_t)((v) >> 16); \
|
# define _le64toh(x) OSSwapLittleToHostInt64(x)
|
||||||
(p)[3] = (uint8_t)((v) >> 24);
|
|
||||||
|
|
||||||
#define U64TO8_LE(p, v) \
|
|
||||||
U32TO8_LE((p), (uint32_t)((v))); \
|
|
||||||
U32TO8_LE((p) + 4, (uint32_t)((v) >> 32));
|
|
||||||
|
|
||||||
#define U8TO64_LE(p) \
|
|
||||||
(((uint64_t)((p)[0])) | ((uint64_t)((p)[1]) << 8) | \
|
|
||||||
((uint64_t)((p)[2]) << 16) | ((uint64_t)((p)[3]) << 24) | \
|
|
||||||
((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40) | \
|
|
||||||
((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56))
|
|
||||||
|
|
||||||
#define SIPROUND \
|
|
||||||
do { \
|
|
||||||
v0 += v1; \
|
|
||||||
v1 = ROTL(v1, 13); \
|
|
||||||
v1 ^= v0; \
|
|
||||||
v0 = ROTL(v0, 32); \
|
|
||||||
v2 += v3; \
|
|
||||||
v3 = ROTL(v3, 16); \
|
|
||||||
v3 ^= v2; \
|
|
||||||
v0 += v3; \
|
|
||||||
v3 = ROTL(v3, 21); \
|
|
||||||
v3 ^= v0; \
|
|
||||||
v2 += v1; \
|
|
||||||
v1 = ROTL(v1, 17); \
|
|
||||||
v1 ^= v2; \
|
|
||||||
v2 = ROTL(v2, 32); \
|
|
||||||
} while (0)
|
|
||||||
|
|
||||||
#ifdef SIPHASHDEBUG
|
|
||||||
#define TRACE \
|
|
||||||
do { \
|
|
||||||
printf("(%3d) v0 %08x %08x\n", (int)inlen, (uint32_t)(v0 >> 32), \
|
|
||||||
(uint32_t)v0); \
|
|
||||||
printf("(%3d) v1 %08x %08x\n", (int)inlen, (uint32_t)(v1 >> 32), \
|
|
||||||
(uint32_t)v1); \
|
|
||||||
printf("(%3d) v2 %08x %08x\n", (int)inlen, (uint32_t)(v2 >> 32), \
|
|
||||||
(uint32_t)v2); \
|
|
||||||
printf("(%3d) v3 %08x %08x\n", (int)inlen, (uint32_t)(v3 >> 32), \
|
|
||||||
(uint32_t)v3); \
|
|
||||||
} while (0)
|
|
||||||
#else
|
#else
|
||||||
#define TRACE
|
|
||||||
|
/* See: http://sourceforge.net/p/predef/wiki/Endianness/ */
|
||||||
|
# if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)
|
||||||
|
# include <sys/endian.h>
|
||||||
|
# else
|
||||||
|
# include <endian.h>
|
||||||
|
# endif
|
||||||
|
# if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && \
|
||||||
|
__BYTE_ORDER == __LITTLE_ENDIAN
|
||||||
|
# define _le64toh(x) ((uint64_t)(x))
|
||||||
|
# else
|
||||||
|
# define _le64toh(x) le64toh(x)
|
||||||
|
# endif
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
// [Bro] We turn this into an internal function. siphash.h defines a wrapper.
|
|
||||||
int _siphash(uint8_t *out, const uint8_t *in, uint64_t inlen, const uint8_t *k) {
|
|
||||||
/* "somepseudorandomlygeneratedbytes" */
|
|
||||||
uint64_t v0 = 0x736f6d6570736575ULL;
|
|
||||||
uint64_t v1 = 0x646f72616e646f6dULL;
|
|
||||||
uint64_t v2 = 0x6c7967656e657261ULL;
|
|
||||||
uint64_t v3 = 0x7465646279746573ULL;
|
|
||||||
uint64_t b;
|
|
||||||
uint64_t k0 = U8TO64_LE(k);
|
|
||||||
uint64_t k1 = U8TO64_LE(k + 8);
|
|
||||||
uint64_t m;
|
|
||||||
int i;
|
|
||||||
const uint8_t *end = in + inlen - (inlen % sizeof(uint64_t));
|
|
||||||
const int left = inlen & 7;
|
|
||||||
b = ((uint64_t)inlen) << 56;
|
|
||||||
v3 ^= k1;
|
|
||||||
v2 ^= k0;
|
|
||||||
v1 ^= k1;
|
|
||||||
v0 ^= k0;
|
|
||||||
|
|
||||||
#ifdef DOUBLE
|
#define ROTATE(x, b) (uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) )
|
||||||
v1 ^= 0xee;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
for (; in != end; in += 8) {
|
#define HALF_ROUND(a,b,c,d,s,t) \
|
||||||
m = U8TO64_LE(in);
|
a += b; c += d; \
|
||||||
v3 ^= m;
|
b = ROTATE(b, s) ^ a; \
|
||||||
|
d = ROTATE(d, t) ^ c; \
|
||||||
|
a = ROTATE(a, 32);
|
||||||
|
|
||||||
TRACE;
|
#define DOUBLE_ROUND(v0,v1,v2,v3) \
|
||||||
for (i = 0; i < cROUNDS; ++i)
|
HALF_ROUND(v0,v1,v2,v3,13,16); \
|
||||||
SIPROUND;
|
HALF_ROUND(v2,v1,v0,v3,17,21); \
|
||||||
|
HALF_ROUND(v0,v1,v2,v3,13,16); \
|
||||||
|
HALF_ROUND(v2,v1,v0,v3,17,21);
|
||||||
|
|
||||||
v0 ^= m;
|
|
||||||
}
|
|
||||||
|
|
||||||
switch (left) {
|
uint64_t siphash24(const void *src, unsigned long src_sz, const uint64_t* key) {
|
||||||
case 7:
|
uint64_t k0 = _le64toh(key[0]);
|
||||||
b |= ((uint64_t)in[6]) << 48;
|
uint64_t k1 = _le64toh(key[1]);
|
||||||
case 6:
|
uint64_t b = (uint64_t)src_sz << 56;
|
||||||
b |= ((uint64_t)in[5]) << 40;
|
const uint64_t *in = (uint64_t*)src;
|
||||||
case 5:
|
|
||||||
b |= ((uint64_t)in[4]) << 32;
|
|
||||||
case 4:
|
|
||||||
b |= ((uint64_t)in[3]) << 24;
|
|
||||||
case 3:
|
|
||||||
b |= ((uint64_t)in[2]) << 16;
|
|
||||||
case 2:
|
|
||||||
b |= ((uint64_t)in[1]) << 8;
|
|
||||||
case 1:
|
|
||||||
b |= ((uint64_t)in[0]);
|
|
||||||
break;
|
|
||||||
case 0:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
v3 ^= b;
|
uint64_t v0 = k0 ^ 0x736f6d6570736575ULL;
|
||||||
|
uint64_t v1 = k1 ^ 0x646f72616e646f6dULL;
|
||||||
|
uint64_t v2 = k0 ^ 0x6c7967656e657261ULL;
|
||||||
|
uint64_t v3 = k1 ^ 0x7465646279746573ULL;
|
||||||
|
|
||||||
TRACE;
|
while (src_sz >= 8) {
|
||||||
for (i = 0; i < cROUNDS; ++i)
|
uint64_t mi = _le64toh(*in);
|
||||||
SIPROUND;
|
in += 1; src_sz -= 8;
|
||||||
|
v3 ^= mi;
|
||||||
|
DOUBLE_ROUND(v0,v1,v2,v3);
|
||||||
|
v0 ^= mi;
|
||||||
|
}
|
||||||
|
|
||||||
v0 ^= b;
|
uint64_t t = 0; uint8_t *pt = (uint8_t *)&t; uint8_t *m = (uint8_t *)in;
|
||||||
|
switch (src_sz) {
|
||||||
|
case 7: pt[6] = m[6];
|
||||||
|
case 6: pt[5] = m[5];
|
||||||
|
case 5: pt[4] = m[4];
|
||||||
|
case 4: *((uint32_t*)&pt[0]) = *((uint32_t*)&m[0]); break;
|
||||||
|
case 3: pt[2] = m[2];
|
||||||
|
case 2: pt[1] = m[1];
|
||||||
|
case 1: pt[0] = m[0];
|
||||||
|
}
|
||||||
|
b |= _le64toh(t);
|
||||||
|
|
||||||
#ifndef DOUBLE
|
v3 ^= b;
|
||||||
v2 ^= 0xff;
|
DOUBLE_ROUND(v0,v1,v2,v3);
|
||||||
#else
|
v0 ^= b; v2 ^= 0xff;
|
||||||
v2 ^= 0xee;
|
DOUBLE_ROUND(v0,v1,v2,v3);
|
||||||
#endif
|
DOUBLE_ROUND(v0,v1,v2,v3);
|
||||||
|
return (v0 ^ v1) ^ (v2 ^ v3);
|
||||||
TRACE;
|
|
||||||
for (i = 0; i < dROUNDS; ++i)
|
|
||||||
SIPROUND;
|
|
||||||
|
|
||||||
b = v0 ^ v1 ^ v2 ^ v3;
|
|
||||||
U64TO8_LE(out, b);
|
|
||||||
|
|
||||||
#ifdef DOUBLE
|
|
||||||
v1 ^= 0xdd;
|
|
||||||
|
|
||||||
TRACE;
|
|
||||||
for (i = 0; i < dROUNDS; ++i)
|
|
||||||
SIPROUND;
|
|
||||||
|
|
||||||
b = v0 ^ v1 ^ v2 ^ v3;
|
|
||||||
U64TO8_LE(out + 8, b);
|
|
||||||
#endif
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,15 +1,15 @@
|
||||||
|
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
#define SIPHASH_KEYLEN 16
|
#define SIPHASH_KEYLEN 16
|
||||||
#define SIPHASH_HASHLEN 8
|
|
||||||
|
|
||||||
extern "C" {
|
extern "C" {
|
||||||
int _siphash(uint8_t *out, const uint8_t *in, uint64_t inlen, const uint8_t *k);
|
uint64_t siphash24(const void* src, unsigned long src_sz, const uint64_t* key);
|
||||||
}
|
}
|
||||||
|
|
||||||
// [Bro] Wrapper for better type-safety.
|
// [Bro] Wrapper for better type-safety.
|
||||||
inline void siphash(uint64_t* digest, const uint8_t *in, uint64_t inlen, const uint8_t* key)
|
inline void siphash(uint64_t* digest, const uint8_t* in, uint64_t inlen, const uint8_t* key)
|
||||||
{
|
{
|
||||||
_siphash((uint8_t*)digest, in, inlen, key);
|
*digest = siphash24(in, inlen, (const uint64_t*)key);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue