Commit: fa545b20f4a3fc080963b96c21827dff078efbb6
Parent: d35d69420938f8d3c9ed488e510e8fa6d70bb1d4
Author: Randy Palamar
Date: Sat, 17 Oct 2026 05:32:24 -0700
external/xxhash: bump to 0.8.4
supposedly this fixes some strict aliasing issue with GCC
Diffstat:
| M | external/xxhash.h | | | 226 | ++++++++++++++++++++++++++++++++++++++++++++++--------------------------------- |
1 file changed, 133 insertions(+), 93 deletions(-)
diff --git a/external/xxhash.h b/external/xxhash.h
@@ -323,6 +323,10 @@ extern "C" {
# undef XXH_NAMESPACE
#endif
+#define XXH_CAT(A,B) A##B
+#define XXH_NAME2(A,B) XXH_CAT(A,B)
+#define XXH_IPREF(Id) XXH_NAME2(XXH_NAMESPACE, Id)
+
#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \
&& !defined(XXH_INLINE_ALL_31684351384)
/* this section should be traversed only once */
@@ -418,7 +422,6 @@ extern "C" {
* However, this requires some #ifdefs, and has a more dispersed impact.
* Meanwhile, renaming can be achieved in a single place.
*/
-# define XXH_IPREF(Id) XXH_NAMESPACE ## Id
# define XXH_OK XXH_IPREF(XXH_OK)
# define XXH_ERROR XXH_IPREF(XXH_ERROR)
# define XXH_errorcode XXH_IPREF(XXH_errorcode)
@@ -457,61 +460,59 @@ extern "C" {
#endif
#ifdef XXH_NAMESPACE
-# define XXH_CAT(A,B) A##B
-# define XXH_NAME2(A,B) XXH_CAT(A,B)
-# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
+# define XXH_versionNumber XXH_IPREF(XXH_versionNumber)
/* XXH32 */
-# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
-# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
-# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
-# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
-# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
-# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
-# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
-# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
-# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
+# define XXH32 XXH_IPREF(XXH32)
+# define XXH32_createState XXH_IPREF(XXH32_createState)
+# define XXH32_freeState XXH_IPREF(XXH32_freeState)
+# define XXH32_reset XXH_IPREF(XXH32_reset)
+# define XXH32_update XXH_IPREF(XXH32_update)
+# define XXH32_digest XXH_IPREF(XXH32_digest)
+# define XXH32_copyState XXH_IPREF(XXH32_copyState)
+# define XXH32_canonicalFromHash XXH_IPREF(XXH32_canonicalFromHash)
+# define XXH32_hashFromCanonical XXH_IPREF(XXH32_hashFromCanonical)
/* XXH64 */
-# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
-# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
-# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
-# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
-# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
-# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
-# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
-# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
-# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
+# define XXH64 XXH_IPREF(XXH64)
+# define XXH64_createState XXH_IPREF(XXH64_createState)
+# define XXH64_freeState XXH_IPREF(XXH64_freeState)
+# define XXH64_reset XXH_IPREF(XXH64_reset)
+# define XXH64_update XXH_IPREF(XXH64_update)
+# define XXH64_digest XXH_IPREF(XXH64_digest)
+# define XXH64_copyState XXH_IPREF(XXH64_copyState)
+# define XXH64_canonicalFromHash XXH_IPREF(XXH64_canonicalFromHash)
+# define XXH64_hashFromCanonical XXH_IPREF(XXH64_hashFromCanonical)
/* XXH3_64bits */
-# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits)
-# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret)
-# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed)
-# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed)
-# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState)
-# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState)
-# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState)
-# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset)
-# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed)
-# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret)
-# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed)
-# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update)
-# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest)
-# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret)
-# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed)
+# define XXH3_64bits XXH_IPREF(XXH3_64bits)
+# define XXH3_64bits_withSecret XXH_IPREF(XXH3_64bits_withSecret)
+# define XXH3_64bits_withSeed XXH_IPREF(XXH3_64bits_withSeed)
+# define XXH3_64bits_withSecretandSeed XXH_IPREF(XXH3_64bits_withSecretandSeed)
+# define XXH3_createState XXH_IPREF(XXH3_createState)
+# define XXH3_freeState XXH_IPREF(XXH3_freeState)
+# define XXH3_copyState XXH_IPREF(XXH3_copyState)
+# define XXH3_64bits_reset XXH_IPREF(XXH3_64bits_reset)
+# define XXH3_64bits_reset_withSeed XXH_IPREF(XXH3_64bits_reset_withSeed)
+# define XXH3_64bits_reset_withSecret XXH_IPREF(XXH3_64bits_reset_withSecret)
+# define XXH3_64bits_reset_withSecretandSeed XXH_IPREF(XXH3_64bits_reset_withSecretandSeed)
+# define XXH3_64bits_update XXH_IPREF(XXH3_64bits_update)
+# define XXH3_64bits_digest XXH_IPREF(XXH3_64bits_digest)
+# define XXH3_generateSecret XXH_IPREF(XXH3_generateSecret)
+# define XXH3_generateSecret_fromSeed XXH_IPREF(XXH3_generateSecret_fromSeed)
/* XXH3_128bits */
-# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128)
-# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits)
-# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed)
-# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret)
-# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed)
-# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset)
-# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed)
-# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret)
-# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed)
-# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update)
-# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest)
-# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual)
-# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp)
-# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash)
-# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical)
+# define XXH128 XXH_IPREF(XXH128)
+# define XXH3_128bits XXH_IPREF(XXH3_128bits)
+# define XXH3_128bits_withSeed XXH_IPREF(XXH3_128bits_withSeed)
+# define XXH3_128bits_withSecret XXH_IPREF(XXH3_128bits_withSecret)
+# define XXH3_128bits_withSecretandSeed XXH_IPREF(XXH3_128bits_withSecretandSeed)
+# define XXH3_128bits_reset XXH_IPREF(XXH3_128bits_reset)
+# define XXH3_128bits_reset_withSeed XXH_IPREF(XXH3_128bits_reset_withSeed)
+# define XXH3_128bits_reset_withSecret XXH_IPREF(XXH3_128bits_reset_withSecret)
+# define XXH3_128bits_reset_withSecretandSeed XXH_IPREF(XXH3_128bits_reset_withSecretandSeed)
+# define XXH3_128bits_update XXH_IPREF(XXH3_128bits_update)
+# define XXH3_128bits_digest XXH_IPREF(XXH3_128bits_digest)
+# define XXH128_isEqual XXH_IPREF(XXH128_isEqual)
+# define XXH128_cmp XXH_IPREF(XXH128_cmp)
+# define XXH128_canonicalFromHash XXH_IPREF(XXH128_canonicalFromHash)
+# define XXH128_hashFromCanonical XXH_IPREF(XXH128_hashFromCanonical)
#endif
@@ -547,7 +548,7 @@ extern "C" {
***************************************/
#define XXH_VERSION_MAJOR 0
#define XXH_VERSION_MINOR 8
-#define XXH_VERSION_RELEASE 3
+#define XXH_VERSION_RELEASE 4
/*! @brief Version number, encoded as two digits each */
#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
@@ -1696,11 +1697,11 @@ struct XXH64_state_s {
#ifndef XXH_NO_XXH3
-#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */
-# define XXH_ALIGN(n) _Alignas(n)
-#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */
+#if defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */
/* In C++ alignas() is a keyword */
# define XXH_ALIGN(n) alignas(n)
+#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* > C11 */
+# define XXH_ALIGN(n) _Alignas(n)
#elif defined(__GNUC__)
# define XXH_ALIGN(n) __attribute__ ((aligned(n)))
#elif defined(_MSC_VER)
@@ -2051,6 +2052,10 @@ XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */
+#if defined (__cplusplus) && !defined(XXH_NO_EXTERNC_GUARD)
+} /* extern "C" */
+#endif
+
/* ======================================================================== */
/* ======================================================================== */
@@ -2460,7 +2465,8 @@ static void XXH_free(void* p) { free(p); }
# define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE
#endif
-#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */
+/* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */
+#if ((defined(sun) || defined(__sun)) && defined(__cplusplus))
# define XXH_RESTRICT /* disable */
#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */
# define XXH_RESTRICT restrict
@@ -2509,10 +2515,10 @@ static void XXH_free(void* p) { free(p); }
/* note: use after variable declarations */
#ifndef XXH_STATIC_ASSERT
-# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */
-# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0)
-# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */
+# if defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */
# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0)
+# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */
+# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0)
# else
# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0)
# endif
@@ -3166,6 +3172,11 @@ XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align)
* @param align Whether @p input is aligned.
* @return The calculated hash.
*/
+#if defined(_MSC_VER)
+# pragma warning(push)
+/* MSVC's LTCG reports valid branches eliminated for constant input lengths. */
+# pragma warning(disable : 4702) /* C4702: unreachable code */
+#endif
XXH_FORCE_INLINE XXH_PUREF xxh_u32
XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align)
{
@@ -3188,6 +3199,9 @@ XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment
return XXH32_finalize(h32, input, len&15, align);
}
+#if defined(_MSC_VER)
+# pragma warning(pop)
+#endif
/*! @ingroup XXH32_family */
XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed)
@@ -4735,30 +4749,59 @@ XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_
}
/*
- * DISCLAIMER: There are known *seed-dependent* multicollisions here due to
- * multiplication by zero, affecting hashes of lengths 17 to 240.
- *
- * However, they are very unlikely.
- *
- * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all
- * unseeded non-cryptographic hashes, it does not attempt to defend itself
- * against specially crafted inputs, only random inputs.
- *
- * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes
- * cancelling out the secret is taken an arbitrary number of times (addressed
- * in XXH3_accumulate_512), this collision is very unlikely with random inputs
- * and/or proper seeding:
- *
- * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a
- * function that is only called up to 16 times per hash with up to 240 bytes of
- * input.
- *
- * This is not too bad for a non-cryptographic hash function, especially with
- * only 64 bit outputs.
- *
- * The 128-bit variant (which trades some speed for strength) is NOT affected
- * by this, although it is always a good idea to use a proper seed if you care
- * about strength.
+ * DISCLAIMER: the mid-size code path built on XXH3_mix16B() (lengths 17 to 240)
+ * has known collision weaknesses. XXH3 is a non-cryptographic hash: collision
+ * resistance is not one of its guarantees, and the seed (or custom secret) only
+ * makes intentional collisions *harder*, it is not a security boundary.
+ * These weaknesses are nonetheless documented here, so that users relying on a
+ * seed for a low per-pair collision probability know what to expect.
+ *
+ * 1) XXH3_mul128_fold64() does not separate complemented multiplicands.
+ * Modulo 2^128, and with s = a + b + 1 :
+ * (~a) * (~b) == a*b + s - (s + 1) * 2^64
+ * so the low half of the product gains `s` while the high half loses `s`.
+ * Since carry chains are short, both halves happen to flip the same bits
+ * fairly often, leaving their xor (the "fold") unchanged with probability
+ * ~2^-27, instead of the ideal 2^-64. Complementing a single multiplicand
+ * behaves the same way.
+ *
+ * Consequence for XXH3_64bits(): for any length in [32, 240], two messages
+ * differing only by the complement of their first 8 (or first 16) bytes
+ * collide with probability ~2^-27 over a uniformly random secret. The
+ * _withSeed variant is also affected, but its collision probability depends
+ * on the chosen pair because the 64-bit seed produces a structured secret.
+ * A fixed pair has been measured to collide with probability ~2^-21 over
+ * random seeds. This is an observed result, not a known upper bound.
+ *
+ * Consequence for XXH128(): XXH128_mix32B() mixes each 16-byte chunk a
+ * second time as a raw 64-bit sum, which normally breaks the pattern above.
+ * But that sum is *also* invariant under complementation when
+ * w0 + w1 == 2^64 - 1, and under that additional constraint both output
+ * halves collide simultaneously, again with probability ~2^-27 over a
+ * uniformly random secret. The seeded variant is also affected, and its
+ * exact probability can similarly depend on the chosen pair.
+ * So the 128-bit variant is _not_ immune, contrarily to what was previously
+ * believed, and its effective margin here (2^-27 vs. an ideal 2^-128) is
+ * even thinner than the 64-bit one.
+ *
+ * Detailed analysis and measurements in
+ * https://github.com/Cyan4973/xxHash/issues/1127 .
+ *
+ * 2) *seed-dependent* multicollisions, due to multiplication by zero: when one
+ * of the two multiplicands below is zero, the whole 16-byte chunk stops
+ * contributing to the hash, so all chunks sharing this property collide.
+ * For random inputs, this is a 1 in 2^63 chance of 8 consecutive bytes
+ * cancelling out, in a function invoked at most 16 times per hash, hence
+ * essentially unreachable by accident. But it is trivially reachable by an
+ * attacker who knows the secret (i.e. the unseeded variants). For
+ * comparison, classic UMAC takes a 1 in 2^31 chance of 4 consecutive bytes
+ * cancelling out the secret an arbitrary number of times (addressed for
+ * long inputs in XXH3_accumulate_512).
+ *
+ * None of the above can be fixed without changing XXH3_mix16B(), which would
+ * change every hash value in the [17, 240] range, and XXH3 output values are
+ * now frozen. Therefore, if adversarial collision resistance matters for your
+ * use case, use a cryptographic hash or a MAC instead.
*/
XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input,
const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64)
@@ -5074,7 +5117,7 @@ XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
(void)(&XXH_writeLE64);
{ int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i);
__m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64);
- __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos);
+ __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_setzero_si512(), seed_pos);
const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret);
__m512i* const dest = ( __m512i*) customSecret;
@@ -6033,7 +6076,7 @@ XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc,
{
size_t i;
/* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */
-#if defined(__GNUC__) && !defined(__clang__) \
+#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 8 \
&& (defined(__arm__) || defined(__thumb2__)) \
&& defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \
&& XXH_SIZE_OPT <= 0
@@ -6854,8 +6897,10 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t*
* XXH3's 128-bit variant has better mixing and strength than the 64-bit variant,
* even without counting the significantly larger output size.
*
- * For example, extra steps are taken to avoid the seed-dependent collisions
- * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B).
+ * For example, extra steps are taken to mitigate the seed-dependent collisions
+ * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). Note that these
+ * steps only mitigate, they do not eliminate the issue: see the DISCLAIMER
+ * above XXH3_mix16B() for the cases that get through.
*
* This strength naturally comes at the cost of some speed, especially on short
* lengths. Note that longer hashes are about as fast as the 64-bit version
@@ -7481,8 +7526,3 @@ XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed)
* @}
*/
#endif /* XXH_IMPLEMENTATION */
-
-
-#if defined (__cplusplus) && !defined(XXH_NO_EXTERNC_GUARD)
-} /* extern "C" */
-#endif