svn commit: r245868 - in head/contrib/jemalloc: . doc include/jemalloc include/jemalloc/internal src
Jason Evans
jasone at FreeBSD.org
Thu Jan 24 03:08:39 UTC 2013
Author: jasone
Date: Thu Jan 24 03:08:37 2013
New Revision: 245868
URL: http://svnweb.freebsd.org/changeset/base/245868
Log:
Import jemalloc 3.3.0. This reduces zeroed memory validation overhead for
non-MALLOC_PRODUCTION builds.
Modified:
head/contrib/jemalloc/COPYING
head/contrib/jemalloc/ChangeLog
head/contrib/jemalloc/FREEBSD-diffs
head/contrib/jemalloc/VERSION
head/contrib/jemalloc/doc/jemalloc.3
head/contrib/jemalloc/include/jemalloc/internal/arena.h
head/contrib/jemalloc/include/jemalloc/internal/ckh.h
head/contrib/jemalloc/include/jemalloc/internal/hash.h
head/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal.h
head/contrib/jemalloc/include/jemalloc/internal/private_namespace.h
head/contrib/jemalloc/include/jemalloc/internal/tcache.h
head/contrib/jemalloc/include/jemalloc/jemalloc.h
head/contrib/jemalloc/include/jemalloc/jemalloc_defs.h
head/contrib/jemalloc/src/arena.c
head/contrib/jemalloc/src/chunk.c
head/contrib/jemalloc/src/chunk_dss.c
head/contrib/jemalloc/src/ckh.c
head/contrib/jemalloc/src/ctl.c
head/contrib/jemalloc/src/jemalloc.c
head/contrib/jemalloc/src/prof.c
head/contrib/jemalloc/src/tcache.c
Modified: head/contrib/jemalloc/COPYING
==============================================================================
--- head/contrib/jemalloc/COPYING Thu Jan 24 02:25:43 2013 (r245867)
+++ head/contrib/jemalloc/COPYING Thu Jan 24 03:08:37 2013 (r245868)
@@ -1,10 +1,10 @@
Unless otherwise specified, files in the jemalloc source distribution are
subject to the following license:
--------------------------------------------------------------------------------
-Copyright (C) 2002-2012 Jason Evans <jasone at canonware.com>.
+Copyright (C) 2002-2013 Jason Evans <jasone at canonware.com>.
All rights reserved.
Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved.
-Copyright (C) 2009-2012 Facebook, Inc. All rights reserved.
+Copyright (C) 2009-2013 Facebook, Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
Modified: head/contrib/jemalloc/ChangeLog
==============================================================================
--- head/contrib/jemalloc/ChangeLog Thu Jan 24 02:25:43 2013 (r245867)
+++ head/contrib/jemalloc/ChangeLog Thu Jan 24 03:08:37 2013 (r245868)
@@ -6,6 +6,23 @@ found in the git revision history:
http://www.canonware.com/cgi-bin/gitweb.cgi?p=jemalloc.git
git://canonware.com/jemalloc.git
+* 3.3.0 (January 23, 2013)
+
+ This version includes a few minor performance improvements in addition to the
+ listed new features and bug fixes.
+
+ New features:
+ - Add clipping support to lg_chunk option processing.
+ - Add the --enable-ivsalloc option.
+ - Add the --without-export option.
+ - Add the --disable-zone-allocator option.
+
+ Bug fixes:
+ - Fix "arenas.extend" mallctl to output the number of arenas.
+ - Fix chunk_recycyle() to unconditionally inform Valgrind that returned memory
+ is undefined.
+ - Fix build break on FreeBSD related to alloca.h.
+
* 3.2.0 (November 9, 2012)
In addition to a couple of bug fixes, this version modifies page run
Modified: head/contrib/jemalloc/FREEBSD-diffs
==============================================================================
--- head/contrib/jemalloc/FREEBSD-diffs Thu Jan 24 02:25:43 2013 (r245867)
+++ head/contrib/jemalloc/FREEBSD-diffs Thu Jan 24 03:08:37 2013 (r245868)
@@ -1,5 +1,5 @@
diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in
-index 54b8747..91c4a4e 100644
+index 0930580..d45fa3d 100644
--- a/doc/jemalloc.xml.in
+++ b/doc/jemalloc.xml.in
@@ -51,12 +51,23 @@
@@ -27,7 +27,7 @@ index 54b8747..91c4a4e 100644
<refsect2>
<title>Standard API</title>
<funcprototype>
-@@ -2170,4 +2181,16 @@ malloc_conf = "lg_chunk:24";]]></programlisting></para>
+@@ -2173,4 +2184,16 @@ malloc_conf = "lg_chunk:24";]]></programlisting></para>
<para>The <function>posix_memalign<parameter/></function> function conforms
to IEEE Std 1003.1-2001 (“POSIX.1”).</para>
</refsect1>
@@ -45,7 +45,7 @@ index 54b8747..91c4a4e 100644
+ </refsect1>
</refentry>
diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in
-index 475821a..73306ac 100644
+index c606c12..0d46d9d 100644
--- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -1,5 +1,8 @@
@@ -97,10 +97,10 @@ index de44e14..564d604 100644
bool malloc_mutex_init(malloc_mutex_t *mutex);
diff --git a/include/jemalloc/internal/private_namespace.h b/include/jemalloc/internal/private_namespace.h
-index 06241cd..7b19906 100644
+index 903fb4d..d6638df 100644
--- a/include/jemalloc/internal/private_namespace.h
+++ b/include/jemalloc/internal/private_namespace.h
-@@ -204,7 +204,6 @@
+@@ -216,7 +216,6 @@
#define iralloc JEMALLOC_N(iralloc)
#define irallocx JEMALLOC_N(irallocx)
#define isalloc JEMALLOC_N(isalloc)
@@ -203,7 +203,7 @@ index 0000000..9c97a13
+#define pthread_mutex_lock _pthread_mutex_lock
+#define pthread_mutex_unlock _pthread_mutex_unlock
diff --git a/src/jemalloc.c b/src/jemalloc.c
-index 8a667b6..aaf5012 100644
+index c117685..665d98f 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -8,6 +8,10 @@ malloc_tsd_data(, arenas, arena_t *, NULL)
@@ -216,8 +216,8 @@ index 8a667b6..aaf5012 100644
+
/* Runtime configuration options. */
const char *je_malloc_conf;
- #ifdef JEMALLOC_DEBUG
-@@ -448,7 +452,8 @@ malloc_conf_init(void)
+ bool opt_abort =
+@@ -453,7 +457,8 @@ malloc_conf_init(void)
#endif
;
Modified: head/contrib/jemalloc/VERSION
==============================================================================
--- head/contrib/jemalloc/VERSION Thu Jan 24 02:25:43 2013 (r245867)
+++ head/contrib/jemalloc/VERSION Thu Jan 24 03:08:37 2013 (r245868)
@@ -1 +1 @@
-3.2.0-0-g87499f6748ebe4817571e817e9f680ccb5bf54a9
+3.3.0-0-g83789f45307379e096c4e8be81d9e9a51e3f5a4a
Modified: head/contrib/jemalloc/doc/jemalloc.3
==============================================================================
--- head/contrib/jemalloc/doc/jemalloc.3 Thu Jan 24 02:25:43 2013 (r245867)
+++ head/contrib/jemalloc/doc/jemalloc.3 Thu Jan 24 03:08:37 2013 (r245868)
@@ -2,12 +2,12 @@
.\" Title: JEMALLOC
.\" Author: Jason Evans
.\" Generator: DocBook XSL Stylesheets v1.76.1 <http://docbook.sf.net/>
-.\" Date: 11/09/2012
+.\" Date: 01/23/2013
.\" Manual: User Manual
-.\" Source: jemalloc 3.2.0-0-g87499f6748ebe4817571e817e9f680ccb5bf54a9
+.\" Source: jemalloc 3.3.0-0-g83789f45307379e096c4e8be81d9e9a51e3f5a4a
.\" Language: English
.\"
-.TH "JEMALLOC" "3" "11/09/2012" "jemalloc 3.2.0-0-g87499f6748eb" "User Manual"
+.TH "JEMALLOC" "3" "01/23/2013" "jemalloc 3.3.0-0-g83789f453073" "User Manual"
.\" -----------------------------------------------------------------
.\" * Define some portability stuff
.\" -----------------------------------------------------------------
@@ -31,7 +31,7 @@
jemalloc \- general purpose memory allocation functions
.SH "LIBRARY"
.PP
-This manual describes jemalloc 3\&.2\&.0\-0\-g87499f6748ebe4817571e817e9f680ccb5bf54a9\&. More information can be found at the
+This manual describes jemalloc 3\&.3\&.0\-0\-g83789f45307379e096c4e8be81d9e9a51e3f5a4a\&. More information can be found at the
\m[blue]\fBjemalloc website\fR\m[]\&\s-2\u[1]\d\s+2\&.
.PP
The following configuration options are enabled in libc\*(Aqs built\-in jemalloc:
@@ -652,7 +652,7 @@ is specified during configuration, in wh
.PP
"opt\&.lg_chunk" (\fBsize_t\fR) r\-
.RS 4
-Virtual memory chunk size (log base 2)\&. The default chunk size is 4 MiB (2^22)\&.
+Virtual memory chunk size (log base 2)\&. If a chunk size outside the supported size range is specified, the size is silently clipped to the minimum/maximum supported size\&. The default chunk size is 4 MiB (2^22)\&.
.RE
.PP
"opt\&.dss" (\fBconst char *\fR) r\-
Modified: head/contrib/jemalloc/include/jemalloc/internal/arena.h
==============================================================================
--- head/contrib/jemalloc/include/jemalloc/internal/arena.h Thu Jan 24 02:25:43 2013 (r245867)
+++ head/contrib/jemalloc/include/jemalloc/internal/arena.h Thu Jan 24 03:08:37 2013 (r245868)
@@ -400,7 +400,6 @@ extern arena_bin_info_t arena_bin_info[N
#define nlclasses (chunk_npages - map_bias)
void arena_purge_all(arena_t *arena);
-void arena_prof_accum(arena_t *arena, uint64_t accumbytes);
void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin,
size_t binind, uint64_t prof_accumbytes);
void arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info,
@@ -464,6 +463,9 @@ void arena_mapbits_small_set(arena_chunk
size_t runind, size_t binind, size_t flags);
void arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind,
size_t unzeroed);
+void arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes);
+void arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes);
+void arena_prof_accum(arena_t *arena, uint64_t accumbytes);
size_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits);
size_t arena_bin_index(arena_t *arena, arena_bin_t *bin);
unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info,
@@ -478,7 +480,7 @@ void arena_dalloc(arena_t *arena, arena_
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_))
# ifdef JEMALLOC_ARENA_INLINE_A
-JEMALLOC_INLINE arena_chunk_map_t *
+JEMALLOC_ALWAYS_INLINE arena_chunk_map_t *
arena_mapp_get(arena_chunk_t *chunk, size_t pageind)
{
@@ -488,21 +490,21 @@ arena_mapp_get(arena_chunk_t *chunk, siz
return (&chunk->map[pageind-map_bias]);
}
-JEMALLOC_INLINE size_t *
+JEMALLOC_ALWAYS_INLINE size_t *
arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind)
{
return (&arena_mapp_get(chunk, pageind)->bits);
}
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_get(arena_chunk_t *chunk, size_t pageind)
{
return (*arena_mapbitsp_get(chunk, pageind));
}
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
@@ -512,7 +514,7 @@ arena_mapbits_unallocated_size_get(arena
return (mapbits & ~PAGE_MASK);
}
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
@@ -523,7 +525,7 @@ arena_mapbits_large_size_get(arena_chunk
return (mapbits & ~PAGE_MASK);
}
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
@@ -534,7 +536,7 @@ arena_mapbits_small_runind_get(arena_chu
return (mapbits >> LG_PAGE);
}
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
@@ -546,7 +548,7 @@ arena_mapbits_binind_get(arena_chunk_t *
return (binind);
}
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
@@ -555,7 +557,7 @@ arena_mapbits_dirty_get(arena_chunk_t *c
return (mapbits & CHUNK_MAP_DIRTY);
}
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
@@ -564,7 +566,7 @@ arena_mapbits_unzeroed_get(arena_chunk_t
return (mapbits & CHUNK_MAP_UNZEROED);
}
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
@@ -573,7 +575,7 @@ arena_mapbits_large_get(arena_chunk_t *c
return (mapbits & CHUNK_MAP_LARGE);
}
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
@@ -582,7 +584,7 @@ arena_mapbits_allocated_get(arena_chunk_
return (mapbits & CHUNK_MAP_ALLOCATED);
}
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size,
size_t flags)
{
@@ -595,7 +597,7 @@ arena_mapbits_unallocated_set(arena_chun
*mapbitsp = size | CHUNK_MAP_BININD_INVALID | flags;
}
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind,
size_t size)
{
@@ -607,7 +609,7 @@ arena_mapbits_unallocated_size_set(arena
*mapbitsp = size | (*mapbitsp & PAGE_MASK);
}
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size,
size_t flags)
{
@@ -622,7 +624,7 @@ arena_mapbits_large_set(arena_chunk_t *c
CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED;
}
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind,
size_t binind)
{
@@ -635,7 +637,7 @@ arena_mapbits_large_binind_set(arena_chu
CHUNK_MAP_BININD_SHIFT);
}
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind,
size_t binind, size_t flags)
{
@@ -651,7 +653,7 @@ arena_mapbits_small_set(arena_chunk_t *c
flags | unzeroed | CHUNK_MAP_ALLOCATED;
}
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind,
size_t unzeroed)
{
@@ -661,7 +663,45 @@ arena_mapbits_unzeroed_set(arena_chunk_t
*mapbitsp = (*mapbitsp & ~CHUNK_MAP_UNZEROED) | unzeroed;
}
-JEMALLOC_INLINE size_t
+JEMALLOC_INLINE void
+arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes)
+{
+
+ cassert(config_prof);
+ assert(prof_interval != 0);
+
+ arena->prof_accumbytes += accumbytes;
+ if (arena->prof_accumbytes >= prof_interval) {
+ prof_idump();
+ arena->prof_accumbytes -= prof_interval;
+ }
+}
+
+JEMALLOC_INLINE void
+arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes)
+{
+
+ cassert(config_prof);
+
+ if (prof_interval == 0)
+ return;
+ arena_prof_accum_impl(arena, accumbytes);
+}
+
+JEMALLOC_INLINE void
+arena_prof_accum(arena_t *arena, uint64_t accumbytes)
+{
+
+ cassert(config_prof);
+
+ if (prof_interval == 0)
+ return;
+ malloc_mutex_lock(&arena->lock);
+ arena_prof_accum_impl(arena, accumbytes);
+ malloc_mutex_unlock(&arena->lock);
+}
+
+JEMALLOC_ALWAYS_INLINE size_t
arena_ptr_small_binind_get(const void *ptr, size_t mapbits)
{
size_t binind;
@@ -856,7 +896,7 @@ arena_prof_ctx_set(const void *ptr, prof
arena_mapp_get(chunk, pageind)->prof_ctx = ctx;
}
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache)
{
tcache_t *tcache;
@@ -887,7 +927,7 @@ arena_malloc(arena_t *arena, size_t size
}
/* Return the size of the allocation pointed to by ptr. */
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
arena_salloc(const void *ptr, bool demote)
{
size_t ret;
@@ -933,7 +973,7 @@ arena_salloc(const void *ptr, bool demot
return (ret);
}
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache)
{
size_t pageind, mapbits;
Modified: head/contrib/jemalloc/include/jemalloc/internal/ckh.h
==============================================================================
--- head/contrib/jemalloc/include/jemalloc/internal/ckh.h Thu Jan 24 02:25:43 2013 (r245867)
+++ head/contrib/jemalloc/include/jemalloc/internal/ckh.h Thu Jan 24 03:08:37 2013 (r245868)
@@ -5,7 +5,7 @@ typedef struct ckh_s ckh_t;
typedef struct ckhc_s ckhc_t;
/* Typedefs to allow easy function pointer passing. */
-typedef void ckh_hash_t (const void *, unsigned, size_t *, size_t *);
+typedef void ckh_hash_t (const void *, size_t[2]);
typedef bool ckh_keycomp_t (const void *, const void *);
/* Maintain counters used to get an idea of performance. */
@@ -75,11 +75,9 @@ bool ckh_insert(ckh_t *ckh, const void *
bool ckh_remove(ckh_t *ckh, const void *searchkey, void **key,
void **data);
bool ckh_search(ckh_t *ckh, const void *seachkey, void **key, void **data);
-void ckh_string_hash(const void *key, unsigned minbits, size_t *hash1,
- size_t *hash2);
+void ckh_string_hash(const void *key, size_t r_hash[2]);
bool ckh_string_keycomp(const void *k1, const void *k2);
-void ckh_pointer_hash(const void *key, unsigned minbits, size_t *hash1,
- size_t *hash2);
+void ckh_pointer_hash(const void *key, size_t r_hash[2]);
bool ckh_pointer_keycomp(const void *k1, const void *k2);
#endif /* JEMALLOC_H_EXTERNS */
Modified: head/contrib/jemalloc/include/jemalloc/internal/hash.h
==============================================================================
--- head/contrib/jemalloc/include/jemalloc/internal/hash.h Thu Jan 24 02:25:43 2013 (r245867)
+++ head/contrib/jemalloc/include/jemalloc/internal/hash.h Thu Jan 24 03:08:37 2013 (r245868)
@@ -1,3 +1,8 @@
+/*
+ * The following hash function is based on MurmurHash3, placed into the public
+ * domain by Austin Appleby. See http://code.google.com/p/smhasher/ for
+ * details.
+ */
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
@@ -14,55 +19,311 @@
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
-uint64_t hash(const void *key, size_t len, uint64_t seed);
+void hash(const void *key, size_t len, const uint32_t seed,
+ size_t r_hash[2]);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_HASH_C_))
-/*
- * The following hash function is based on MurmurHash64A(), placed into the
- * public domain by Austin Appleby. See http://murmurhash.googlepages.com/ for
- * details.
- */
+/******************************************************************************/
+/* Internal implementation. */
+JEMALLOC_INLINE uint32_t
+hash_rotl_32(uint32_t x, int8_t r)
+{
+
+ return (x << r) | (x >> (32 - r));
+}
+
JEMALLOC_INLINE uint64_t
-hash(const void *key, size_t len, uint64_t seed)
+hash_rotl_64(uint64_t x, int8_t r)
+{
+ return (x << r) | (x >> (64 - r));
+}
+
+JEMALLOC_INLINE uint32_t
+hash_get_block_32(const uint32_t *p, int i)
+{
+
+ return p[i];
+}
+
+JEMALLOC_INLINE uint64_t
+hash_get_block_64(const uint64_t *p, int i)
+{
+
+ return p[i];
+}
+
+JEMALLOC_INLINE uint32_t
+hash_fmix_32(uint32_t h)
+{
+
+ h ^= h >> 16;
+ h *= 0x85ebca6b;
+ h ^= h >> 13;
+ h *= 0xc2b2ae35;
+ h ^= h >> 16;
+
+ return h;
+}
+
+JEMALLOC_INLINE uint64_t
+hash_fmix_64(uint64_t k)
+{
+
+ k ^= k >> 33;
+ k *= QU(0xff51afd7ed558ccdLLU);
+ k ^= k >> 33;
+ k *= QU(0xc4ceb9fe1a85ec53LLU);
+ k ^= k >> 33;
+
+ return k;
+}
+
+JEMALLOC_INLINE uint32_t
+hash_x86_32(const void *key, int len, uint32_t seed)
+{
+ const uint8_t *data = (const uint8_t *) key;
+ const int nblocks = len / 4;
+
+ uint32_t h1 = seed;
+
+ const uint32_t c1 = 0xcc9e2d51;
+ const uint32_t c2 = 0x1b873593;
+
+ /* body */
+ {
+ const uint32_t *blocks = (const uint32_t *) (data + nblocks*4);
+ int i;
+
+ for (i = -nblocks; i; i++) {
+ uint32_t k1 = hash_get_block_32(blocks, i);
+
+ k1 *= c1;
+ k1 = hash_rotl_32(k1, 15);
+ k1 *= c2;
+
+ h1 ^= k1;
+ h1 = hash_rotl_32(h1, 13);
+ h1 = h1*5 + 0xe6546b64;
+ }
+ }
+
+ /* tail */
+ {
+ const uint8_t *tail = (const uint8_t *) (data + nblocks*4);
+
+ uint32_t k1 = 0;
+
+ switch (len & 3) {
+ case 3: k1 ^= tail[2] << 16;
+ case 2: k1 ^= tail[1] << 8;
+ case 1: k1 ^= tail[0]; k1 *= c1; k1 = hash_rotl_32(k1, 15);
+ k1 *= c2; h1 ^= k1;
+ }
+ }
+
+ /* finalization */
+ h1 ^= len;
+
+ h1 = hash_fmix_32(h1);
+
+ return h1;
+}
+
+UNUSED JEMALLOC_INLINE void
+hash_x86_128(const void *key, const int len, uint32_t seed,
+ uint64_t r_out[2])
{
- const uint64_t m = UINT64_C(0xc6a4a7935bd1e995);
- const int r = 47;
- uint64_t h = seed ^ (len * m);
- const uint64_t *data = (const uint64_t *)key;
- const uint64_t *end = data + (len/8);
- const unsigned char *data2;
+ const uint8_t * data = (const uint8_t *) key;
+ const int nblocks = len / 16;
+
+ uint32_t h1 = seed;
+ uint32_t h2 = seed;
+ uint32_t h3 = seed;
+ uint32_t h4 = seed;
+
+ const uint32_t c1 = 0x239b961b;
+ const uint32_t c2 = 0xab0e9789;
+ const uint32_t c3 = 0x38b34ae5;
+ const uint32_t c4 = 0xa1e38b93;
+
+ /* body */
+ {
+ const uint32_t *blocks = (const uint32_t *) (data + nblocks*16);
+ int i;
+
+ for (i = -nblocks; i; i++) {
+ uint32_t k1 = hash_get_block_32(blocks, i*4 + 0);
+ uint32_t k2 = hash_get_block_32(blocks, i*4 + 1);
+ uint32_t k3 = hash_get_block_32(blocks, i*4 + 2);
+ uint32_t k4 = hash_get_block_32(blocks, i*4 + 3);
+
+ k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1;
+
+ h1 = hash_rotl_32(h1, 19); h1 += h2;
+ h1 = h1*5 + 0x561ccd1b;
- assert(((uintptr_t)key & 0x7) == 0);
+ k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2;
- while(data != end) {
- uint64_t k = *data++;
+ h2 = hash_rotl_32(h2, 17); h2 += h3;
+ h2 = h2*5 + 0x0bcaa747;
- k *= m;
- k ^= k >> r;
- k *= m;
+ k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3;
- h ^= k;
- h *= m;
+ h3 = hash_rotl_32(h3, 15); h3 += h4;
+ h3 = h3*5 + 0x96cd1c35;
+
+ k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4;
+
+ h4 = hash_rotl_32(h4, 13); h4 += h1;
+ h4 = h4*5 + 0x32ac3b17;
+ }
}
- data2 = (const unsigned char *)data;
- switch(len & 7) {
- case 7: h ^= ((uint64_t)(data2[6])) << 48;
- case 6: h ^= ((uint64_t)(data2[5])) << 40;
- case 5: h ^= ((uint64_t)(data2[4])) << 32;
- case 4: h ^= ((uint64_t)(data2[3])) << 24;
- case 3: h ^= ((uint64_t)(data2[2])) << 16;
- case 2: h ^= ((uint64_t)(data2[1])) << 8;
- case 1: h ^= ((uint64_t)(data2[0]));
- h *= m;
+ /* tail */
+ {
+ const uint8_t *tail = (const uint8_t *) (data + nblocks*16);
+ uint32_t k1 = 0;
+ uint32_t k2 = 0;
+ uint32_t k3 = 0;
+ uint32_t k4 = 0;
+
+ switch (len & 15) {
+ case 15: k4 ^= tail[14] << 16;
+ case 14: k4 ^= tail[13] << 8;
+ case 13: k4 ^= tail[12] << 0;
+ k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4;
+
+ case 12: k3 ^= tail[11] << 24;
+ case 11: k3 ^= tail[10] << 16;
+ case 10: k3 ^= tail[ 9] << 8;
+ case 9: k3 ^= tail[ 8] << 0;
+ k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3;
+
+ case 8: k2 ^= tail[ 7] << 24;
+ case 7: k2 ^= tail[ 6] << 16;
+ case 6: k2 ^= tail[ 5] << 8;
+ case 5: k2 ^= tail[ 4] << 0;
+ k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2;
+
+ case 4: k1 ^= tail[ 3] << 24;
+ case 3: k1 ^= tail[ 2] << 16;
+ case 2: k1 ^= tail[ 1] << 8;
+ case 1: k1 ^= tail[ 0] << 0;
+ k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1;
+ }
}
- h ^= h >> r;
- h *= m;
- h ^= h >> r;
+ /* finalization */
+ h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len;
+
+ h1 += h2; h1 += h3; h1 += h4;
+ h2 += h1; h3 += h1; h4 += h1;
+
+ h1 = hash_fmix_32(h1);
+ h2 = hash_fmix_32(h2);
+ h3 = hash_fmix_32(h3);
+ h4 = hash_fmix_32(h4);
+
+ h1 += h2; h1 += h3; h1 += h4;
+ h2 += h1; h3 += h1; h4 += h1;
+
+ r_out[0] = (((uint64_t) h2) << 32) | h1;
+ r_out[1] = (((uint64_t) h4) << 32) | h3;
+}
+
+UNUSED JEMALLOC_INLINE void
+hash_x64_128(const void *key, const int len, const uint32_t seed,
+ uint64_t r_out[2])
+{
+ const uint8_t *data = (const uint8_t *) key;
+ const int nblocks = len / 16;
+
+ uint64_t h1 = seed;
+ uint64_t h2 = seed;
- return (h);
+ const uint64_t c1 = QU(0x87c37b91114253d5LLU);
+ const uint64_t c2 = QU(0x4cf5ad432745937fLLU);
+
+ /* body */
+ {
+ const uint64_t *blocks = (const uint64_t *) (data);
+ int i;
+
+ for (i = 0; i < nblocks; i++) {
+ uint64_t k1 = hash_get_block_64(blocks, i*2 + 0);
+ uint64_t k2 = hash_get_block_64(blocks, i*2 + 1);
+
+ k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1;
+
+ h1 = hash_rotl_64(h1, 27); h1 += h2;
+ h1 = h1*5 + 0x52dce729;
+
+ k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2;
+
+ h2 = hash_rotl_64(h2, 31); h2 += h1;
+ h2 = h2*5 + 0x38495ab5;
+ }
+ }
+
+ /* tail */
+ {
+ const uint8_t *tail = (const uint8_t*)(data + nblocks*16);
+ uint64_t k1 = 0;
+ uint64_t k2 = 0;
+
+ switch (len & 15) {
+ case 15: k2 ^= ((uint64_t)(tail[14])) << 48;
+ case 14: k2 ^= ((uint64_t)(tail[13])) << 40;
+ case 13: k2 ^= ((uint64_t)(tail[12])) << 32;
+ case 12: k2 ^= ((uint64_t)(tail[11])) << 24;
+ case 11: k2 ^= ((uint64_t)(tail[10])) << 16;
+ case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8;
+ case 9: k2 ^= ((uint64_t)(tail[ 8])) << 0;
+ k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2;
+
+ case 8: k1 ^= ((uint64_t)(tail[ 7])) << 56;
+ case 7: k1 ^= ((uint64_t)(tail[ 6])) << 48;
+ case 6: k1 ^= ((uint64_t)(tail[ 5])) << 40;
+ case 5: k1 ^= ((uint64_t)(tail[ 4])) << 32;
+ case 4: k1 ^= ((uint64_t)(tail[ 3])) << 24;
+ case 3: k1 ^= ((uint64_t)(tail[ 2])) << 16;
+ case 2: k1 ^= ((uint64_t)(tail[ 1])) << 8;
+ case 1: k1 ^= ((uint64_t)(tail[ 0])) << 0;
+ k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1;
+ }
+ }
+
+ /* finalization */
+ h1 ^= len; h2 ^= len;
+
+ h1 += h2;
+ h2 += h1;
+
+ h1 = hash_fmix_64(h1);
+ h2 = hash_fmix_64(h2);
+
+ h1 += h2;
+ h2 += h1;
+
+ r_out[0] = h1;
+ r_out[1] = h2;
+}
+
+
+/******************************************************************************/
+/* API. */
+JEMALLOC_INLINE void
+hash(const void *key, size_t len, const uint32_t seed, size_t r_hash[2])
+{
+#if (LG_SIZEOF_PTR == 3)
+ hash_x64_128(key, len, seed, (uint64_t *)r_hash);
+#else
+ uint64_t hashes[2];
+ hash_x86_128(key, len, seed, hashes);
+ r_hash[0] = (size_t)hashes[0];
+ r_hash[1] = (size_t)hashes[1];
+#endif
}
#endif
Modified: head/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal.h
==============================================================================
--- head/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal.h Thu Jan 24 02:25:43 2013 (r245867)
+++ head/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal.h Thu Jan 24 03:08:37 2013 (r245868)
@@ -226,6 +226,7 @@ static const bool config_ivsalloc =
#define ALLOCM_LG_ALIGN_MASK ((int)0x3f)
#define ZU(z) ((size_t)z)
+#define QU(q) ((uint64_t)q)
#ifndef __DECONST
# define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
@@ -233,10 +234,17 @@ static const bool config_ivsalloc =
#ifdef JEMALLOC_DEBUG
/* Disable inlining to make debugging easier. */
+# define JEMALLOC_ALWAYS_INLINE
# define JEMALLOC_INLINE
# define inline
#else
# define JEMALLOC_ENABLE_INLINE
+# ifdef JEMALLOC_HAVE_ATTR
+# define JEMALLOC_ALWAYS_INLINE \
+ static inline JEMALLOC_ATTR(unused) JEMALLOC_ATTR(always_inline)
+# else
+# define JEMALLOC_ALWAYS_INLINE static inline
+# endif
# define JEMALLOC_INLINE static inline
# ifdef _MSC_VER
# define inline _inline
@@ -359,7 +367,11 @@ static const bool config_ivsalloc =
# include <malloc.h>
# define alloca _alloca
# else
-# include <alloca.h>
+# ifdef JEMALLOC_HAS_ALLOCA_H
+# include <alloca.h>
+# else
+# include <stdlib.h>
+# endif
# endif
# define VARIABLE_ARRAY(type, name, count) \
type *name = alloca(sizeof(type) * count)
@@ -591,13 +603,14 @@ arena_t *choose_arena(arena_t *arena);
* for allocations.
*/
malloc_tsd_externs(arenas, arena_t *)
-malloc_tsd_funcs(JEMALLOC_INLINE, arenas, arena_t *, NULL, arenas_cleanup)
+malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, arenas, arena_t *, NULL,
+ arenas_cleanup)
/*
* Compute usable size that would result from allocating an object with the
* specified size.
*/
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
s2u(size_t size)
{
@@ -612,7 +625,7 @@ s2u(size_t size)
* Compute usable size that would result from allocating an object with the
* specified size and alignment.
*/
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
sa2u(size_t size, size_t alignment)
{
size_t usize;
@@ -757,7 +770,7 @@ malloc_tsd_protos(JEMALLOC_ATTR(unused),
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
imallocx(size_t size, bool try_tcache, arena_t *arena)
{
@@ -769,14 +782,14 @@ imallocx(size_t size, bool try_tcache, a
return (huge_malloc(size, false));
}
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
imalloc(size_t size)
{
return (imallocx(size, true, NULL));
}
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
icallocx(size_t size, bool try_tcache, arena_t *arena)
{
@@ -786,14 +799,14 @@ icallocx(size_t size, bool try_tcache, a
return (huge_malloc(size, true));
}
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
icalloc(size_t size)
{
return (icallocx(size, true, NULL));
}
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
ipallocx(size_t usize, size_t alignment, bool zero, bool try_tcache,
arena_t *arena)
{
@@ -818,7 +831,7 @@ ipallocx(size_t usize, size_t alignment,
return (ret);
}
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
ipalloc(size_t usize, size_t alignment, bool zero)
{
@@ -830,7 +843,7 @@ ipalloc(size_t usize, size_t alignment,
* void *ptr = [...]
* size_t sz = isalloc(ptr, config_prof);
*/
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
isalloc(const void *ptr, bool demote)
{
size_t ret;
@@ -849,7 +862,7 @@ isalloc(const void *ptr, bool demote)
return (ret);
}
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
ivsalloc(const void *ptr, bool demote)
{
@@ -882,7 +895,7 @@ p2rz(const void *ptr)
return (u2rz(usize));
}
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
idallocx(void *ptr, bool try_tcache)
{
arena_chunk_t *chunk;
@@ -896,14 +909,14 @@ idallocx(void *ptr, bool try_tcache)
huge_dalloc(ptr, true);
}
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
idalloc(void *ptr)
{
idallocx(ptr, true);
}
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
iqallocx(void *ptr, bool try_tcache)
{
@@ -913,14 +926,14 @@ iqallocx(void *ptr, bool try_tcache)
idallocx(ptr, try_tcache);
}
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
iqalloc(void *ptr)
{
iqallocx(ptr, true);
}
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
irallocx(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
bool no_move, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena)
{
@@ -989,7 +1002,7 @@ irallocx(void *ptr, size_t size, size_t
}
}
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
bool no_move)
{
@@ -999,7 +1012,7 @@ iralloc(void *ptr, size_t size, size_t e
}
malloc_tsd_externs(thread_allocated, thread_allocated_t)
-malloc_tsd_funcs(JEMALLOC_INLINE, thread_allocated, thread_allocated_t,
+malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, thread_allocated, thread_allocated_t,
THREAD_ALLOCATED_INITIALIZER, malloc_tsd_no_cleanup)
#endif
Modified: head/contrib/jemalloc/include/jemalloc/internal/private_namespace.h
==============================================================================
--- head/contrib/jemalloc/include/jemalloc/internal/private_namespace.h Thu Jan 24 02:25:43 2013 (r245867)
+++ head/contrib/jemalloc/include/jemalloc/internal/private_namespace.h Thu Jan 24 03:08:37 2013 (r245868)
@@ -41,6 +41,8 @@
#define arena_postfork_parent JEMALLOC_N(arena_postfork_parent)
#define arena_prefork JEMALLOC_N(arena_prefork)
#define arena_prof_accum JEMALLOC_N(arena_prof_accum)
+#define arena_prof_accum_impl JEMALLOC_N(arena_prof_accum_impl)
+#define arena_prof_accum_locked JEMALLOC_N(arena_prof_accum_locked)
#define arena_prof_ctx_get JEMALLOC_N(arena_prof_ctx_get)
#define arena_prof_ctx_set JEMALLOC_N(arena_prof_ctx_set)
#define arena_prof_promoted JEMALLOC_N(arena_prof_promoted)
@@ -63,6 +65,7 @@
#define arenas_tsd_boot JEMALLOC_N(arenas_tsd_boot)
#define arenas_tsd_cleanup_wrapper JEMALLOC_N(arenas_tsd_cleanup_wrapper)
#define arenas_tsd_get JEMALLOC_N(arenas_tsd_get)
+#define arenas_tsd_get_wrapper JEMALLOC_N(arenas_tsd_get_wrapper)
#define arenas_tsd_set JEMALLOC_N(arenas_tsd_set)
#define atomic_add_u JEMALLOC_N(atomic_add_u)
#define atomic_add_uint32 JEMALLOC_N(atomic_add_uint32)
@@ -174,6 +177,15 @@
#define extent_tree_szad_search JEMALLOC_N(extent_tree_szad_search)
#define get_errno JEMALLOC_N(get_errno)
#define hash JEMALLOC_N(hash)
+#define hash_fmix_32 JEMALLOC_N(hash_fmix_32)
+#define hash_fmix_64 JEMALLOC_N(hash_fmix_64)
+#define hash_get_block_32 JEMALLOC_N(hash_get_block_32)
+#define hash_get_block_64 JEMALLOC_N(hash_get_block_64)
+#define hash_rotl_32 JEMALLOC_N(hash_rotl_32)
+#define hash_rotl_64 JEMALLOC_N(hash_rotl_64)
+#define hash_x64_128 JEMALLOC_N(hash_x64_128)
+#define hash_x86_128 JEMALLOC_N(hash_x86_128)
+#define hash_x86_32 JEMALLOC_N(hash_x86_32)
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-head
mailing list