svn commit: r308689 - in head/libexec/rtld-elf: . aarch64 amd64 arm i386 mips powerpc powerpc64 riscv sparc64
Konstantin Belousov
kib at FreeBSD.org
Tue Nov 15 09:43:29 UTC 2016
Author: kib
Date: Tue Nov 15 09:43:26 2016
New Revision: 308689
URL: https://svnweb.freebsd.org/changeset/base/308689
Log:
Pass CPUID[1] %edx (cpu_feature), %ecx (cpu_feature2) and
CPUID[7].%ebx (cpu_stdext_feature), %ecx (cpu_stdext_feature2) to the
ifunc resolvers on x86.
It is much more clean to use CPUID instruction in usermode to retrieve
this information than to pass AT_HWCAP aux vector from kernel, on
x86. Still, the change does allow for use of AT_HWCAP on arches where it is
needed, by passing aux array to ifunc_init() initializer which should
prepare arguments for ifunc resolvers.
Current signature for resolvers on x86 is
func_t iresolve(uint32_t cpu_feature, uint32_t cpu_feature2,
uint32_t cpu_stdext_feature, uint32_t cpu_stdext_feature2);
where arguments have identical meaning as the kernel variables of the
same name. The ABIs allow to use resolvers with the void or shortened
list of arguments.
Reviewed by: jhb
Sponsored by: The FreeBSD Foundation
MFC after: 1 week
Differential revision: https://reviews.freebsd.org/D8448
Modified:
head/libexec/rtld-elf/aarch64/reloc.c
head/libexec/rtld-elf/aarch64/rtld_machdep.h
head/libexec/rtld-elf/amd64/reloc.c
head/libexec/rtld-elf/amd64/rtld_machdep.h
head/libexec/rtld-elf/arm/reloc.c
head/libexec/rtld-elf/arm/rtld_machdep.h
head/libexec/rtld-elf/i386/reloc.c
head/libexec/rtld-elf/i386/rtld_machdep.h
head/libexec/rtld-elf/mips/reloc.c
head/libexec/rtld-elf/mips/rtld_machdep.h
head/libexec/rtld-elf/powerpc/reloc.c
head/libexec/rtld-elf/powerpc/rtld_machdep.h
head/libexec/rtld-elf/powerpc64/reloc.c
head/libexec/rtld-elf/powerpc64/rtld_machdep.h
head/libexec/rtld-elf/riscv/reloc.c
head/libexec/rtld-elf/riscv/rtld_machdep.h
head/libexec/rtld-elf/rtld.c
head/libexec/rtld-elf/rtld.h
head/libexec/rtld-elf/sparc64/reloc.c
head/libexec/rtld-elf/sparc64/rtld_machdep.h
Modified: head/libexec/rtld-elf/aarch64/reloc.c
==============================================================================
--- head/libexec/rtld-elf/aarch64/reloc.c Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/aarch64/reloc.c Tue Nov 15 09:43:26 2016 (r308689)
@@ -299,6 +299,11 @@ reloc_jmpslot(Elf_Addr *where, Elf_Addr
return target;
}
+void
+ifunc_init(Elf_Auxinfo aux_info[static AT_COUNT] __unused)
+{
+}
+
/*
* Process non-PLT relocations
*/
Modified: head/libexec/rtld-elf/aarch64/rtld_machdep.h
==============================================================================
--- head/libexec/rtld-elf/aarch64/rtld_machdep.h Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/aarch64/rtld_machdep.h Tue Nov 15 09:43:26 2016 (r308689)
@@ -61,7 +61,10 @@ Elf_Addr reloc_jmpslot(Elf_Addr *where,
#define call_init_pointer(obj, target) \
(((InitArrFunc)(target))(main_argc, main_argv, environ))
-#define round(size, align) \
+#define call_ifunc_resolver(ptr) \
+ (((Elf_Addr (*)(void))ptr)())
+
+#define round(size, align) \
(((size) + (align) - 1) & ~((align) - 1))
#define calculate_first_tls_offset(size, align) \
round(16, align)
Modified: head/libexec/rtld-elf/amd64/reloc.c
==============================================================================
--- head/libexec/rtld-elf/amd64/reloc.c Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/amd64/reloc.c Tue Nov 15 09:43:26 2016 (r308689)
@@ -34,6 +34,7 @@
#include <sys/param.h>
#include <sys/mman.h>
#include <machine/sysarch.h>
+#include <machine/cpufunc.h>
#include <dlfcn.h>
#include <err.h>
@@ -406,7 +407,7 @@ reloc_iresolve(Obj_Entry *obj, RtldLockS
ptr = (Elf_Addr *)(obj->relocbase + rela->r_addend);
where = (Elf_Addr *)(obj->relocbase + rela->r_offset);
lock_release(rtld_bind_lock, lockstate);
- target = ((Elf_Addr (*)(void))ptr)();
+ target = call_ifunc_resolver(ptr);
wlock_acquire(rtld_bind_lock, lockstate);
*where = target;
break;
@@ -450,6 +451,25 @@ reloc_gnu_ifunc(Obj_Entry *obj, int flag
return (0);
}
+uint32_t cpu_feature, cpu_feature2, cpu_stdext_feature, cpu_stdext_feature2;
+
+void
+ifunc_init(Elf_Auxinfo aux_info[static AT_COUNT] __unused)
+{
+ u_int p[4], cpu_high;
+
+ do_cpuid(1, p);
+ cpu_feature = p[3];
+ cpu_feature2 = p[2];
+ do_cpuid(0, p);
+ cpu_high = p[0];
+ if (cpu_high >= 7) {
+ cpuid_count(7, 0, p);
+ cpu_stdext_feature = p[1];
+ cpu_stdext_feature2 = p[2];
+ }
+}
+
void
allocate_initial_tls(Obj_Entry *objs)
{
Modified: head/libexec/rtld-elf/amd64/rtld_machdep.h
==============================================================================
--- head/libexec/rtld-elf/amd64/rtld_machdep.h Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/amd64/rtld_machdep.h Tue Nov 15 09:43:26 2016 (r308689)
@@ -61,6 +61,14 @@ reloc_jmpslot(Elf_Addr *where, Elf_Addr
#define call_init_pointer(obj, target) \
(((InitArrFunc)(target))(main_argc, main_argv, environ))
+extern uint32_t cpu_feature;
+extern uint32_t cpu_feature2;
+extern uint32_t cpu_stdext_feature;
+extern uint32_t cpu_stdext_feature2;
+#define call_ifunc_resolver(ptr) \
+ (((Elf_Addr (*)(uint32_t, uint32_t, uint32_t, uint32_t))ptr)( \
+ cpu_feature, cpu_feature2, cpu_stdext_feature, cpu_stdext_feature2))
+
#define round(size, align) \
(((size) + (align) - 1) & ~((align) - 1))
#define calculate_first_tls_offset(size, align) \
Modified: head/libexec/rtld-elf/arm/reloc.c
==============================================================================
--- head/libexec/rtld-elf/arm/reloc.c Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/arm/reloc.c Tue Nov 15 09:43:26 2016 (r308689)
@@ -480,6 +480,11 @@ reloc_jmpslot(Elf_Addr *where, Elf_Addr
}
void
+ifunc_init(Elf_Auxinfo aux_info[static AT_COUNT] __unused)
+{
+}
+
+void
allocate_initial_tls(Obj_Entry *objs)
{
#ifdef ARM_TP_ADDRESS
Modified: head/libexec/rtld-elf/arm/rtld_machdep.h
==============================================================================
--- head/libexec/rtld-elf/arm/rtld_machdep.h Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/arm/rtld_machdep.h Tue Nov 15 09:43:26 2016 (r308689)
@@ -51,6 +51,9 @@ Elf_Addr reloc_jmpslot(Elf_Addr *where,
#define call_init_pointer(obj, target) \
(((InitArrFunc)(target))(main_argc, main_argv, environ))
+#define call_ifunc_resolver(ptr) \
+ (((Elf_Addr (*)(void))ptr)())
+
#define TLS_TCB_SIZE 8
typedef struct {
unsigned long ti_module;
Modified: head/libexec/rtld-elf/i386/reloc.c
==============================================================================
--- head/libexec/rtld-elf/i386/reloc.c Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/i386/reloc.c Tue Nov 15 09:43:26 2016 (r308689)
@@ -33,6 +33,7 @@
#include <sys/param.h>
#include <sys/mman.h>
+#include <machine/cpufunc.h>
#include <machine/segments.h>
#include <machine/sysarch.h>
@@ -359,7 +360,7 @@ reloc_iresolve(Obj_Entry *obj, RtldLockS
case R_386_IRELATIVE:
where = (Elf_Addr *)(obj->relocbase + rel->r_offset);
lock_release(rtld_bind_lock, lockstate);
- target = ((Elf_Addr (*)(void))(obj->relocbase + *where))();
+ target = call_ifunc_resolver(obj->relocbase + *where);
wlock_acquire(rtld_bind_lock, lockstate);
*where = target;
break;
@@ -404,6 +405,45 @@ reloc_gnu_ifunc(Obj_Entry *obj, int flag
return (0);
}
+uint32_t cpu_feature, cpu_feature2, cpu_stdext_feature, cpu_stdext_feature2;
+
+void
+ifunc_init(Elf_Auxinfo aux_info[static AT_COUNT] __unused)
+{
+ u_int p[4], cpu_high;
+ int cpuid_supported;
+
+ __asm __volatile(
+ " pushfl\n"
+ " popl %%eax\n"
+ " movl %%eax,%%ecx\n"
+ " xorl $0x200000,%%eax\n"
+ " pushl %%eax\n"
+ " popfl\n"
+ " pushfl\n"
+ " popl %%eax\n"
+ " xorl %%eax,%%ecx\n"
+ " je 1f\n"
+ " movl $1,%0\n"
+ " jmp 2f\n"
+ "1: movl $0,%0\n"
+ "2:\n"
+ : "=r" (cpuid_supported) : : "eax", "ecx");
+ if (!cpuid_supported)
+ return;
+
+ do_cpuid(1, p);
+ cpu_feature = p[3];
+ cpu_feature2 = p[2];
+ do_cpuid(0, p);
+ cpu_high = p[0];
+ if (cpu_high >= 7) {
+ cpuid_count(7, 0, p);
+ cpu_stdext_feature = p[1];
+ cpu_stdext_feature2 = p[2];
+ }
+}
+
void
allocate_initial_tls(Obj_Entry *objs)
{
Modified: head/libexec/rtld-elf/i386/rtld_machdep.h
==============================================================================
--- head/libexec/rtld-elf/i386/rtld_machdep.h Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/i386/rtld_machdep.h Tue Nov 15 09:43:26 2016 (r308689)
@@ -61,6 +61,14 @@ reloc_jmpslot(Elf_Addr *where, Elf_Addr
#define call_init_pointer(obj, target) \
(((InitArrFunc)(target))(main_argc, main_argv, environ))
+extern uint32_t cpu_feature;
+extern uint32_t cpu_feature2;
+extern uint32_t cpu_stdext_feature;
+extern uint32_t cpu_stdext_feature2;
+#define call_ifunc_resolver(ptr) \
+ (((Elf_Addr (*)(uint32_t, uint32_t, uint32_t, uint32_t))ptr)( \
+ cpu_feature, cpu_feature2, cpu_stdext_feature, cpu_stdext_feature2))
+
#define round(size, align) \
(((size) + (align) - 1) & ~((align) - 1))
#define calculate_first_tls_offset(size, align) \
Modified: head/libexec/rtld-elf/mips/reloc.c
==============================================================================
--- head/libexec/rtld-elf/mips/reloc.c Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/mips/reloc.c Tue Nov 15 09:43:26 2016 (r308689)
@@ -618,6 +618,11 @@ reloc_jmpslot(Elf_Addr *where, Elf_Addr
}
void
+ifunc_init(Elf_Auxinfo aux_info[static AT_COUNT] __unused)
+{
+}
+
+void
allocate_initial_tls(Obj_Entry *objs)
{
char *tls;
Modified: head/libexec/rtld-elf/mips/rtld_machdep.h
==============================================================================
--- head/libexec/rtld-elf/mips/rtld_machdep.h Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/mips/rtld_machdep.h Tue Nov 15 09:43:26 2016 (r308689)
@@ -52,6 +52,9 @@ Elf_Addr reloc_jmpslot(Elf_Addr *where,
#define call_init_pointer(obj, target) \
(((InitArrFunc)(target))(main_argc, main_argv, environ))
+#define call_ifunc_resolver(ptr) \
+ (((Elf_Addr (*)(void))ptr)())
+
typedef struct {
unsigned long ti_module;
unsigned long ti_offset;
Modified: head/libexec/rtld-elf/powerpc/reloc.c
==============================================================================
--- head/libexec/rtld-elf/powerpc/reloc.c Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/powerpc/reloc.c Tue Nov 15 09:43:26 2016 (r308689)
@@ -620,6 +620,11 @@ init_pltgot(Obj_Entry *obj)
}
void
+ifunc_init(Elf_Auxinfo aux_info[static AT_COUNT] __unused)
+{
+}
+
+void
allocate_initial_tls(Obj_Entry *list)
{
Elf_Addr **tp;
Modified: head/libexec/rtld-elf/powerpc/rtld_machdep.h
==============================================================================
--- head/libexec/rtld-elf/powerpc/rtld_machdep.h Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/powerpc/rtld_machdep.h Tue Nov 15 09:43:26 2016 (r308689)
@@ -51,6 +51,9 @@ Elf_Addr reloc_jmpslot(Elf_Addr *where,
#define call_init_pointer(obj, target) \
(((InitArrFunc)(target))(main_argc, main_argv, environ))
+#define call_ifunc_resolver(ptr) \
+ (((Elf_Addr (*)(void))ptr)())
+
/*
* Lazy binding entry point, called via PLT.
*/
Modified: head/libexec/rtld-elf/powerpc64/reloc.c
==============================================================================
--- head/libexec/rtld-elf/powerpc64/reloc.c Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/powerpc64/reloc.c Tue Nov 15 09:43:26 2016 (r308689)
@@ -524,6 +524,11 @@ init_pltgot(Obj_Entry *obj)
}
void
+ifunc_init(Elf_Auxinfo aux_info[static AT_COUNT] __unused)
+{
+}
+
+void
allocate_initial_tls(Obj_Entry *list)
{
Elf_Addr **tp;
Modified: head/libexec/rtld-elf/powerpc64/rtld_machdep.h
==============================================================================
--- head/libexec/rtld-elf/powerpc64/rtld_machdep.h Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/powerpc64/rtld_machdep.h Tue Nov 15 09:43:26 2016 (r308689)
@@ -51,6 +51,9 @@ Elf_Addr reloc_jmpslot(Elf_Addr *where,
#define call_init_pointer(obj, target) \
(((InitArrFunc)(target))(main_argc, main_argv, environ))
+#define call_ifunc_resolver(ptr) \
+ (((Elf_Addr (*)(void))ptr)())
+
/*
* Lazy binding entry point, called via PLT.
*/
Modified: head/libexec/rtld-elf/riscv/reloc.c
==============================================================================
--- head/libexec/rtld-elf/riscv/reloc.c Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/riscv/reloc.c Tue Nov 15 09:43:26 2016 (r308689)
@@ -367,6 +367,11 @@ reloc_non_plt(Obj_Entry *obj, Obj_Entry
}
void
+ifunc_init(Elf_Auxinfo aux_info[static AT_COUNT] __unused)
+{
+}
+
+void
allocate_initial_tls(Obj_Entry *objs)
{
Elf_Addr **tp;
Modified: head/libexec/rtld-elf/riscv/rtld_machdep.h
==============================================================================
--- head/libexec/rtld-elf/riscv/rtld_machdep.h Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/riscv/rtld_machdep.h Tue Nov 15 09:43:26 2016 (r308689)
@@ -78,6 +78,9 @@ Elf_Addr reloc_jmpslot(Elf_Addr *where,
__asm __volatile("mv gp, %0" :: "r"(old1)); \
})
+#define call_ifunc_resolver(ptr) \
+ (((Elf_Addr (*)(void))ptr)())
+
/*
* Lazy binding entry point, called via PLT.
*/
Modified: head/libexec/rtld-elf/rtld.c
==============================================================================
--- head/libexec/rtld-elf/rtld.c Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/rtld.c Tue Nov 15 09:43:26 2016 (r308689)
@@ -642,6 +642,7 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_
r_debug_state(NULL, &obj_main->linkmap); /* say hello to gdb! */
map_stacks_exec(NULL);
+ ifunc_init(aux);
dbg("resolving ifuncs");
if (resolve_objects_ifunc(obj_main,
@@ -690,7 +691,7 @@ rtld_resolve_ifunc(const Obj_Entry *obj,
Elf_Addr target;
ptr = (void *)make_function_pointer(def, obj);
- target = ((Elf_Addr (*)(void))ptr)();
+ target = call_ifunc_resolver(ptr);
return ((void *)target);
}
Modified: head/libexec/rtld-elf/rtld.h
==============================================================================
--- head/libexec/rtld-elf/rtld.h Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/rtld.h Tue Nov 15 09:43:26 2016 (r308689)
@@ -367,6 +367,7 @@ void dump_Elf_Rela(Obj_Entry *, const El
unsigned long elf_hash(const char *);
const Elf_Sym *find_symdef(unsigned long, const Obj_Entry *,
const Obj_Entry **, int, SymCache *, struct Struct_RtldLockState *);
+void ifunc_init(Elf_Auxinfo[static AT_COUNT]);
void init_pltgot(Obj_Entry *);
void lockdflt_init(void);
void digest_notes(Obj_Entry *, Elf_Addr, Elf_Addr);
Modified: head/libexec/rtld-elf/sparc64/reloc.c
==============================================================================
--- head/libexec/rtld-elf/sparc64/reloc.c Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/sparc64/reloc.c Tue Nov 15 09:43:26 2016 (r308689)
@@ -786,6 +786,11 @@ reloc_jmpslot(Elf_Addr *wherep, Elf_Addr
return (target);
}
+void
+ifunc_init(Elf_Auxinfo aux_info[static AT_COUNT] __unused)
+{
+}
+
/*
* Install rtld function call into this PLT slot.
*/
Modified: head/libexec/rtld-elf/sparc64/rtld_machdep.h
==============================================================================
--- head/libexec/rtld-elf/sparc64/rtld_machdep.h Tue Nov 15 09:40:03 2016 (r308688)
+++ head/libexec/rtld-elf/sparc64/rtld_machdep.h Tue Nov 15 09:43:26 2016 (r308689)
@@ -53,7 +53,10 @@ Elf_Addr reloc_jmpslot(Elf_Addr *, Elf_A
#define call_init_pointer(obj, target) \
(((InitArrFunc)(target))(main_argc, main_argv, environ))
-#define round(size, align) \
+#define call_ifunc_resolver(ptr) \
+ (((Elf_Addr (*)(void))ptr)())
+
+#define round(size, align) \
(((size) + (align) - 1) & ~((align) - 1))
#define calculate_first_tls_offset(size, align) \
round(size, align)
More information about the svn-src-head
mailing list