svn commit: r339349 - in head/sys/amd64: amd64 include
Mateusz Guzik
mjg at FreeBSD.org
Sat Oct 13 21:18:33 UTC 2018
Author: mjg
Date: Sat Oct 13 21:18:31 2018
New Revision: 339349
URL: https://svnweb.freebsd.org/changeset/base/339349
Log:
amd64: partially depessimize cpu_fetch_syscall_args and cpu_set_syscall_retval
Vast majority of syscalls take 6 or less arguments. Move handling of other
cases to a fallback function. Similarly, special casing for _syscall
and __syscall
magic syscalls is moved away.
Return is almost always 0. The change replaces 3 branches with 1 in the common
case. Also the 'frame' variable convinces clang not to reload it on each access.
Reviewed by: kib
Approved by: re (gjb)
Sponsored by: The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D17542
Modified:
head/sys/amd64/amd64/trap.c
head/sys/amd64/amd64/vm_machdep.c
head/sys/amd64/include/proc.h
Modified: head/sys/amd64/amd64/trap.c
==============================================================================
--- head/sys/amd64/amd64/trap.c Sat Oct 13 21:17:28 2018 (r339348)
+++ head/sys/amd64/amd64/trap.c Sat Oct 13 21:18:31 2018 (r339349)
@@ -970,21 +970,19 @@ dblfault_handler(struct trapframe *frame)
panic("double fault");
}
-int
-cpu_fetch_syscall_args(struct thread *td)
+static int __noinline
+cpu_fetch_syscall_args_fallback(struct thread *td, struct syscall_args *sa)
{
struct proc *p;
struct trapframe *frame;
register_t *argp;
- struct syscall_args *sa;
caddr_t params;
int reg, regcnt, error;
p = td->td_proc;
frame = td->td_frame;
- sa = &td->td_sa;
reg = 0;
- regcnt = 6;
+ regcnt = NARGREGS;
sa->code = frame->tf_rax;
@@ -1002,24 +1000,58 @@ cpu_fetch_syscall_args(struct thread *td)
sa->callp = &p->p_sysent->sv_table[sa->code];
sa->narg = sa->callp->sy_narg;
- KASSERT(sa->narg <= sizeof(sa->args) / sizeof(sa->args[0]),
- ("Too many syscall arguments!"));
- error = 0;
+ KASSERT(sa->narg <= nitems(sa->args), ("Too many syscall arguments!"));
argp = &frame->tf_rdi;
argp += reg;
- memcpy(sa->args, argp, sizeof(sa->args[0]) * 6);
+ memcpy(sa->args, argp, sizeof(sa->args[0]) * NARGREGS);
if (sa->narg > regcnt) {
params = (caddr_t)frame->tf_rsp + sizeof(register_t);
error = copyin(params, &sa->args[regcnt],
(sa->narg - regcnt) * sizeof(sa->args[0]));
+ if (__predict_false(error != 0))
+ return (error);
}
- if (error == 0) {
- td->td_retval[0] = 0;
- td->td_retval[1] = frame->tf_rdx;
- }
+ td->td_retval[0] = 0;
+ td->td_retval[1] = frame->tf_rdx;
- return (error);
+ return (0);
+}
+
+int
+cpu_fetch_syscall_args(struct thread *td)
+{
+ struct proc *p;
+ struct trapframe *frame;
+ struct syscall_args *sa;
+
+ p = td->td_proc;
+ frame = td->td_frame;
+ sa = &td->td_sa;
+
+ sa->code = frame->tf_rax;
+
+ if (__predict_false(sa->code == SYS_syscall ||
+ sa->code == SYS___syscall ||
+ sa->code >= p->p_sysent->sv_size))
+ return (cpu_fetch_syscall_args_fallback(td, sa));
+
+ sa->callp = &p->p_sysent->sv_table[sa->code];
+ sa->narg = sa->callp->sy_narg;
+ KASSERT(sa->narg <= nitems(sa->args), ("Too many syscall arguments!"));
+
+ if (p->p_sysent->sv_mask)
+ sa->code &= p->p_sysent->sv_mask;
+
+ if (__predict_false(sa->narg > NARGREGS))
+ return (cpu_fetch_syscall_args_fallback(td, sa));
+
+ memcpy(sa->args, &frame->tf_rdi, sizeof(sa->args[0]) * NARGREGS);
+
+ td->td_retval[0] = 0;
+ td->td_retval[1] = frame->tf_rdx;
+
+ return (0);
}
#include "../../kern/subr_syscall.c"
Modified: head/sys/amd64/amd64/vm_machdep.c
==============================================================================
--- head/sys/amd64/amd64/vm_machdep.c Sat Oct 13 21:17:28 2018 (r339348)
+++ head/sys/amd64/amd64/vm_machdep.c Sat Oct 13 21:18:31 2018 (r339349)
@@ -372,14 +372,17 @@ cpu_thread_free(struct thread *td)
void
cpu_set_syscall_retval(struct thread *td, int error)
{
+ struct trapframe *frame;
- switch (error) {
- case 0:
- td->td_frame->tf_rax = td->td_retval[0];
- td->td_frame->tf_rdx = td->td_retval[1];
- td->td_frame->tf_rflags &= ~PSL_C;
- break;
+ frame = td->td_frame;
+ if (__predict_true(error == 0)) {
+ frame->tf_rax = td->td_retval[0];
+ frame->tf_rdx = td->td_retval[1];
+ frame->tf_rflags &= ~PSL_C;
+ return;
+ }
+ switch (error) {
case ERESTART:
/*
* Reconstruct pc, we know that 'syscall' is 2 bytes,
@@ -393,8 +396,8 @@ cpu_set_syscall_retval(struct thread *td, int error)
* Require full context restore to get the arguments
* in the registers reloaded at return to usermode.
*/
- td->td_frame->tf_rip -= td->td_frame->tf_err;
- td->td_frame->tf_r10 = td->td_frame->tf_rcx;
+ frame->tf_rip -= frame->tf_err;
+ frame->tf_r10 = frame->tf_rcx;
set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
break;
@@ -402,8 +405,8 @@ cpu_set_syscall_retval(struct thread *td, int error)
break;
default:
- td->td_frame->tf_rax = SV_ABI_ERRNO(td->td_proc, error);
- td->td_frame->tf_rflags |= PSL_C;
+ frame->tf_rax = SV_ABI_ERRNO(td->td_proc, error);
+ frame->tf_rflags |= PSL_C;
break;
}
}
Modified: head/sys/amd64/include/proc.h
==============================================================================
--- head/sys/amd64/include/proc.h Sat Oct 13 21:17:28 2018 (r339348)
+++ head/sys/amd64/include/proc.h Sat Oct 13 21:18:31 2018 (r339349)
@@ -101,6 +101,9 @@ int amd64_set_ldt_data(struct thread *td, int start, i
extern struct mtx dt_lock;
extern int max_ldt_segment;
+
+#define NARGREGS 6
+
#endif /* _KERNEL */
#endif /* !_MACHINE_PROC_H_ */
More information about the svn-src-all
mailing list