svn commit: r220430 - head/sys/amd64/amd64
John Baldwin
jhb at FreeBSD.org
Thu Apr 7 21:32:25 UTC 2011
Author: jhb
Date: Thu Apr 7 21:32:25 2011
New Revision: 220430
URL: http://svn.freebsd.org/changeset/base/220430
Log:
If a system call does not request a full interrupt return, use a fast
path via the sysretq instruction to return from the system call. This was
removed in 190620 and not quite fully restored in 195486. This resolves
most of the performance regression in system call microbenchmarks between
7 and 8 on amd64.
Reviewed by: kib
MFC after: 1 week
Modified:
head/sys/amd64/amd64/exception.S
Modified: head/sys/amd64/amd64/exception.S
==============================================================================
--- head/sys/amd64/amd64/exception.S Thu Apr 7 21:29:34 2011 (r220429)
+++ head/sys/amd64/amd64/exception.S Thu Apr 7 21:32:25 2011 (r220430)
@@ -339,6 +339,9 @@ IDTVEC(prot)
* and the new privilige level. We are still running on the old user stack
* pointer. We have to juggle a few things around to find our stack etc.
* swapgs gives us access to our PCPU space only.
+ *
+ * We do not support invoking this from a custom %cs or %ss (e.g. using
+ * entries from an LDT).
*/
IDTVEC(fast_syscall)
swapgs
@@ -380,6 +383,36 @@ IDTVEC(fast_syscall)
movq %rsp,%rdi
call syscall
movq PCPU(CURPCB),%rax
+ testq $PCB_FULL_IRET,PCB_FLAGS(%rax)
+ jne 3f
+1: /* Check for and handle AST's on return to userland. */
+ cli
+ movq PCPU(CURTHREAD),%rax
+ testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax)
+ je 2f
+ sti
+ movq %rsp, %rdi
+ call ast
+ jmp 1b
+2: /* Restore preserved registers. */
+ MEXITCOUNT
+ movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */
+ movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */
+ movq TF_RDX(%rsp),%rdx /* return value 2 */
+ movq TF_RAX(%rsp),%rax /* return value 1 */
+ movq TF_RBX(%rsp),%rbx /* C preserved */
+ movq TF_RBP(%rsp),%rbp /* C preserved */
+ movq TF_R12(%rsp),%r12 /* C preserved */
+ movq TF_R13(%rsp),%r13 /* C preserved */
+ movq TF_R14(%rsp),%r14 /* C preserved */
+ movq TF_R15(%rsp),%r15 /* C preserved */
+ movq TF_RFLAGS(%rsp),%r11 /* original %rflags */
+ movq TF_RIP(%rsp),%rcx /* original %rip */
+ movq TF_RSP(%rsp),%r9 /* user stack pointer */
+ movq %r9,%rsp /* original %rsp */
+ swapgs
+ sysretq
+3: /* Requested full context restore, use doreti for that. */
MEXITCOUNT
jmp doreti
More information about the svn-src-all
mailing list