svn commit: r338973 - head/lib/libc/amd64/string
Mateusz Guzik
mjg at FreeBSD.org
Thu Sep 27 17:08:31 UTC 2018
Author: mjg
Date: Thu Sep 27 17:08:29 2018
New Revision: 338973
URL: https://svnweb.freebsd.org/changeset/base/338973
Log:
amd64: reimplement libc memcmp and bcmp with kernel memcmp
Both are significantly slower than hand-coded loops. See r338963 for
kernel commit.
bcmp differs from memcmp by always returning 1 when a difference is
found, as opposed to going for a value bigger or lower than 0
depending on what it is. This means it can do less work. For now the
code is duplicated and modified. This will get deduplicated after
another round of optimization when memcmp will get a longer-term form.
Both tested with the glibc suite. While the suite does not have a test
for bcmp, I created a wrapper routine which verified that values match
(0 vs 0, 1 vs non-zero).
Reviewed by: kib
Approved by: re (gjb)
Sponsored by: The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D17336
Modified:
head/lib/libc/amd64/string/bcmp.S
head/lib/libc/amd64/string/memcmp.S
Modified: head/lib/libc/amd64/string/bcmp.S
==============================================================================
--- head/lib/libc/amd64/string/bcmp.S Thu Sep 27 16:43:41 2018 (r338972)
+++ head/lib/libc/amd64/string/bcmp.S Thu Sep 27 17:08:29 2018 (r338973)
@@ -1,27 +1,121 @@
+/*-
+ * Copyright (c) 2018 The FreeBSD Foundation
+ *
+ * This software was developed by Mateusz Guzik <mjg at FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
#include <machine/asm.h>
__FBSDID("$FreeBSD$");
-#if 0
- RCSID("$NetBSD: bcmp.S,v 1.1 2001/06/19 00:25:04 fvdl Exp $")
-#endif
-
ENTRY(bcmp)
- cld /* set compare direction forward */
+ cmpq $16,%rdx
+ jae 5f
+1:
+ testq %rdx,%rdx
+ je 3f
+ xorl %ecx,%ecx
+2:
+ movzbl (%rdi,%rcx,1),%eax
+ movzbl (%rsi,%rcx,1),%r8d
+ cmpb %r8b,%al
+ jne 4f
+ addq $1,%rcx
+ cmpq %rcx,%rdx
+ jz 3f
+ movzbl (%rdi,%rcx,1),%eax
+ movzbl (%rsi,%rcx,1),%r8d
+ cmpb %r8b,%al
+ jne 4f
+ addq $1,%rcx
+ cmpq %rcx,%rdx
+ jz 3f
+ movzbl (%rdi,%rcx,1),%eax
+ movzbl (%rsi,%rcx,1),%r8d
+ cmpb %r8b,%al
+ jne 4f
+ addq $1,%rcx
+ cmpq %rcx,%rdx
+ jz 3f
+ movzbl (%rdi,%rcx,1),%eax
+ movzbl (%rsi,%rcx,1),%r8d
+ cmpb %r8b,%al
+ jne 4f
+ addq $1,%rcx
+ cmpq %rcx,%rdx
+ jne 2b
+3:
+ xorl %eax,%eax
+ ret
+4:
+ movl $1,%eax
+ ret
+5:
+ cmpq $32,%rdx
+ jae 7f
+6:
+ /*
+ * 8 bytes
+ */
+ movq (%rdi),%r8
+ movq (%rsi),%r9
+ cmpq %r8,%r9
+ jne 4b
+ leaq 8(%rdi),%rdi
+ leaq 8(%rsi),%rsi
+ subq $8,%rdx
+ cmpq $8,%rdx
+ jae 6b
+ jl 1b
+ jmp 3b
+7:
+ /*
+ * 32 bytes
+ */
+ movq (%rsi),%r8
+ movq 8(%rsi),%r9
+ subq (%rdi),%r8
+ subq 8(%rdi),%r9
+ or %r8,%r9
+ jnz 4b
- movq %rdx,%rcx /* compare by words */
- shrq $3,%rcx
- repe
- cmpsq
- jne L1
+ movq 16(%rsi),%r8
+ movq 24(%rsi),%r9
+ subq 16(%rdi),%r8
+ subq 24(%rdi),%r9
+ or %r8,%r9
+ jnz 4b
- movq %rdx,%rcx /* compare remainder by bytes */
- andq $7,%rcx
- repe
- cmpsb
-L1:
- setne %al
- movsbl %al,%eax
- ret
+ leaq 32(%rdi),%rdi
+ leaq 32(%rsi),%rsi
+ subq $32,%rdx
+ cmpq $32,%rdx
+ jae 7b
+ jnz 1b
+ jmp 3b
END(bcmp)
.section .note.GNU-stack,"",%progbits
Modified: head/lib/libc/amd64/string/memcmp.S
==============================================================================
--- head/lib/libc/amd64/string/memcmp.S Thu Sep 27 16:43:41 2018 (r338972)
+++ head/lib/libc/amd64/string/memcmp.S Thu Sep 27 17:08:29 2018 (r338973)
@@ -1,44 +1,121 @@
-/*
- * Written by J.T. Conklin <jtc at NetBSD.org>.
- * Public domain.
- * Adapted for NetBSD/x86_64 by Frank van der Linden <fvdl at wasabisystems.com>
+/*-
+ * Copyright (c) 2018 The FreeBSD Foundation
+ *
+ * This software was developed by Mateusz Guzik <mjg at FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
*/
#include <machine/asm.h>
__FBSDID("$FreeBSD$");
-#if 0
- RCSID("$NetBSD: memcmp.S,v 1.2 2003/07/26 19:24:39 salo Exp $")
-#endif
-
ENTRY(memcmp)
- cld /* set compare direction forward */
- movq %rdx,%rcx /* compare by longs */
- shrq $3,%rcx
- repe
- cmpsq
- jne L5 /* do we match so far? */
-
- movq %rdx,%rcx /* compare remainder by bytes */
- andq $7,%rcx
- repe
- cmpsb
- jne L6 /* do we match? */
-
- xorl %eax,%eax /* we match, return zero */
+ cmpq $16,%rdx
+ jae 5f
+1:
+ testq %rdx,%rdx
+ je 3f
+ xorl %ecx,%ecx
+2:
+ movzbl (%rdi,%rcx,1),%eax
+ movzbl (%rsi,%rcx,1),%r8d
+ cmpb %r8b,%al
+ jne 4f
+ addq $1,%rcx
+ cmpq %rcx,%rdx
+ jz 3f
+ movzbl (%rdi,%rcx,1),%eax
+ movzbl (%rsi,%rcx,1),%r8d
+ cmpb %r8b,%al
+ jne 4f
+ addq $1,%rcx
+ cmpq %rcx,%rdx
+ jz 3f
+ movzbl (%rdi,%rcx,1),%eax
+ movzbl (%rsi,%rcx,1),%r8d
+ cmpb %r8b,%al
+ jne 4f
+ addq $1,%rcx
+ cmpq %rcx,%rdx
+ jz 3f
+ movzbl (%rdi,%rcx,1),%eax
+ movzbl (%rsi,%rcx,1),%r8d
+ cmpb %r8b,%al
+ jne 4f
+ addq $1,%rcx
+ cmpq %rcx,%rdx
+ jne 2b
+3:
+ xorl %eax,%eax
ret
-
-L5: movl $8,%ecx /* We know that one of the next */
- subq %rcx,%rdi /* eight pairs of bytes do not */
- subq %rcx,%rsi /* match. */
- repe
- cmpsb
-L6: xorl %eax,%eax /* Perform unsigned comparison */
- movb -1(%rdi),%al
- xorl %edx,%edx
- movb -1(%rsi),%dl
- subl %edx,%eax
+4:
+ subl %r8d,%eax
ret
+5:
+ cmpq $32,%rdx
+ jae 7f
+6:
+ /*
+ * 8 bytes
+ */
+ movq (%rdi),%r8
+ movq (%rsi),%r9
+ cmpq %r8,%r9
+ jne 1b
+ leaq 8(%rdi),%rdi
+ leaq 8(%rsi),%rsi
+ subq $8,%rdx
+ cmpq $8,%rdx
+ jae 6b
+ jl 1b
+ jmp 3b
+7:
+ /*
+ * 32 bytes
+ */
+ movq (%rsi),%r8
+ movq 8(%rsi),%r9
+ subq (%rdi),%r8
+ subq 8(%rdi),%r9
+ or %r8,%r9
+ jnz 1b
+
+ movq 16(%rsi),%r8
+ movq 24(%rsi),%r9
+ subq 16(%rdi),%r8
+ subq 24(%rdi),%r9
+ or %r8,%r9
+ jnz 1b
+
+ leaq 32(%rdi),%rdi
+ leaq 32(%rsi),%rsi
+ subq $32,%rdx
+ cmpq $32,%rdx
+ jae 7b
+ jnz 1b
+ jmp 3b
END(memcmp)
.section .note.GNU-stack,"",%progbits
More information about the svn-src-head
mailing list