svn commit: r199603 - in head/sys: amd64/amd64 i386/i386 net
Jung-uk Kim
jkim at FreeBSD.org
Fri Nov 20 18:49:21 UTC 2009
Author: jkim
Date: Fri Nov 20 18:49:20 2009
New Revision: 199603
URL: http://svn.freebsd.org/changeset/base/199603
Log:
- Allocate scratch memory on stack instead of pre-allocating it with
the filter as we do from bpf_filter()[1].
- Revert experimental use of contigmalloc(9)/contigfree(9). It has no
performance benefit over malloc(9)/free(9)[2].
Requested by: rwatson[1]
Pointed out by: rwatson, jhb, alc[2]
Modified:
head/sys/amd64/amd64/bpf_jit_machdep.c
head/sys/amd64/amd64/bpf_jit_machdep.h
head/sys/i386/i386/bpf_jit_machdep.c
head/sys/i386/i386/bpf_jit_machdep.h
head/sys/net/bpf_jitter.c
head/sys/net/bpf_jitter.h
Modified: head/sys/amd64/amd64/bpf_jit_machdep.c
==============================================================================
--- head/sys/amd64/amd64/bpf_jit_machdep.c Fri Nov 20 17:59:50 2009 (r199602)
+++ head/sys/amd64/amd64/bpf_jit_machdep.c Fri Nov 20 18:49:20 2009 (r199603)
@@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$");
#include <amd64/amd64/bpf_jit_machdep.h>
-bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *);
+bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *);
/*
* emit routine to update the jump table
@@ -97,7 +97,7 @@ emit_code(bpf_bin_stream *stream, u_int
* Function that does the real stuff
*/
bpf_filter_func
-bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
+bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size)
{
bpf_bin_stream stream;
struct bpf_insn *ins;
@@ -111,10 +111,9 @@ bpf_jit_compile(struct bpf_insn *prog, u
/* Allocate the reference table for the jumps */
#ifdef _KERNEL
- stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int),
- M_BPFJIT, M_NOWAIT);
+ stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, M_NOWAIT);
#else
- stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int));
+ stream.refs = malloc((nins + 1) * sizeof(u_int));
#endif
if (stream.refs == NULL)
return (NULL);
@@ -137,8 +136,10 @@ bpf_jit_compile(struct bpf_insn *prog, u
ins = prog;
/* create the procedure header */
- MOVrq2(RBX, R8);
- MOVrq(RDI, RBX);
+ PUSH(RBP);
+ MOVrq(RSP, RBP);
+ SUBib(BPF_MEMWORDS * sizeof(uint32_t), RSP);
+ MOVrq2(RDI, R8);
MOVrd2(ESI, R9D);
MOVrd(EDX, EDI);
@@ -155,13 +156,11 @@ bpf_jit_compile(struct bpf_insn *prog, u
case BPF_RET|BPF_K:
MOVid(ins->k, EAX);
- MOVrq3(R8, RBX);
- RET();
+ LEAVE_RET();
break;
case BPF_RET|BPF_A:
- MOVrq3(R8, RBX);
- RET();
+ LEAVE_RET();
break;
case BPF_LD|BPF_W|BPF_ABS:
@@ -171,11 +170,11 @@ bpf_jit_compile(struct bpf_insn *prog, u
MOVrd(EDI, ECX);
SUBrd(ESI, ECX);
CMPid(sizeof(int32_t), ECX);
- JAEb(6);
+ JAEb(4);
ZEROrd(EAX);
- MOVrq3(R8, RBX);
- RET();
- MOVobd(RBX, RSI, EAX);
+ LEAVE_RET();
+ MOVrq3(R8, RCX);
+ MOVobd(RCX, RSI, EAX);
BSWAP(EAX);
break;
@@ -187,10 +186,10 @@ bpf_jit_compile(struct bpf_insn *prog, u
MOVrd(EDI, ECX);
SUBrd(ESI, ECX);
CMPid(sizeof(int16_t), ECX);
- JAEb(4);
- MOVrq3(R8, RBX);
- RET();
- MOVobw(RBX, RSI, AX);
+ JAEb(2);
+ LEAVE_RET();
+ MOVrq3(R8, RCX);
+ MOVobw(RCX, RSI, AX);
SWAP_AX();
break;
@@ -198,10 +197,10 @@ bpf_jit_compile(struct bpf_insn *prog, u
ZEROrd(EAX);
MOVid(ins->k, ESI);
CMPrd(EDI, ESI);
- JBb(4);
- MOVrq3(R8, RBX);
- RET();
- MOVobb(RBX, RSI, AL);
+ JBb(2);
+ LEAVE_RET();
+ MOVrq3(R8, RCX);
+ MOVobb(RCX, RSI, AL);
break;
case BPF_LD|BPF_W|BPF_LEN:
@@ -224,11 +223,11 @@ bpf_jit_compile(struct bpf_insn *prog, u
MOVrd(EDI, ECX);
SUBrd(ESI, ECX);
CMPid(sizeof(int32_t), ECX);
- JAEb(6);
+ JAEb(4);
ZEROrd(EAX);
- MOVrq3(R8, RBX);
- RET();
- MOVobd(RBX, RSI, EAX);
+ LEAVE_RET();
+ MOVrq3(R8, RCX);
+ MOVobd(RCX, RSI, EAX);
BSWAP(EAX);
break;
@@ -245,10 +244,10 @@ bpf_jit_compile(struct bpf_insn *prog, u
MOVrd(EDI, ECX);
SUBrd(ESI, ECX);
CMPid(sizeof(int16_t), ECX);
- JAEb(4);
- MOVrq3(R8, RBX);
- RET();
- MOVobw(RBX, RSI, AX);
+ JAEb(2);
+ LEAVE_RET();
+ MOVrq3(R8, RCX);
+ MOVobw(RCX, RSI, AX);
SWAP_AX();
break;
@@ -260,22 +259,22 @@ bpf_jit_compile(struct bpf_insn *prog, u
MOVrd(EDI, ECX);
SUBrd(EDX, ECX);
CMPrd(ESI, ECX);
- JAb(4);
- MOVrq3(R8, RBX);
- RET();
+ JAb(2);
+ LEAVE_RET();
+ MOVrq3(R8, RCX);
ADDrd(EDX, ESI);
- MOVobb(RBX, RSI, AL);
+ MOVobb(RCX, RSI, AL);
break;
case BPF_LDX|BPF_MSH|BPF_B:
MOVid(ins->k, ESI);
CMPrd(EDI, ESI);
- JBb(6);
+ JBb(4);
ZEROrd(EAX);
- MOVrq3(R8, RBX);
- RET();
+ LEAVE_RET();
ZEROrd(EDX);
- MOVobb(RBX, RSI, DL);
+ MOVrq3(R8, RCX);
+ MOVobb(RCX, RSI, DL);
ANDib(0x0f, DL);
SHLib(2, EDX);
break;
@@ -289,15 +288,13 @@ bpf_jit_compile(struct bpf_insn *prog, u
break;
case BPF_LD|BPF_MEM:
- MOViq((uintptr_t)mem, RCX);
- MOVid(ins->k * 4, ESI);
- MOVobd(RCX, RSI, EAX);
+ MOVid(ins->k * sizeof(uint32_t), ESI);
+ MOVobd(RSP, RSI, EAX);
break;
case BPF_LDX|BPF_MEM:
- MOViq((uintptr_t)mem, RCX);
- MOVid(ins->k * 4, ESI);
- MOVobd(RCX, RSI, EDX);
+ MOVid(ins->k * sizeof(uint32_t), ESI);
+ MOVobd(RSP, RSI, EDX);
break;
case BPF_ST:
@@ -306,15 +303,13 @@ bpf_jit_compile(struct bpf_insn *prog, u
* be optimized if the previous instruction
* was already of this type
*/
- MOViq((uintptr_t)mem, RCX);
- MOVid(ins->k * 4, ESI);
- MOVomd(EAX, RCX, RSI);
+ MOVid(ins->k * sizeof(uint32_t), ESI);
+ MOVomd(EAX, RSP, RSI);
break;
case BPF_STX:
- MOViq((uintptr_t)mem, RCX);
- MOVid(ins->k * 4, ESI);
- MOVomd(EDX, RCX, RSI);
+ MOVid(ins->k * sizeof(uint32_t), ESI);
+ MOVomd(EDX, RSP, RSI);
break;
case BPF_JMP|BPF_JA:
@@ -394,10 +389,9 @@ bpf_jit_compile(struct bpf_insn *prog, u
case BPF_ALU|BPF_DIV|BPF_X:
TESTrd(EDX, EDX);
- JNEb(6);
+ JNEb(4);
ZEROrd(EAX);
- MOVrq3(R8, RBX);
- RET();
+ LEAVE_RET();
MOVrd(EDX, ECX);
ZEROrd(EDX);
DIVrd(ECX);
@@ -490,13 +484,12 @@ bpf_jit_compile(struct bpf_insn *prog, u
}
#ifdef _KERNEL
- stream.ibuf = (char *)contigmalloc(stream.cur_ip, M_BPFJIT,
- M_NOWAIT, 0, ~0UL, 16, 0);
+ stream.ibuf = malloc(stream.cur_ip, M_BPFJIT, M_NOWAIT);
if (stream.ibuf == NULL)
break;
#else
- stream.ibuf = (char *)mmap(NULL, stream.cur_ip,
- PROT_READ | PROT_WRITE, MAP_ANON, -1, 0);
+ stream.ibuf = mmap(NULL, stream.cur_ip, PROT_READ | PROT_WRITE,
+ MAP_ANON, -1, 0);
if (stream.ibuf == MAP_FAILED) {
stream.ibuf = NULL;
break;
Modified: head/sys/amd64/amd64/bpf_jit_machdep.h
==============================================================================
--- head/sys/amd64/amd64/bpf_jit_machdep.h Fri Nov 20 17:59:50 2009 (r199602)
+++ head/sys/amd64/amd64/bpf_jit_machdep.h Fri Nov 20 18:49:20 2009 (r199603)
@@ -215,9 +215,14 @@ typedef void (*emit_func)(bpf_bin_stream
emitm(&stream, 0xc486, 2); \
} while (0)
-/* ret */
-#define RET() do { \
- emitm(&stream, 0xc3, 1); \
+/* pushq r64 */
+#define PUSH(r64) do { \
+ emitm(&stream, (5 << 4) | (0 << 3) | (r64 & 0x7), 1); \
+} while (0)
+
+/* leave/ret */
+#define LEAVE_RET() do { \
+ emitm(&stream, 0xc3c9, 2); \
} while (0)
/* addl sr32,dr32 */
@@ -253,6 +258,13 @@ typedef void (*emit_func)(bpf_bin_stream
emitm(&stream, i32, 4); \
} while (0)
+/* subq i8,r64 */
+#define SUBib(i8, r64) do { \
+ emitm(&stream, 0x8348, 2); \
+ emitm(&stream, (29 << 3) | (r64 & 0x7), 1); \
+ emitm(&stream, i8, 1); \
+} while (0)
+
/* mull r32 */
#define MULrd(r32) do { \
emitm(&stream, 0xf7, 1); \
Modified: head/sys/i386/i386/bpf_jit_machdep.c
==============================================================================
--- head/sys/i386/i386/bpf_jit_machdep.c Fri Nov 20 17:59:50 2009 (r199602)
+++ head/sys/i386/i386/bpf_jit_machdep.c Fri Nov 20 18:49:20 2009 (r199603)
@@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$");
#include <i386/i386/bpf_jit_machdep.h>
-bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *);
+bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *);
/*
* emit routine to update the jump table
@@ -97,7 +97,7 @@ emit_code(bpf_bin_stream *stream, u_int
* Function that does the real stuff
*/
bpf_filter_func
-bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
+bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size)
{
bpf_bin_stream stream;
struct bpf_insn *ins;
@@ -111,10 +111,9 @@ bpf_jit_compile(struct bpf_insn *prog, u
/* Allocate the reference table for the jumps */
#ifdef _KERNEL
- stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int),
- M_BPFJIT, M_NOWAIT);
+ stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, M_NOWAIT);
#else
- stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int));
+ stream.refs = malloc((nins + 1) * sizeof(u_int));
#endif
if (stream.refs == NULL)
return (NULL);
@@ -139,6 +138,7 @@ bpf_jit_compile(struct bpf_insn *prog, u
/* create the procedure header */
PUSH(EBP);
MOVrd(ESP, EBP);
+ SUBib(BPF_MEMWORDS * sizeof(uint32_t), ESP);
PUSH(EDI);
PUSH(ESI);
PUSH(EBX);
@@ -310,14 +310,16 @@ bpf_jit_compile(struct bpf_insn *prog, u
break;
case BPF_LD|BPF_MEM:
- MOVid((uintptr_t)mem, ECX);
- MOVid(ins->k * 4, ESI);
+ MOVrd(EBP, ECX);
+ MOVid(((int)ins->k - BPF_MEMWORDS) *
+ sizeof(uint32_t), ESI);
MOVobd(ECX, ESI, EAX);
break;
case BPF_LDX|BPF_MEM:
- MOVid((uintptr_t)mem, ECX);
- MOVid(ins->k * 4, ESI);
+ MOVrd(EBP, ECX);
+ MOVid(((int)ins->k - BPF_MEMWORDS) *
+ sizeof(uint32_t), ESI);
MOVobd(ECX, ESI, EDX);
break;
@@ -327,14 +329,16 @@ bpf_jit_compile(struct bpf_insn *prog, u
* be optimized if the previous instruction
* was already of this type
*/
- MOVid((uintptr_t)mem, ECX);
- MOVid(ins->k * 4, ESI);
+ MOVrd(EBP, ECX);
+ MOVid(((int)ins->k - BPF_MEMWORDS) *
+ sizeof(uint32_t), ESI);
MOVomd(EAX, ECX, ESI);
break;
case BPF_STX:
- MOVid((uintptr_t)mem, ECX);
- MOVid(ins->k * 4, ESI);
+ MOVrd(EBP, ECX);
+ MOVid(((int)ins->k - BPF_MEMWORDS) *
+ sizeof(uint32_t), ESI);
MOVomd(EDX, ECX, ESI);
break;
@@ -513,13 +517,12 @@ bpf_jit_compile(struct bpf_insn *prog, u
}
#ifdef _KERNEL
- stream.ibuf = (char *)contigmalloc(stream.cur_ip, M_BPFJIT,
- M_NOWAIT, 0, ~0UL, 16, 0);
+ stream.ibuf = malloc(stream.cur_ip, M_BPFJIT, M_NOWAIT);
if (stream.ibuf == NULL)
break;
#else
- stream.ibuf = (char *)mmap(NULL, stream.cur_ip,
- PROT_READ | PROT_WRITE, MAP_ANON, -1, 0);
+ stream.ibuf = mmap(NULL, stream.cur_ip, PROT_READ | PROT_WRITE,
+ MAP_ANON, -1, 0);
if (stream.ibuf == MAP_FAILED) {
stream.ibuf = NULL;
break;
Modified: head/sys/i386/i386/bpf_jit_machdep.h
==============================================================================
--- head/sys/i386/i386/bpf_jit_machdep.h Fri Nov 20 17:59:50 2009 (r199602)
+++ head/sys/i386/i386/bpf_jit_machdep.h Fri Nov 20 18:49:20 2009 (r199603)
@@ -203,6 +203,13 @@ typedef void (*emit_func)(bpf_bin_stream
emitm(&stream, i32, 4); \
} while (0)
+/* subl i8,r32 */
+#define SUBib(i8, r32) do { \
+ emitm(&stream, 0x83, 1); \
+ emitm(&stream, (29 << 3) | (r32 & 0x7), 1); \
+ emitm(&stream, i8, 1); \
+} while (0)
+
/* mull r32 */
#define MULrd(r32) do { \
emitm(&stream, 0xf7, 1); \
Modified: head/sys/net/bpf_jitter.c
==============================================================================
--- head/sys/net/bpf_jitter.c Fri Nov 20 17:59:50 2009 (r199602)
+++ head/sys/net/bpf_jitter.c Fri Nov 20 18:49:20 2009 (r199603)
@@ -51,7 +51,7 @@ __FBSDID("$FreeBSD$");
#include <net/bpf.h>
#include <net/bpf_jitter.h>
-bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *);
+bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *);
static u_int bpf_jit_accept_all(u_char *, u_int, u_int);
@@ -70,7 +70,7 @@ bpf_jitter(struct bpf_insn *fp, int nins
/* Allocate the filter structure */
filter = (struct bpf_jit_filter *)malloc(sizeof(*filter),
- M_BPFJIT, M_NOWAIT | M_ZERO);
+ M_BPFJIT, M_NOWAIT);
if (filter == NULL)
return (NULL);
@@ -81,8 +81,7 @@ bpf_jitter(struct bpf_insn *fp, int nins
}
/* Create the binary */
- if ((filter->func = bpf_jit_compile(fp, nins, &filter->size,
- filter->mem)) == NULL) {
+ if ((filter->func = bpf_jit_compile(fp, nins, &filter->size)) == NULL) {
free(filter, M_BPFJIT);
return (NULL);
}
@@ -95,7 +94,7 @@ bpf_destroy_jit_filter(bpf_jit_filter *f
{
if (filter->func != bpf_jit_accept_all)
- contigfree(filter->func, filter->size, M_BPFJIT);
+ free(filter->func, M_BPFJIT);
free(filter, M_BPFJIT);
}
#else
@@ -108,7 +107,6 @@ bpf_jitter(struct bpf_insn *fp, int nins
filter = (struct bpf_jit_filter *)malloc(sizeof(*filter));
if (filter == NULL)
return (NULL);
- memset(filter, 0, sizeof(*filter));
/* No filter means accept all */
if (fp == NULL || nins == 0) {
@@ -117,8 +115,7 @@ bpf_jitter(struct bpf_insn *fp, int nins
}
/* Create the binary */
- if ((filter->func = bpf_jit_compile(fp, nins, &filter->size,
- filter->mem)) == NULL) {
+ if ((filter->func = bpf_jit_compile(fp, nins, &filter->size)) == NULL) {
free(filter);
return (NULL);
}
Modified: head/sys/net/bpf_jitter.h
==============================================================================
--- head/sys/net/bpf_jitter.h Fri Nov 20 17:59:50 2009 (r199602)
+++ head/sys/net/bpf_jitter.h Fri Nov 20 18:49:20 2009 (r199603)
@@ -54,7 +54,6 @@ typedef struct bpf_jit_filter {
/* The native filtering binary, in the form of a bpf_filter_func. */
bpf_filter_func func;
size_t size;
- int mem[BPF_MEMWORDS]; /* Scratch memory */
} bpf_jit_filter;
/*
More information about the svn-src-head
mailing list