kern/92593: panic when used ipfw uid/gid checks and ipfw_ether

Andrey V. Elsukov bu7cher at yandex.ru
Tue Jan 31 01:00:14 PST 2006


>Number:         92593
>Category:       kern
>Synopsis:       panic when used ipfw uid/gid checks and ipfw_ether
>Confidential:   no
>Severity:       serious
>Priority:       high
>Responsible:    freebsd-bugs
>State:          open
>Quarter:        
>Keywords:       
>Date-Required:
>Class:          sw-bug
>Submitter-Id:   current-users
>Arrival-Date:   Tue Jan 31 09:00:12 GMT 2006
>Closed-Date:
>Last-Modified:
>Originator:     Andrey V. Elsukov
>Release:        FreeBSD 7.0-CURRENT
>Organization:
>Environment:
7.0-CURRENT, but i have some reports from my friend, that this panic can get
on the 5.4-RELEASE and 6.0-RELEASE.

I have IPFW in the kernel.
# cat /boot/loader.conf
debug.mpsafenet=0

>Description:
I get a kernel panic when receive some ip packet 
(in this backtrace - udp broadcast).

>How-To-Repeat:
My system is in the local network, i try a folowing commands:
# ipfw add 1 count ip from any to any uid 0
# sysctl net.inet.ether.ipfw=1


--- bt.log begins here ---
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB.  Type "show warranty" for details.
This GDB was configured as "i386-marcel-freebsd".

Unread portion of the kernel message buffer:


Fatal trap 12: page fault while in kernel mode
cpuid = 0; apic id = 00
fault virtual address	= 0x203a7325
fault code		= supervisor read, page not present
instruction pointer	= 0x20:0xc06bb03c
stack pointer	        = 0x28:0xc608a8bc
frame pointer	        = 0x28:0xc608a8bc
code segment		= base 0x0, limit 0xfffff, type 0x1b
			= DPL 0, pres 1, def32 1, gran 1
processor eflags	= interrupt enabled, resume, IOPL = 0
current process		= 31 (em0 taskq)
Dumping 63 MB (2 chunks)
  chunk 0: 1MB (159 pages) ... ok
  chunk 1: 63MB (16128 pages) 48 32 16

#0  doadump () at pcpu.h:166
	in pcpu.h
(kgdb) bt full
#0  doadump () at pcpu.h:166
No locals.
#1  0xc046b3c7 in db_fncall (dummy1=-1063420928, dummy2=0, 
    dummy3=-1065352653, dummy4=0xc608a68c "¸¦\bÆxå\177À¤¦\bƨ¦\bÆ\220\a")
    at /usr/src/sys/ddb/db_command.c:489
	fn_addr = -1067104744
	args = {1, 0, 539020440, 78, -1063540320, -1063540544, 0, -972511628, 
  2, -1064560544}
	nargs = 0
	retval = 0
	t = 0
#2  0xc046b1cc in db_command (last_cmdp=0xc093f604, cmd_table=0x0, 
    aux_cmd_tablep=0xc08b8840, aux_cmd_tablep_end=0xc08b885c)
    at /usr/src/sys/ddb/db_command.c:404
	cmd = (struct command *) 0xc08c1a00
	t = 0
	modif = "¸¦\bÆxå\177À¤¦\bƨ¦\bÆ\220\a\000\000\220\a\000\000Ï\a\000\000\000\000\000\000\000|\235À\r\000\000\000\000|\235À\000|\235À\r\000\000\000\001\000\000\000ä¦\bÆ«Þ\177Àä¦\bÆÄÞ\177ÀÀ¨\233À\200I\232Àx\000\000\000\000ÿ\223À\f\000\000\000\004§\bÆhÒFÀ/¯\210À@ÏFÀ\f\000\000\000\000ÿ\223ÀòÆFÀ"
	addr = -1063420928
	count = -1065352653
	have_addr = 0
	result = 0
#3  0xc046b294 in db_command_loop () at /usr/src/sys/ddb/db_command.c:455
No locals.
#4  0xc046cead in db_trap (type=12, code=0) at /usr/src/sys/ddb/db_main.c:221
	jb = {{_jb = {-972511420, -972511440, -972511368, 1, 12, -1069101498, 
      -972511108, 2, 1, -972511108, 1, 12}}}
	prev_jb = (void *) 0x0
	bkpt = 0
#5  0xc066fc7c in kdb_trap (type=12, code=0, tf=0xc608a87c)
    at /usr/src/sys/kern/subr_kdb.c:485
	did_stop_cpus = 1
	handled = -972511108
#6  0xc081d6b8 in trap_fatal (frame=0xc608a87c, eva=540701477)
    at /usr/src/sys/i386/i386/trap.c:853
	eflags = 514
	code = 514
	type = 12
	ss = 514
	esp = 0
	softseg = {ssd_base = 0, ssd_limit = 1048575, ssd_type = 27, 
  ssd_dpl = 0, ssd_p = 1, ssd_xx = 13, ssd_xx1 = 1, ssd_def32 = 1, 
  ssd_gran = 1}
	msg = 0x0
#7  0xc081d3fb in trap_pfault (frame=0xc608a87c, usermode=0, eva=540701477)
    at /usr/src/sys/i386/i386/trap.c:770
	va = 540700672
	vm = (struct vmspace *) 0x0
	map = 0xc0956a40
	rv = 1
	ftype = 1 '\001'
	td = (struct thread *) 0xc1682d00
	p = (struct proc *) 0xc1681d38
#8  0xc081d015 in trap (frame=
      {tf_fs = 8, tf_es = 40, tf_ds = 40, tf_edi = 0, tf_esi = 1, tf_ebp = -972511044, tf_isp = -972511064, tf_ebx = -1064787347, tf_edx = 540701477, tf_ecx = 0, tf_eax = 540701477, tf_trapno = 12, tf_err = 0, tf_eip = -1066684356, tf_cs = 32, tf_eflags = 66118, tf_esp = -972510844, tf_ss = -1066983424})
    at /usr/src/sys/i386/i386/trap.c:455
	td = (struct thread *) 0xc1682d00
	p = (struct proc *) 0xc1681d38
	sticks = 0
	i = 0
	ucode = 0
	type = 12
	code = 0
	addr = 0
	eva = 540701477
	ksi = {ksi_link = {tqe_next = 0x0, tqe_prev = 0x0}, ksi_info = {
    si_signo = 0, si_errno = 0, si_code = 0, si_pid = 0, si_uid = 0, 
    si_status = 0, si_addr = 0x0, si_value = {sival_int = 0, 
      sival_ptr = 0x0}, _reason = {_fault = {_trapno = 0}, _timer = {
        _timerid = 0, _overrun = 0}, _mesgq = {_mqd = 0}, _poll = {
        _band = 0}, __spare__ = {__spare1__ = 0, __spare2__ = {0, 0, 0, 0, 0, 
          0, 0}}}}, ksi_flags = 0, ksi_sigq = 0x0}
#9  0xc08096da in calltrap () at /usr/src/sys/i386/i386/exception.s:137
No locals.
#10 0xc06bb03c in strlen (str=0x203a7325 <Address 0x203a7325 out of bounds>)
    at /usr/src/sys/libkern/strlen.c:41
	s = 0x203a7325 <Address 0x203a7325 out of bounds>
#11 0xc0672000 in kvprintf (fmt=0xc088a26d " not owned at %s:%d", 
    func=0xc0671928 <snprintf_func>, arg=0xc608a9a0, radix=10, 
    ap=0xc608a9e8 "\214\224\211À\207\a") at /usr/src/sys/kern/subr_prf.c:679
	nbuf = "ÿÿÿÿ\000Ø\000\000\225zA\000Ü©\bÆè[\202À¶©\bÆ\001\000\000\000\n\000\000\000À©\bÆ\004\000\000\000\n\000\000\000\000\000\000\000zÚ\000\000\000d\000\000zÚ\000\000\000\000\000\000"
	d = 0x0
	p = 0x203a7325 <Address 0x203a7325 out of bounds>
	percent = 0xc088a26b "%s not owned at %s:%d"
	q = 0x1 <Address 0x1 out of bounds>
	up = (u_char *) 0x0
	ch = 540701477
	n = 1
	num = 0
	base = 0
	lflag = 0
	qflag = 0
	tmp = 1
	width = 0
	ladjust = 0
	sharpflag = 0
	neg = 0
	sign = 0
	dot = 0
	cflag = 0
	hflag = 0
	jflag = 0
	tflag = 0
	zflag = 0
	dwidth = 0
	padc = 32 ' '
	stop = 0
	retval = 6
#12 0xc06718c5 in vsnprintf (
    str=0x203a7325 <Address 0x203a7325 out of bounds>, size=540701477, 
    format=0xc088a265 "mutex %s not owned at %s:%d", 
    ap=0xc608a9e4 "%s: \214\224\211À\207\a")
    at /usr/src/sys/kern/subr_prf.c:413
	info = {str = 0xc0958266 "", remain = 250}
	retval = 540701477
#13 0xc0654dd2 in panic (fmt=0xc088a265 "mutex %s not owned at %s:%d")
    at /usr/src/sys/kern/kern_shutdown.c:522
	td = (struct thread *) 0xc1682d00
	bootopt = 256
	newpanic = 1
	ap = 0xc608a9e4 "%s: \214\224\211À\207\a"
	buf = "mutex ", '\0' <repeats 249 times>
#14 0xc064d31a in _mtx_assert (m=0xc088ebd2, what=0, 
    file=0xc089948c "/usr/src/sys/netinet/ip_fw2.c", line=1927)
    at /usr/src/sys/kern/kern_mutex.c:754
No locals.
#15 0xc06ee15e in check_uidgid (insn=0xc1b193a4, proto=17, oif=0x0, dst_ip=
      {s_addr = 4283504044}, dst_port=138, src_ip={s_addr = 961615276}, 
    src_port=138, ugp=0xc608aae0, lookup=0xc608aacc, inp=0xc088eb42)
    at /usr/src/sys/netinet/ip_fw2.c:1927
	pi = (struct inpcbinfo *) 0x99
	wildcard = -1064768702
	pcb = (struct inpcb *) 0xc088eb42
	match = -1063623928
	gp = (gid_t *) 0x203a7325
#16 0xc06ef037 in ipfw_chk (args=0xc608ab54)
    at /usr/src/sys/netinet/ip_fw2.c:2467
	match = 0
	tablearg = 0
	skip_or = 0
	cmd = (ipfw_insn *) 0xc1b193a4
	l = 3
	cmdlen = 2
	m = (struct mbuf *) 0xc1870300
	ip = (struct ip *) 0xc18af810
	fw_ugid_cache = {fw_groups = {1, 0, 416612352, 4259840, 3322456984, 
    0, 3230777056, 4291221, 3322456860, 3229769270, 0, 2147483648, 3579545, 
    0, 0, 3322456964}, fw_ngroups = -1067066171, fw_uid = 0, 
  fw_prid = -2147483648}
	ugid_lookup = 0
	divinput_flags = 0
	oif = (struct ifnet *) 0x0
	f = (struct ip_fw *) 0xc1b19380
	retval = 3
	hlen = 20
	offset = 0
	proto = 17 '\021'
	src_port = 138
	dst_port = 138
	src_ip = {s_addr = 961615276}
	dst_ip = {s_addr = 4283504044}
	ip_len = 244
	pktlen = 244
	dyn_dir = 3
	q = (ipfw_dyn_rule *) 0x0
	mtag = (struct m_tag *) 0xc1b19380
	ulp = (void *) 0xc18af824
	is_ipv6 = 0
	ext_hd = 0
	is_ipv4 = 1
#17 0xc06c292d in ether_ipfw_chk (m0=0xc608ac74, dst=0x0, rule=0xc608ac58, 
    shared=0) at /usr/src/sys/net/if_ethersubr.c:429
	eh = (struct ether_header *) 0xc18af802
	save_eh = {ether_dhost = "ÿÿÿÿÿÿ", ether_shost = "\000P¿Óòh", 
  ether_type = 8}
	m = (struct mbuf *) 0xc1870300
	i = -1047857136
	args = {m = 0xc1870300, oif = 0x0, next_hop = 0x0, rule = 0x0, 
  eh = 0xc608ac24, flags = -1067021237, f_id = {dst_ip = 2887078399, 
    src_ip = 2887078201, dst_port = 138, src_port = 138, proto = 17 '\021', 
    flags = 2 '\002', addr_type = 4 '\004', dst_ip6 = {__u6_addr = {
        __u6_addr8 = "\000-hÁ\214«\bÆF\002\000\000tà\225À", __u6_addr16 = {
          11520, 49512, 43916, 50696, 582, 0, 57460, 49301}, __u6_addr32 = {
          3244829952, 3322456972, 582, 3231047796}}}, src_ip6 = {__u6_addr = {
        __u6_addr8 = "\230«\bÆAÏdÀÀF\232ÀÄ«\bÆ", __u6_addr16 = {43928, 50696, 
          53057, 49252, 18112, 49306, 43972, 50696}, __u6_addr32 = {
          3322456984, 3227832129, 3231336128, 3322457028}}}, 
    flow_id6 = 3228020860, frag_id6 = 3231047796}, cookie = 0, 
  inp = 0xc088eb42, dummypar = {opt_or = 0x6ae, ro_or = {ro_rt = 0xc095e074, 
      ro_dst = {sin6_len = 0 '\0', sin6_family = 0 '\0', sin6_port = 0, 
        sin6_flowinfo = 3230198594, sin6_addr = {__u6_addr = {
            __u6_addr8 = "«\006\000\000\000-hÁ\003\000\000\000\020\000\000", 
            __u6_addr16 = {1707, 0, 11520, 49512, 3, 0, 16, 0}, 
            __u6_addr32 = {1707, 3244829952, 3, 16}}}, 
        sin6_scope_id = 3322457064}}, flags_or = -1067021237, 
    im6o_or = 0xc0958030, origifp_or = 0x2, ifp_or = 0xc088c812, dst_or = {
      sin6_len = 110 'n', sin6_family = 2 '\002', sin6_port = 0, 
      sin6_flowinfo = 3244829952, sin6_addr = {__u6_addr = {
          __u6_addr8 = "ô«\bÆF\002\000\000 :\225À\000¬\bÆ", __u6_addr16 = {
            44020, 50696, 582, 0, 14880, 49301, 44032, 50696}, __u6_addr32 = {
            3322457076, 582, 3231005216, 3322457088}}}, 
      sin6_scope_id = 3227832129}, mtu_or = 3243774656, ro_pmtu_or = {
      ro_rt = 0xc608ac24, ro_dst = {sin6_len = 197 'Å', sin6_family = 52 '4', 
        sin6_port = 49242, sin6_flowinfo = 3231005216, sin6_addr = {
          __u6_addr = {
            __u6_addr8 = "\000\000\000\000È\214\207ÀV\001\000\000F\000\000", 
            __u6_addr16 = {0, 0, 36040, 49287, 342, 0, 70, 0}, __u6_addr32 = {
              0, 3230108872, 342, 70}}}, sin6_scope_id = 3247048704}}}}
	__func__ = "ether_ipfw_chk"
#18 0xc06c2df7 in ether_demux (ifp=0xc16a5800, m=0xc1870300)
    at /usr/src/sys/net/if_ethersubr.c:683
	eh = (struct ether_header *) 0xc18af802
	isr = 540701477
	ether_type = 2048
	rule = (struct ip_fw *) 0x0
	__func__ = "ether_demux"
#19 0xc06c2caa in ether_input (ifp=0xc16a5800, m=0xc1870300)
    at /usr/src/sys/net/if_ethersubr.c:595
	eh = (struct ether_header *) 0x203a7325
	etype = 2048
	__func__ = "ether_input"
#20 0xc0516313 in em_process_receive_interrupts (adapter=0xc1657800, count=99)
    at /usr/src/sys/dev/em/if_em.c:3180
	m = (struct mbuf *) 0xc1870300
	ifp = (struct ifnet *) 0xc16a5800
	mp = (struct mbuf *) 0xc1870300
	accept_frame = 1 '\001'
	eop = 1 '\001'
	len = 258
	desc_len = 29477
	prev_len_adj = 0
	i = 251
	current_desc = (struct em_rx_desc *) 0xc1699fa0
#21 0xc0512f2f in em_handle_rxtx (context=0xc1657800, pending=1)
    at /usr/src/sys/dev/em/if_em.c:1110
	adapter = (struct adapter *) 0xc1657800
	ifp = (struct ifnet *) 0xc16a5800
#22 0xc0676a4c in taskqueue_run (queue=0xc167ec00)
    at /usr/src/sys/kern/subr_taskqueue.c:255
	task = (struct task *) 0xc16579d0
	owned = 1
	pending = 1
#23 0xc0676d76 in taskqueue_thread_loop (arg=0x203a7325)
    at /usr/src/sys/kern/subr_taskqueue.c:358
	tq = (struct taskqueue *) 0xc167ec00
#24 0xc0640f0c in fork_exit (callout=0xc0676d2c <taskqueue_thread_loop>, 
    arg=0xc16579e0, frame=0xc608ad38) at /usr/src/sys/kern/kern_fork.c:790
	p = (struct proc *) 0xc1681d38
	td = (struct thread *) 0x203a7325
#25 0xc080973c in fork_trampoline () at /usr/src/sys/i386/i386/exception.s:198
No locals.
(kgdb) f 16
#16 0xc06ef037 in ipfw_chk (args=0xc608ab54)
    at /usr/src/sys/netinet/ip_fw2.c:2467
2467						match = check_uidgid(
(kgdb) info locals             p *args
$1 = {m = 0xc1870300, oif = 0x0, next_hop = 0x0, rule = 0x0, eh = 0xc608ac24, 
  flags = -1067021237, f_id = {dst_ip = 2887078399, src_ip = 2887078201, 
    dst_port = 138, src_port = 138, proto = 17 '\021', flags = 2 '\002', 
    addr_type = 4 '\004', dst_ip6 = {__u6_addr = {
        __u6_addr8 = "\000-hÁ\214«\bÆF\002\000\000tà\225À", __u6_addr16 = {
          11520, 49512, 43916, 50696, 582, 0, 57460, 49301}, __u6_addr32 = {
          3244829952, 3322456972, 582, 3231047796}}}, src_ip6 = {__u6_addr = {
        __u6_addr8 = "\230«\bÆAÏdÀÀF\232ÀÄ«\bÆ", __u6_addr16 = {43928, 50696, 
          53057, 49252, 18112, 49306, 43972, 50696}, __u6_addr32 = {
          3322456984, 3227832129, 3231336128, 3322457028}}}, 
    flow_id6 = 3228020860, frag_id6 = 3231047796}, cookie = 0, 
  inp = 0xc088eb42, dummypar = {opt_or = 0x6ae, ro_or = {ro_rt = 0xc095e074, 
      ro_dst = {sin6_len = 0 '\0', sin6_family = 0 '\0', sin6_port = 0, 
        sin6_flowinfo = 3230198594, sin6_addr = {__u6_addr = {
            __u6_addr8 = "«\006\000\000\000-hÁ\003\000\000\000\020\000\000", 
            __u6_addr16 = {1707, 0, 11520, 49512, 3, 0, 16, 0}, 
            __u6_addr32 = {1707, 3244829952, 3, 16}}}, 
        sin6_scope_id = 3322457064}}, flags_or = -1067021237, 
    im6o_or = 0xc0958030, origifp_or = 0x2, ifp_or = 0xc088c812, dst_or = {
      sin6_len = 110 'n', sin6_family = 2 '\002', sin6_port = 0, 
      sin6_flowinfo = 3244829952, sin6_addr = {__u6_addr = {
          __u6_addr8 = "ô«\bÆF\002\000\000 :\225À\000¬\bÆ", __u6_addr16 = {
            44020, 50696, 582, 0, 14880, 49301, 44032, 50696}, __u6_addr32 = {
            3322457076, 582, 3231005216, 3322457088}}}, 
      sin6_scope_id = 3227832129}, mtu_or = 3243774656, ro_pmtu_or = {
      ro_rt = 0xc608ac24, ro_dst = {sin6_len = 197 'Å', sin6_family = 52 '4', 
        sin6_port = 49242, sin6_flowinfo = 3231005216, sin6_addr = {
          __u6_addr = {
            __u6_addr8 = "\000\000\000\000È\214\207ÀV\001\000\000F\000\000", 
            __u6_addr16 = {0, 0, 36040, 49287, 342, 0, 70, 0}, __u6_addr32 = {
              0, 3230108872, 342, 70}}}, sin6_scope_id = 3247048704}}}}
(kgdb) p *args->eh
$2 = {ether_dhost = "ÿÿÿÿÿÿ", ether_shost = "\000P¿Óòh", ether_type = 8}
(kgdb) f 15
#15 0xc06ee15e in check_uidgid (insn=0xc1b193a4, proto=17, oif=0x0, dst_ip=
      {s_addr = 4283504044}, dst_port=138, src_ip={s_addr = 961615276}, 
    src_port=138, ugp=0xc608aae0, lookup=0xc608aacc, inp=0xc088eb42)
    at /usr/src/sys/netinet/ip_fw2.c:1927
1927			INP_LOCK_ASSERT(inp);
(kgdb) p *inp
$3 = {inp_hash = {le_next = 0x7273752f, le_prev = 0x6372732f}, inp_list = {
    le_next = 0x7379732f, le_prev = 0x72656b2f}, inp_flow = 1970483054, 
  inp_inc = {inc_flags = 98 'b', inc_len = 114 'r', inc_pad = 30559, 
    inc_ie = {ie_fport = 29801, ie_lport = 25966, ie_dependfaddr = {
        ie46_foreign = {ia46_pad32 = {1663988595, 1953068800, 1936942446}, 
          ia46_addr4 = {s_addr = 1668246560}}, ie6_foreign = {__u6_addr = {
            __u6_addr8 = "ss.c\000witness loc", __u6_addr16 = {29555, 25390, 
              30464, 29801, 25966, 29555, 27680, 25455}, __u6_addr32 = {
              1663988595, 1953068800, 1936942446, 1668246560}}}}, 
      ie_dependladdr = {ie46_local = {ia46_pad32 = {1919877227, 544367972, 
            1953720684}, ia46_addr4 = {s_addr = 1953451520}}, ie6_local = {
          __u6_addr = {__u6_addr8 = "k\000order list\000Not", __u6_addr16 = {
              107, 29295, 25956, 8306, 26988, 29811, 19968, 29807}, 
            __u6_addr32 = {1919877227, 544367972, 1953720684, 
              1953451520}}}}}}, 
  inp_ppcb = 0x6f6e6520 <Address 0x6f6e6520 out of bounds>, 
  inp_pcbinfo = 0x20686775, inp_socket = 0x6f6d656d, inp_label = 0x66207972, 
  inp_flags = 1931506287, inp_sp = 0x69746174, inp_vflag = 99 'c', 
  inp_ip_ttl = 32 ' ', inp_ip_p = 111 'o', inp_ip_minttl = 114 'r', 
  inp_depend4 = {inp4_ip_tos = 100 'd', inp4_options = 0x73250021, 
    inp4_moptions = 0x6f6c203a}, inp_depend6 = {inp6_options = 0x25206b63, 
    inp6_outputopts = 0x73692073, inp6_moptions = 0x206e6f20, 
    inp6_icmp6filt = 0x646e6570, inp6_cksum = 543649385, 
    inp6_ifindex = 26988, inp6_hops = 29811}, inp_portlist = {
    le_next = 0x74756220, le_prev = 0x746f6e20}, inp_phd = 0x5f4f4c20, 
  inp_gencnt = 23453980198979927, inp_mtx = {mtx_object = {
      lo_name = 0x203a7325 <Address 0x203a7325 out of bounds>, 
      lo_type = 0x6b636f6c <Address 0x6b636f6c out of bounds>, 
      lo_flags = 1931814944, lo_witness_data = {lod_list = {
          stqe_next = 0x73252029}, lod_witness = 0x73252029}}, 
    mtx_lock = 1851876128, mtx_recurse = 1953459744}}
(kgdb) 
--- bt.log ends here ---


>Fix:

	
--- src/sys/netinet/ip_fw2.c    Tue Jan 24 13:38:06 2006
+++ src/sys/netinet/ip_fw2.c    Tue Jan 31 10:31:12 2006
@@ -2462,6 +2462,12 @@
                                        break;
                                if (is_ipv6) /* XXX to be fixed later */
                                        break;
+                               /*
+                                * XXX uid/gid checks don't work with
+                                * a layer2 packets
+                                */
+                               if (args->eh != NULL)
+                                       break;
                                if (proto == IPPROTO_TCP ||
                                    proto == IPPROTO_UDP)
                                        match = check_uidgid(

>Release-Note:
>Audit-Trail:
>Unformatted:


More information about the freebsd-bugs mailing list