usb/130736: Page fault unplugging USB stick

Theo van Klaveren Theo.van.Klaveren at ats-global.com
Mon Jan 19 05:20:02 PST 2009


The following reply was made to PR usb/130736; it has been noted by GNATS.

From: "Theo van Klaveren" <Theo.van.Klaveren at ats-global.com>
To: <bug-followup at FreeBSD.org>
Cc:  
Subject: Re: usb/130736: Page fault unplugging USB stick
Date: Mon, 19 Jan 2009 14:04:22 +0100

 This is a multi-part message in MIME format.
 
 ------_=_NextPart_001_01C97A36.87213BC8
 Content-Type: text/plain;
 	charset="iso-8859-1"
 Content-Transfer-Encoding: quoted-printable
 
 Yup, crash is at the TAILQ_INSERT_TAIL, line 4835:
 
 =20
 
 (kgdb) bt
 
 #0  doadump () at pcpu.h:196
 
 #1  0xc0790ea7 in boot (howto=3D260) at =
 /usr/src/sys/kern/kern_shutdown.c:418
 
 #2  0xc0791179 in panic (fmt=3DVariable "fmt" is not available.
 
 ) at /usr/src/sys/kern/kern_shutdown.c:574
 
 #3  0xc0aa338c in trap_fatal (frame=3D0xe406c974, eva=3D0)
 
     at /usr/src/sys/i386/i386/trap.c:939
 
 #4  0xc0aa3610 in trap_pfault (frame=3D0xe406c974, usermode=3D0, =
 eva=3D0)
 
     at /usr/src/sys/i386/i386/trap.c:852
 
 #5  0xc0aa3fcc in trap (frame=3D0xe406c974) at =
 /usr/src/sys/i386/i386/trap.c:530
 
 #6  0xc0a89e3b in calltrap () at /usr/src/sys/i386/i386/exception.s:159
 
 #7  0xc046ae6b in xpt_done (done_ccb=3D0xc4030400)
 
     at /usr/src/sys/cam/cam_xpt.c:4835
 
 #8  0xc047154e in probedone (periph=3D0xc47ee200, done_ccb=3DVariable =
 "done_ccb" is not available.
 
 )
 
     at /usr/src/sys/cam/cam_xpt.c:6392
 
 #9  0xc046cff1 in camisr_runqueue (V_queue=3DVariable "V_queue" is not =
 available.
 
 ) at /usr/src/sys/cam/cam_xpt.c:7316
 
 #10 0xc047093f in xpt_bus_deregister (pathid=3D0)
 
     at /usr/src/sys/cam/cam_xpt.c:4421
 
 #11 0xc06f6dc0 in umass_cam_detach_sim (sc=3D0xc4467a00)
 
     at /usr/src/sys/dev/usb/umass.c:2716
 
 #12 0xc06f6e6d in umass_detach (self=3D0xc47ee680)
 
     at /usr/src/sys/dev/usb/umass.c:1564
 
 #13 0xc07b5e38 in device_detach (dev=3D0xc47ee680) at device_if.h:212
 
 #14 0xc06fdfe2 in usb_disconnect_port (up=3D0xc3fd9494, =
 parent=3D0xc3fece80)
 
     at /usr/src/sys/dev/usb/usb_subr.c:1380
 
 #15 0xc06f3e6e in uhub_explore (dev=3D0xc3fecc80)
 
     at /usr/src/sys/dev/usb/uhub.c:462
 
 #16 0xc06fc195 in usb_discover (v=3DVariable "v" is not available.
 
 ) at /usr/src/sys/dev/usb/usb.c:724
 
 #17 0xc06fd187 in usb_event_thread (arg=3D0xc3fd4880)
 
     at /usr/src/sys/dev/usb/usb.c:440
 
 #18 0xc076ca19 in fork_exit (callout=3D0xc06fd0d0 <usb_event_thread>,
 
     arg=3D0xc3fd4880, frame=3D0xe406cd38) at =
 /usr/src/sys/kern/kern_fork.c:804
 
 #19 0xc0a89eb0 in fork_trampoline () at =
 /usr/src/sys/i386/i386/exception.s:264
 
 =20
 
 (kgdb) print *sim
 
 $2 =3D {sim_action =3D 0xc046af70 <dead_sim_action>,
 
   sim_poll =3D 0xc0469b00 <dead_sim_poll>, sim_name =3D 0xc0af0b2e =
 "dead_sim",
 
   softc =3D 0x0, mtx =3D 0x0, sim_doneq =3D {tqh_first =3D 0x0, tqh_last =
 =3D 0x0},
 
   links =3D {tqe_next =3D 0x0, tqe_prev =3D 0x0}, path_id =3D 0, =
 unit_number =3D 0,
 
   bus_id =3D 0, max_tagged_dev_openings =3D 0, max_dev_openings =3D 0, =
 flags =3D 0,
 
   callout =3D {c_links =3D {sle =3D {sle_next =3D 0x0}, tqe =3D =
 {tqe_next =3D 0x0,
 
         tqe_prev =3D 0x0}}, c_time =3D 0, c_arg =3D 0x0, c_func =3D 0, =
 c_mtx =3D 0x0,
 
     c_flags =3D 0}, devq =3D 0x0, ccb_freeq =3D {slh_first =3D =
 0xc4022400},
 
   max_ccbs =3D 0, ccb_count =3D 0}
 
 =20
 
 (kgdb) print done_ccb->ccb_h
 
 $5 =3D {pinfo =3D {priority =3D 5, generation =3D 5, index =3D -1}, =
 xpt_links =3D {le =3D {
 
       le_next =3D 0x0, le_prev =3D 0x0}, sle =3D {sle_next =3D 0x0}, tqe =
 =3D {
 
       tqe_next =3D 0x0, tqe_prev =3D 0x0}, stqe =3D {stqe_next =3D =
 0x0}}, sim_links =3D {
 
     le =3D {le_next =3D 0x0, le_prev =3D 0x0}, sle =3D {sle_next =3D =
 0x0}, tqe =3D {
 
       tqe_next =3D 0x0, tqe_prev =3D 0x0}, stqe =3D {stqe_next =3D =
 0x0}},
 
   periph_links =3D {le =3D {le_next =3D 0x0, le_prev =3D 0xc4015c00}, =
 sle =3D {
 
       sle_next =3D 0x0}, tqe =3D {tqe_next =3D 0x0, tqe_prev =3D =
 0xc4015c00}, stqe =3D {
 
       stqe_next =3D 0x0}}, retry_count =3D 0, cbfcnp =3D 0xc046e8f0 =
 <xpt_scan_bus>,
 
   func_code =3D XPT_SCAN_LUN, status =3D 1, path =3D 0xc480d290, path_id =
 =3D 0,
 
   target_id =3D 0, target_lun =3D 0, flags =3D 0, periph_priv =3D =
 {entries =3D {{
 
         ptr =3D 0xc480d2a0, field =3D 3296776864, bytes =3D " =
 =D2\200=C4"}, {ptr =3D 0x0,
 
         field =3D 0, bytes =3D "\000\000\000"}}, bytes =3D " =
 =D2\200=C4\000\000\000"},
 
   sim_priv =3D {entries =3D {{ptr =3D 0x0, field =3D 0, bytes =3D =
 "\000\000\000"}, {
 
         ptr =3D 0x0, field =3D 0, bytes =3D "\000\000\000"}},
 
     bytes =3D "\000\000\000\000\000\000\000"}, timeout =3D 0, timeout_ch =
 =3D {
 
     callout =3D 0x0}}
 
 =20
 
 Hope this helps.
 
 =20
 
 
 ------_=_NextPart_001_01C97A36.87213BC8
 Content-Type: text/html;
 	charset="iso-8859-1"
 Content-Transfer-Encoding: quoted-printable
 
 <META HTTP-EQUIV=3D"Content-Type" CONTENT=3D"text/html; =
 charset=3Diso-8859-1">
 <html xmlns:v=3D"urn:schemas-microsoft-com:vml" =
 xmlns:o=3D"urn:schemas-microsoft-com:office:office" =
 xmlns:w=3D"urn:schemas-microsoft-com:office:word" =
 xmlns:m=3D"http://schemas.microsoft.com/office/2004/12/omml" =
 xmlns=3D"http://www.w3.org/TR/REC-html40">
 
 <head>
 
 <meta name=3DGenerator content=3D"Microsoft Word 12 (filtered medium)">
 <style>
 <!--
  /* Font Definitions */
  @font-face
 	{font-family:Calibri;
 	panose-1:2 15 5 2 2 2 4 3 2 4;}
  /* Style Definitions */
  p.MsoNormal, li.MsoNormal, div.MsoNormal
 	{margin:0cm;
 	margin-bottom:.0001pt;
 	font-size:11.0pt;
 	font-family:"Calibri","sans-serif";}
 a:link, span.MsoHyperlink
 	{mso-style-priority:99;
 	color:blue;
 	text-decoration:underline;}
 a:visited, span.MsoHyperlinkFollowed
 	{mso-style-priority:99;
 	color:purple;
 	text-decoration:underline;}
 span.EmailStyle17
 	{mso-style-type:personal-compose;
 	font-family:"Calibri","sans-serif";
 	color:windowtext;}
 .MsoChpDefault
 	{mso-style-type:export-only;}
 @page Section1
 	{size:612.0pt 792.0pt;
 	margin:70.85pt 70.85pt 70.85pt 70.85pt;}
 div.Section1
 	{page:Section1;}
 -->
 </style>
 <!--[if gte mso 9]><xml>
  <o:shapedefaults v:ext=3D"edit" spidmax=3D"1026" />
 </xml><![endif]--><!--[if gte mso 9]><xml>
  <o:shapelayout v:ext=3D"edit">
   <o:idmap v:ext=3D"edit" data=3D"1" />
  </o:shapelayout></xml><![endif]-->
 </head>
 
 <body lang=3DNL link=3Dblue vlink=3Dpurple>
 
 <div class=3DSection1>
 
 <p class=3DMsoNormal><span lang=3DEN-US>Yup, crash is at the =
 TAILQ_INSERT_TAIL,
 line 4835:<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US><o:p>&nbsp;</o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>(kgdb) bt<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#0&nbsp; doadump () at =
 pcpu.h:196<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#1&nbsp; 0xc0790ea7 in boot =
 (howto=3D260) at
 /usr/src/sys/kern/kern_shutdown.c:418<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#2&nbsp; 0xc0791179 in panic =
 (fmt=3DVariable
 &quot;fmt&quot; is not available.<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>) at =
 /usr/src/sys/kern/kern_shutdown.c:574<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#3&nbsp; 0xc0aa338c in =
 trap_fatal
 (frame=3D0xe406c974, eva=3D0)<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp; at
 /usr/src/sys/i386/i386/trap.c:939<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#4&nbsp; 0xc0aa3610 in =
 trap_pfault
 (frame=3D0xe406c974, usermode=3D0, eva=3D0)<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp; at
 /usr/src/sys/i386/i386/trap.c:852<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#5&nbsp; 0xc0aa3fcc in trap
 (frame=3D0xe406c974) at =
 /usr/src/sys/i386/i386/trap.c:530<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#6&nbsp; 0xc0a89e3b in calltrap =
 () at
 /usr/src/sys/i386/i386/exception.s:159<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#7&nbsp; 0xc046ae6b in xpt_done
 (done_ccb=3D0xc4030400)<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp; at
 /usr/src/sys/cam/cam_xpt.c:4835<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#8&nbsp; 0xc047154e in probedone =
 (periph=3D0xc47ee200,
 done_ccb=3DVariable &quot;done_ccb&quot; is not =
 available.<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>)<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp; at
 /usr/src/sys/cam/cam_xpt.c:6392<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#9&nbsp; 0xc046cff1 in =
 camisr_runqueue
 (V_queue=3DVariable &quot;V_queue&quot; is not =
 available.<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>) at =
 /usr/src/sys/cam/cam_xpt.c:7316<o:p></o:p></span></p>
 
 <p class=3DMsoNormal>#10 0xc047093f in xpt_bus_deregister =
 (pathid=3D0)<o:p></o:p></p>
 
 <p class=3DMsoNormal>&nbsp;&nbsp;&nbsp; <span lang=3DEN-US>at
 /usr/src/sys/cam/cam_xpt.c:4421<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#11 0xc06f6dc0 in =
 umass_cam_detach_sim
 (sc=3D0xc4467a00)<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp; at
 /usr/src/sys/dev/usb/umass.c:2716<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#12 0xc06f6e6d in umass_detach
 (self=3D0xc47ee680)<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp; at
 /usr/src/sys/dev/usb/umass.c:1564<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#13 0xc07b5e38 in device_detach
 (dev=3D0xc47ee680) at device_if.h:212<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#14 0xc06fdfe2 in =
 usb_disconnect_port
 (up=3D0xc3fd9494, parent=3D0xc3fece80)<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp; at
 /usr/src/sys/dev/usb/usb_subr.c:1380<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#15 0xc06f3e6e in uhub_explore
 (dev=3D0xc3fecc80)<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp; at =
 /usr/src/sys/dev/usb/uhub.c:462<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#16 0xc06fc195 in usb_discover =
 (v=3DVariable
 &quot;v&quot; is not available.<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>) at =
 /usr/src/sys/dev/usb/usb.c:724<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#17 0xc06fd187 in =
 usb_event_thread
 (arg=3D0xc3fd4880)<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp; at
 /usr/src/sys/dev/usb/usb.c:440<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#18 0xc076ca19 in fork_exit
 (callout=3D0xc06fd0d0 &lt;usb_event_thread&gt;,<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp; =
 arg=3D0xc3fd4880,
 frame=3D0xe406cd38) at =
 /usr/src/sys/kern/kern_fork.c:804<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>#19 0xc0a89eb0 in =
 fork_trampoline () at
 /usr/src/sys/i386/i386/exception.s:264<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US><o:p>&nbsp;</o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>(kgdb) print =
 *sim<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>$2 =3D {sim_action =3D =
 0xc046af70
 &lt;dead_sim_action&gt;,<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp; sim_poll =3D 0xc0469b00
 &lt;dead_sim_poll&gt;, sim_name =3D 0xc0af0b2e =
 &quot;dead_sim&quot;,<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp; softc =3D 0x0, mtx =3D =
 0x0, sim_doneq =3D
 {tqh_first =3D 0x0, tqh_last =3D 0x0},<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp; links =3D {tqe_next =3D =
 0x0, tqe_prev =3D
 0x0}, path_id =3D 0, unit_number =3D 0,<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp; bus_id =3D 0, =
 max_tagged_dev_openings
 =3D 0, max_dev_openings =3D 0, flags =3D 0,<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp; callout =3D {c_links =3D =
 {sle =3D
 {sle_next =3D 0x0}, tqe =3D {tqe_next =3D 0x0,<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span =
 lang=3DEN-US>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 tqe_prev =3D 0x0}}, c_time =3D 0, c_arg =3D 0x0, c_func =3D 0, c_mtx =3D =
 0x0,<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp; c_flags =3D =
 0}, devq =3D
 0x0, ccb_freeq =3D {slh_first =3D 0xc4022400},<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp; </span>max_ccbs =3D 0, =
 ccb_count =3D 0}<o:p></o:p></p>
 
 <p class=3DMsoNormal><o:p>&nbsp;</o:p></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>(kgdb) print =
 done_ccb-&gt;ccb_h<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>$5 =3D {pinfo =3D {priority =3D =
 5, generation =3D
 5, index =3D -1}, xpt_links =3D {le =3D {<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; =
 le_next =3D
 0x0, le_prev =3D 0x0}, sle =3D {sle_next =3D 0x0}, tqe =3D =
 {<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; =
 tqe_next =3D
 0x0, tqe_prev =3D 0x0}, stqe =3D {stqe_next =3D 0x0}}, sim_links =3D =
 {<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp; le =3D =
 {le_next =3D 0x0,
 le_prev =3D 0x0}, sle =3D {sle_next =3D 0x0}, tqe =3D =
 {<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; =
 tqe_next =3D
 0x0, tqe_prev =3D 0x0}, stqe =3D {stqe_next =3D =
 0x0}},<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp; periph_links =3D {le =3D =
 {le_next =3D 0x0,
 le_prev =3D 0xc4015c00}, sle =3D {<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; =
 sle_next =3D
 0x0}, tqe =3D {tqe_next =3D 0x0, tqe_prev =3D 0xc4015c00}, stqe =3D =
 {<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; =
 stqe_next =3D
 0x0}}, retry_count =3D 0, cbfcnp =3D 0xc046e8f0 =
 &lt;xpt_scan_bus&gt;,<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp; func_code =3D =
 XPT_SCAN_LUN, status =3D
 1, path =3D 0xc480d290, path_id =3D 0,<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp; target_id =3D 0, =
 target_lun =3D 0, flags
 =3D 0, periph_priv =3D {entries =3D {{<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span =
 lang=3DEN-US>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 ptr =3D 0xc480d2a0, field =3D 3296776864, bytes =3D
 &quot;&nbsp;&Ograve;\200&Auml;&quot;}, {ptr =3D =
 0x0,<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span =
 lang=3DEN-US>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 field =3D 0, bytes =3D &quot;\000\000\000&quot;}}, bytes =3D
 &quot;&nbsp;&Ograve;\200&Auml;\000\000\000&quot;},<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp; sim_priv =3D {entries =3D =
 {{ptr =3D 0x0,
 field =3D 0, bytes =3D &quot;\000\000\000&quot;}, =
 {<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span =
 lang=3DEN-US>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 ptr =3D 0x0, field =3D 0, bytes =3D =
 &quot;\000\000\000&quot;}},<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp; bytes =3D
 &quot;\000\000\000\000\000\000\000&quot;}, timeout =3D 0, timeout_ch =3D =
 {<o:p></o:p></span></p>
 
 <p class=3DMsoNormal><span lang=3DEN-US>&nbsp;&nbsp;&nbsp; =
 </span>callout =3D 0x0}}<o:p></o:p></p>
 
 <p class=3DMsoNormal><o:p>&nbsp;</o:p></p>
 
 <p class=3DMsoNormal>Hope this helps.<o:p></o:p></p>
 
 <p class=3DMsoNormal><o:p>&nbsp;</o:p></p>
 
 </div>
 
 </body>
 
 </html>
 
 ------_=_NextPart_001_01C97A36.87213BC8--


More information about the freebsd-usb mailing list