git: 845f9747380a - main - iavf: Handle queue disable during reset recovery
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Mon, 02 Nov 2026 01:50:56 UTC
The branch main has been updated by kbowling:
URL: https://cgit.FreeBSD.org/src/commit/?id=845f9747380a0db4d8a174794a5e199e80de81c0
commit 845f9747380a0db4d8a174794a5e199e80de81c0
Author: Kevin Bowling <kbowling@FreeBSD.org>
AuthorDate: 2026-11-02 00:39:27 +0000
Commit: Kevin Bowling <kbowling@FreeBSD.org>
CommitDate: 2026-11-02 01:50:51 +0000
iavf: Handle queue disable during reset recovery
Some PFs reject a redundant DISABLE_QUEUES request after queues have
already stopped. The historical driver retried the request but
continued. The mailbox recovery change instead treated this NACK as a
transport failure, tore down the Admin Queue without a VF reset, and
could not recover the interface during API rediscovery.
Track a PF's pre-reset warning separately from an observed reset and
from exclusive ownership of polled AdminQ replies. RESET_IMPENDING
leaves asynchronous AdminQ processing available so a concurrent stop
can consume its DISABLE_QUEUES reply. VC_POLLING protects VERSION and
GET_VF_RESOURCES rediscovery without blocking that reply. Record
completed resets so subsequent stop/init operations do not issue a
redundant queue-disable request.
If queue disable is not confirmed, wait for an observed reset or request
a VF reset and wait for completion. Give queue shutdown a separate
60-second monotonic deadline to accommodate slow PF reset recovery.
Propagate reset submission failures and restore the previous RSTAT
value when no request was submitted. Clear the attach-time recovery
marker before the first runtime stop/init cycle, and suppress RSS
status reporting during reset recovery.
This implements the native recovery portion of D58904, with PCI
containment and permanent failure handling deferred. If native shutdown
cannot be confirmed, report the failure and leave recovery pending.
Shutdown remains best effort because IFDI_STOP cannot return an error
to prevent iflib from releasing queue resources.
Matched hardware testing against the parent on E810 with two four-queue
VFs covers 18 TCP smoke runs, 40 down/up cycles, three PF resets, and
three CORE resets per kernel. The change eliminates all 12 observed
"PF mailbox rediscovery failed: 60" messages and reduces the longest
UDP delivery gap per CORE reset from 3.43-3.57 seconds to 2.62-2.67
seconds.
MFC after: 2 weeks
Sponsored by: BBOX.io
Differential Revision: https://reviews.freebsd.org/D58904
---
sys/dev/iavf/iavf_lib.c | 30 ++++--
sys/dev/iavf/iavf_lib.h | 2 +
sys/dev/iavf/iavf_vc_common.c | 29 ++++--
sys/dev/iavf/if_iavf_iflib.c | 219 +++++++++++++++++++++++++++++++++++++-----
4 files changed, 236 insertions(+), 44 deletions(-)
diff --git a/sys/dev/iavf/iavf_lib.c b/sys/dev/iavf/iavf_lib.c
index 01d14a58f0b9..f4f14f799f7d 100644
--- a/sys/dev/iavf/iavf_lib.c
+++ b/sys/dev/iavf/iavf_lib.c
@@ -315,8 +315,11 @@ iavf_reset(struct iavf_sc *sc)
int error = 0;
/* Ask the PF to reset us if we are initiating */
- if (!iavf_test_state(&sc->state, IAVF_STATE_RESET_PENDING))
- iavf_request_reset(sc);
+ if (!iavf_test_state(&sc->state, IAVF_STATE_RESET_PENDING)) {
+ error = iavf_request_reset(sc);
+ if (error != 0)
+ return (error);
+ }
iavf_msec_pause(100);
error = iavf_reset_complete(hw);
@@ -325,6 +328,12 @@ iavf_reset(struct iavf_sc *sc)
__func__);
return (error);
}
+ /*
+ * The completed reset stopped the old queues. Record that fact so
+ * stop and initialization need not issue DISABLE_QUEUES again.
+ */
+ iavf_set_state(&sc->state, IAVF_STATE_RESET_RECOVERY);
+ atomic_store_rel_32(&sc->queues_enabled, 0);
pci_enable_busmaster(dev);
error = iavf_shutdown_adminq(hw);
@@ -1488,8 +1497,9 @@ iavf_mark_del_vlan_filter(struct iavf_sc *sc, u16 vtag)
* to IAVF_MAX_DIS_Q_RETRY times if the response says that it wasn't
* successful. This is intended to workaround a bug that can appear on the PF.
*
- * @returns zero on success, or an error code if the request could not be sent
- * or acknowledged.
+ * @returns zero on success, EBUSY when the PF replied but did not confirm
+ * that the queues were disabled, or another error if the request could not
+ * be sent or acknowledged.
*/
int
iavf_disable_queues_with_retries(struct iavf_sc *sc)
@@ -1498,9 +1508,9 @@ iavf_disable_queues_with_retries(struct iavf_sc *sc)
int max_attempts = IAVF_MAX_DIS_Q_RETRY;
int error = 0, msg_count = 0;
- /* While the driver is detaching, it doesn't care if the queue
- * disable finishes successfully or not. Just send one message
- * to just notify the PF driver.
+ /*
+ * The detach path cannot wait for a reply. Send one request and let
+ * the caller attempt a VF reset if the queues remain unconfirmed.
*/
if (in_detach)
max_attempts = 1;
@@ -1519,13 +1529,13 @@ iavf_disable_queues_with_retries(struct iavf_sc *sc)
iavf_dbg_vc(sc, "DISABLE_QUEUES messages sent: %d\n",
msg_count);
- if (!in_detach && msg_count >= max_attempts &&
+ if (msg_count >= max_attempts &&
atomic_load_acq_32(&sc->queues_enabled)) {
- if (iavf_mbx_log_allowed(sc))
+ if (!in_detach && iavf_mbx_log_allowed(sc))
device_printf(sc->dev,
"%s: DISABLE_QUEUES may have failed\n", __func__);
if (error == 0)
- error = EIO;
+ error = EBUSY;
}
return (error);
}
diff --git a/sys/dev/iavf/iavf_lib.h b/sys/dev/iavf/iavf_lib.h
index c56e55b555a0..fd707051aef8 100644
--- a/sys/dev/iavf/iavf_lib.h
+++ b/sys/dev/iavf/iavf_lib.h
@@ -313,6 +313,8 @@ enum iavf_state {
IAVF_STATE_INITIALIZED,
IAVF_STATE_RESET_REQUIRED,
IAVF_STATE_RESET_PENDING,
+ IAVF_STATE_RESET_RECOVERY,
+ IAVF_STATE_VC_POLLING,
IAVF_STATE_RUNNING,
/* This entry must be last */
IAVF_STATE_LAST,
diff --git a/sys/dev/iavf/iavf_vc_common.c b/sys/dev/iavf/iavf_vc_common.c
index 2a11f02db59b..94fdb402bc30 100644
--- a/sys/dev/iavf/iavf_vc_common.c
+++ b/sys/dev/iavf/iavf_vc_common.c
@@ -638,19 +638,32 @@ iavf_del_ether_filters(struct iavf_sc *sc)
*
* Request that the PF reset this VF. No response is expected.
*
- * @returns zero
+ * @returns zero on success, or an error code if the request could not be
+ * submitted.
*/
int
iavf_request_reset(struct iavf_sc *sc)
{
+ struct iavf_hw *hw;
+ u32 old_rstat;
+ int error;
+
+ hw = &sc->hw;
+ old_rstat = rd32(hw, IAVF_VFGEN_RSTAT) &
+ IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
/*
- ** Set the reset status to "in progress" before
- ** the request, this avoids any possibility of
- ** a mistaken early detection of completion.
- */
- wr32(&sc->hw, IAVF_VFGEN_RSTAT, VIRTCHNL_VFR_INPROGRESS);
- iavf_send_pf_msg(sc, VIRTCHNL_OP_RESET_VF, NULL, 0);
- return (0);
+ * The VF must publish INPROGRESS before asking the PF to reset it. This
+ * also prevents a stale VFACTIVE value from looking like completion.
+ */
+ wr32(hw, IAVF_VFGEN_RSTAT, VIRTCHNL_VFR_INPROGRESS);
+ iavf_flush(hw);
+ error = iavf_send_pf_msg(sc, VIRTCHNL_OP_RESET_VF, NULL, 0);
+ if (error != 0) {
+ /* No reset was submitted; do not leave a synthetic reset behind. */
+ wr32(hw, IAVF_VFGEN_RSTAT, old_rstat);
+ iavf_flush(hw);
+ }
+ return (error);
}
/**
diff --git a/sys/dev/iavf/if_iavf_iflib.c b/sys/dev/iavf/if_iavf_iflib.c
index e761ac86c9b0..5b077a4bdfac 100644
--- a/sys/dev/iavf/if_iavf_iflib.c
+++ b/sys/dev/iavf/if_iavf_iflib.c
@@ -56,6 +56,7 @@ static const struct timeval iavf_mbx_log_interval = { 60, 0 };
#define IAVF_MBX_RECOVERY_ASQ_RETRIES 10
#define IAVF_MBX_RECOVERY_VERSION_RETRIES 3
#define IAVF_MBX_RECOVERY_CONFIG_RETRIES 10
+#define IAVF_QUEUE_RESET_TIMEOUT (60 * SBT_1S)
/*********************************************************************
* Function prototypes
@@ -92,6 +93,7 @@ static void iavf_if_stop(if_ctx_t ctx);
static bool iavf_if_needs_restart(if_ctx_t, enum iflib_restart_event);
static void iavf_mbx_lost(struct iavf_sc *);
+static void iavf_enter_reset_recovery(struct iavf_sc *);
static void iavf_mbx_retry_detach(struct iavf_sc *);
static void iavf_mbx_retry_failed(if_ctx_t);
static void iavf_mbx_retry_prepare(struct iavf_sc *);
@@ -99,6 +101,8 @@ static void iavf_mbx_retry_stop(struct iavf_sc *);
static void iavf_mbx_retry_succeeded(struct iavf_sc *);
static int iavf_reestablish_vc(struct iavf_sc *);
static void iavf_replay_filters(struct iavf_sc *);
+static void iavf_hold_vc_polling(struct iavf_sc *);
+static int iavf_quiesce_queues(struct iavf_sc *);
static int iavf_wait_asq(struct iavf_sc *, u32);
static int iavf_allocate_pci_resources(struct iavf_sc *);
@@ -502,7 +506,7 @@ iavf_setup_vc_tq(struct iavf_sc *sc)
static int
iavf_if_attach_post(if_ctx_t ctx)
{
-#ifdef IXL_DEBUG
+#ifdef IAVF_DEBUG
device_t dev = iflib_get_dev(ctx);
#endif
struct iavf_sc *sc;
@@ -529,6 +533,12 @@ iavf_if_attach_post(if_ctx_t ctx)
bzero(&sc->vsi.eth_stats, sizeof(struct iavf_eth_stats));
iavf_add_device_sysctls(sc);
+ /*
+ * attach_pre completed the initial reset and resource handshake. Do
+ * not carry the runtime reset-recovery state into iflib's first
+ * stop/init cycle.
+ */
+ iavf_clear_state(&sc->state, IAVF_STATE_RESET_RECOVERY);
atomic_store_rel_32(&sc->queues_enabled, 0);
atomic_store_rel_32(&sc->mbx_ready, 1);
atomic_store_rel_32(&sc->vc_reinit_required, 0);
@@ -822,6 +832,135 @@ iavf_mbx_retry_stop(struct iavf_sc *sc)
sc->mbx_retry_stage = 0;
}
+static void
+iavf_enter_reset_recovery(struct iavf_sc *sc)
+{
+
+ iavf_set_state(&sc->state, IAVF_STATE_RESET_PENDING);
+ iavf_set_state(&sc->state, IAVF_STATE_RESET_RECOVERY);
+}
+
+/*
+ * Record that a reset has stopped the old LAN queues and invalidated the
+ * current virtchnl connection. The next initialization must rediscover the
+ * VF resources before it can rebuild and enable the LAN queues.
+ */
+static void
+iavf_reset_queues_stopped(struct iavf_sc *sc)
+{
+
+ iavf_enter_reset_recovery(sc);
+ atomic_store_rel_32(&sc->queues_enabled, 0);
+ atomic_store_rel_32(&sc->mbx_ready, 0);
+ atomic_store_rel_32(&sc->vc_reinit_required, 1);
+}
+
+/* Give synchronous reset/discovery code exclusive ownership of the AdminQ. */
+static void
+iavf_hold_vc_polling(struct iavf_sc *sc)
+{
+
+ iavf_set_state(&sc->state, IAVF_STATE_VC_POLLING);
+ iavf_disable_adminq_irq(&sc->hw);
+ taskqueue_drain(sc->vc_tq, &sc->vc_task);
+ /* A task already running when interrupts were masked can re-enable it. */
+ iavf_disable_adminq_irq(&sc->hw);
+}
+
+/*
+ * Wait for a reset which has already started. A completed reset proves the
+ * old queues are stopped; a timeout does not establish quiescence.
+ * Allow longer than the retryable attach wait: E830/E825C PFs delay 20 seconds
+ * during EMPR before checking completion and rebuilding VF state.
+ */
+static int
+iavf_wait_for_queue_reset(struct iavf_sc *sc)
+{
+ sbintime_t deadline;
+
+ deadline = sbinuptime() + IAVF_QUEUE_RESET_TIMEOUT;
+ while (!iavf_reset_is_complete(&sc->hw)) {
+ if (sbinuptime() >= deadline)
+ return (EBUSY);
+ iavf_msec_pause(100);
+ }
+ iavf_reset_queues_stopped(sc);
+ return (EAGAIN);
+}
+
+/*
+ * Attempt to quiesce the hardware before queue resources are reused.
+ * Success means the queues are stopped and the current AdminQ is still
+ * usable. EAGAIN means a reset stopped the queues but virtchnl must be
+ * rediscovered. Other errors do not prove that queue DMA has stopped.
+ */
+static int
+iavf_quiesce_queues(struct iavf_sc *sc)
+{
+ struct iavf_hw *hw;
+ bool asq_alive;
+ int error;
+
+ hw = &sc->hw;
+ if (iavf_test_state(&sc->state, IAVF_STATE_RESET_RECOVERY)) {
+ if (!iavf_reset_is_complete(hw)) {
+ iavf_hold_vc_polling(sc);
+ return (iavf_wait_for_queue_reset(sc));
+ }
+ atomic_store_rel_32(&sc->queues_enabled, 0);
+ return (EAGAIN);
+ }
+ if (!iavf_reset_is_complete(hw)) {
+ iavf_enter_reset_recovery(sc);
+ iavf_hold_vc_polling(sc);
+ return (iavf_wait_for_queue_reset(sc));
+ }
+ if (atomic_load_acq_32(&sc->queues_enabled) == 0)
+ return (0);
+
+ asq_alive = iavf_check_asq_alive(hw);
+ if (!asq_alive) {
+ /*
+ * A dead AdminQ does not prove that LAN DMA stopped. Rebuild
+ * the virtchnl connection so queue disable or VF reset remains
+ * available before iflib releases the queue resources.
+ */
+ error = iavf_reestablish_vc(sc);
+ if (error == 0) {
+ /* A PF reset may have discarded the VF's filters. */
+ atomic_store_rel_32(&sc->vc_reinit_required, 1);
+ atomic_store_rel_32(&sc->mbx_ready, 1);
+ asq_alive = true;
+ }
+ }
+ if (atomic_load_acq_32(&sc->mbx_ready) != 0 && asq_alive)
+ error = iavf_disable_queues_with_retries(sc);
+ else if (asq_alive)
+ error = ENXIO;
+ if (atomic_load_acq_32(&sc->queues_enabled) == 0)
+ return (0);
+
+ /* The PF might have started its advertised reset while we waited. */
+ if (!iavf_reset_is_complete(hw)) {
+ iavf_enter_reset_recovery(sc);
+ iavf_hold_vc_polling(sc);
+ return (iavf_wait_for_queue_reset(sc));
+ }
+
+ if (iavf_mbx_log_allowed(sc))
+ device_printf(sc->dev,
+ "Queue disable was not confirmed (%d); requesting a VF reset\n",
+ error);
+ iavf_set_state(&sc->state, IAVF_STATE_RESET_PENDING);
+ iavf_hold_vc_polling(sc);
+ error = iavf_request_reset(sc);
+ if (error != 0)
+ return (error);
+ /* RESET_VF has no reply; allow the PF to begin VFR before polling. */
+ iavf_msec_pause(100);
+ return (iavf_wait_for_queue_reset(sc));
+}
+
static void
iavf_mbx_retry_failed(if_ctx_t ctx)
{
@@ -873,6 +1012,8 @@ iavf_mbx_retry_succeeded(struct iavf_sc *sc)
sc->mbx_last_log.tv_usec = 0;
iavf_clear_state(&sc->state, IAVF_STATE_RESET_REQUIRED);
iavf_clear_state(&sc->state, IAVF_STATE_RESET_PENDING);
+ iavf_clear_state(&sc->state, IAVF_STATE_RESET_RECOVERY);
+ iavf_clear_state(&sc->state, IAVF_STATE_VC_POLLING);
if (recovered)
device_printf(sc->dev, "PF mailbox communication restored\n");
}
@@ -927,10 +1068,9 @@ iavf_reestablish_vc(struct iavf_sc *sc)
hw = &sc->hw;
vsi = &sc->vsi;
- iavf_disable_adminq_irq(hw);
- taskqueue_drain(sc->vc_tq, &sc->vc_task);
- /* A task already running when interrupts were masked can re-enable it. */
- iavf_disable_adminq_irq(hw);
+ /* Keep the ordinary AdminQ task from consuming polled replies. */
+ iavf_set_state(&sc->state, IAVF_STATE_RESET_PENDING);
+ iavf_hold_vc_polling(sc);
pci_enable_busmaster(sc->dev);
status = iavf_shutdown_adminq(hw);
@@ -980,12 +1120,18 @@ iavf_reestablish_vc(struct iavf_sc *sc)
}
/*
- * RESET_PENDING prevents the ordinary AdminQ task from consuming
- * messages while the queue may still belong to the pre-reset device.
- * The successful VERSION and GET_VF_RESOURCES exchange above proves
- * that the reset has completed and this is the replacement AdminQ.
- * Clear the stale indication before normal virtchnl requests resume.
+ * VC_POLLING prevents the ordinary AdminQ task from consuming the
+ * replies above. The successful exchange proves that the reset has
+ * completed and this is the replacement AdminQ.
+ * If a reset stopped the old queues, consume that fact now that the
+ * replacement AdminQ is established. This permits ordinary AdminQ
+ * processing while making the following quiesce check a no-op.
*/
+ if (iavf_test_state(&sc->state, IAVF_STATE_RESET_RECOVERY)) {
+ atomic_store_rel_32(&sc->queues_enabled, 0);
+ iavf_clear_state(&sc->state, IAVF_STATE_RESET_RECOVERY);
+ }
+ iavf_clear_state(&sc->state, IAVF_STATE_VC_POLLING);
iavf_clear_state(&sc->state, IAVF_STATE_RESET_PENDING);
iavf_enable_adminq_irq(hw);
return (0);
@@ -1042,6 +1188,7 @@ iavf_if_init(if_ctx_t ctx)
replay_filters = atomic_load_acq_32(&sc->vc_reinit_required) != 0;
if (!iavf_reset_is_complete(hw)) {
+ iavf_enter_reset_recovery(sc);
atomic_store_rel_32(&sc->vc_reinit_required, 1);
if (iavf_mbx_log_allowed(sc))
device_printf(dev,
@@ -1063,10 +1210,12 @@ iavf_if_init(if_ctx_t ctx)
}
}
- /* Make sure queues are disabled */
- error = iavf_disable_queues_with_retries(sc);
- if (error != 0)
- goto fail;
+ if (!iavf_test_state(&sc->state, IAVF_STATE_RESET_RECOVERY)) {
+ /* Make sure queues are disabled. */
+ error = iavf_quiesce_queues(sc);
+ if (error != 0)
+ goto fail;
+ }
bcopy(if_getlladdr(ifp), tmpaddr, ETHER_ADDR_LEN);
if (!cmp_etheraddr(hw->mac.addr, tmpaddr) &&
@@ -1141,6 +1290,7 @@ iavf_rss_query_status(struct iavf_sc *sc)
if (!iavf_test_state(&sc->state, IAVF_STATE_RUNNING) ||
iavf_test_state(&sc->state, IAVF_STATE_RESET_PENDING) ||
iavf_test_state(&sc->state, IAVF_STATE_RESET_REQUIRED) ||
+ iavf_test_state(&sc->state, IAVF_STATE_RESET_RECOVERY) ||
atomic_load_acq_32(&sc->mbx_ready) == 0 ||
atomic_load_acq_32(&sc->vc_reinit_required) != 0 ||
sc->rss_configured != (IAVF_FLAG_AQ_CONFIG_RSS_KEY |
@@ -1599,9 +1749,10 @@ iavf_process_adminq(struct iavf_sc *sc, u16 *pending)
int error = 0, loop = 0;
u32 reg;
- if (iavf_test_state(&sc->state, IAVF_STATE_RESET_PENDING)) {
- status = IAVF_ERR_ADMIN_QUEUE_ERROR;
- goto reenable_interrupt;
+ if (iavf_test_state(&sc->state, IAVF_STATE_VC_POLLING) ||
+ iavf_test_state(&sc->state, IAVF_STATE_RESET_RECOVERY)) {
+ *pending = 0;
+ return (IAVF_ERR_ADMIN_QUEUE_ERROR);
}
error = iavf_check_aq_errors(sc);
@@ -1783,6 +1934,8 @@ iavf_if_timer(if_ctx_t ctx, uint16_t qid)
{
struct iavf_sc *sc = iavf_sc_from_ctx(ctx);
struct iavf_hw *hw = &sc->hw;
+ bool asq_alive;
+ bool reset_active;
u32 val;
if (qid != 0)
@@ -1791,16 +1944,25 @@ iavf_if_timer(if_ctx_t ctx, uint16_t qid)
/* Check for a PF-triggered VF reset or a dead admin send queue. */
val = rd32(hw, IAVF_VFGEN_RSTAT) &
IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
- if (iavf_test_state(&sc->state, IAVF_STATE_RESET_PENDING) ||
- !iavf_check_asq_alive(hw) ||
- (val != VIRTCHNL_VFR_VFACTIVE &&
- val != VIRTCHNL_VFR_COMPLETED)) {
+ reset_active = val != VIRTCHNL_VFR_VFACTIVE &&
+ val != VIRTCHNL_VFR_COMPLETED;
+ asq_alive = iavf_check_asq_alive(hw);
+ if (reset_active)
+ iavf_enter_reset_recovery(sc);
+ if (!asq_alive || reset_active) {
iavf_dbg_info(sc, "PF mailbox unavailable (reset state %d)\n",
val);
iavf_mbx_lost(sc);
return;
}
+ /*
+ * RESET_IMPENDING is only a warning. Preserve the running queues until
+ * RSTAT proves that the PF started the reset.
+ */
+ if (iavf_test_state(&sc->state, IAVF_STATE_RESET_PENDING))
+ return;
+
/* Fire off the adminq task */
iflib_admin_intr_deferred(ctx);
@@ -2208,6 +2370,7 @@ iavf_stop(struct iavf_sc *sc)
{
struct iavf_vsi *vsi;
bool mailbox_ready;
+ int error;
vsi = &sc->vsi;
iavf_mbx_retry_stop(sc);
@@ -2216,12 +2379,16 @@ iavf_stop(struct iavf_sc *sc)
iavf_disable_intr(vsi);
- mailbox_ready = atomic_load_acq_32(&sc->mbx_ready) != 0;
- if (mailbox_ready && iavf_reset_is_complete(&sc->hw) &&
- iavf_disable_queues_with_retries(sc) != 0)
- mailbox_ready = false;
- atomic_store_rel_32(&sc->mbx_ready, 0);
+ error = iavf_quiesce_queues(sc);
+ /* IFDI_STOP cannot report failure; failed shutdown remains best effort. */
+ if (error != 0 && error != EAGAIN && iavf_mbx_log_allowed(sc))
+ device_printf(sc->dev,
+ "Unable to confirm VF queue shutdown: %d\n", error);
+ mailbox_ready = error == 0 &&
+ atomic_load_acq_32(&sc->mbx_ready) != 0 &&
+ !iavf_test_state(&sc->state, IAVF_STATE_RESET_RECOVERY);
if (!mailbox_ready) {
+ atomic_store_rel_32(&sc->mbx_ready, 0);
atomic_store_rel_32(&sc->vc_reinit_required, 1);
iavf_dbg_vc(sc, "PF mailbox unavailable while stopping\n");
}