Hast locking up under 9.2

[Pete French]

I just tried to apply the pathc, but it didnt apply cleanly _ I
am patching against 9.2 STABLE latest version, the rejected bits
of the patch are below...

***************
*** 1322,1334 ****
  		}
  		pjdlog_debug(2,
  		    "ggate_recv: (%p) Moving request to the send queues.", hio);
- 		if (hio->hio_replication == HAST_REPLICATION_MEMSYNC &&
- 		    ggio->gctl_cmd == BIO_WRITE) {
- 			/* Each remote request needs two responses in memsync. */
- 			refcnt_init(&hio->hio_countdown, ncomps + 1);
- 		} else {
- 			refcnt_init(&hio->hio_countdown, ncomps);
- 		}
  		for (ii = ncomp; ii < ncomps; ii++)
  			QUEUE_INSERT1(hio, send, ii);
  	}
--- 1338,1344 ----
  		}
  		pjdlog_debug(2,
  		    "ggate_recv: (%p) Moving request to the send queues.", hio);
+ 		refcnt_init(&hio->hio_countdown, ncomps);
  		for (ii = ncomp; ii < ncomps; ii++)
  			QUEUE_INSERT1(hio, send, ii);
  	}
***************
*** 1808,1889 ****
  		hio->hio_errors[ncomp] = 0;
  		nv_free(nv);
  done_queue:
- 		if (hio->hio_replication != HAST_REPLICATION_MEMSYNC ||
- 		    hio->hio_ggio.gctl_cmd != BIO_WRITE || ISSYNCREQ(hio)) {
- 			if (refcnt_release(&hio->hio_countdown) > 0)
- 				continue;
- 		} else {
- 			/*
- 			 * Depending on hio_countdown value, requests finished
- 			 * in the following order:
- 			 *
- 			 * 0: local write, remote memsync, remote final
- 			 * or
- 			 * 0: remote memsync, local write, remote final
- 			 *
- 			 * 1: local write, remote memsync, (remote final)
- 			 * or
- 			 * 1: remote memsync, remote final, (local write)
- 			 *
- 			 * 2: remote memsync, (local write), (remote final)
- 			 * or
- 			 * 2: remote memsync, (remote final), (local write)
- 			 */
- 			switch (refcnt_release(&hio->hio_countdown)) {
- 			case 0:
- 				/*
- 				 * Remote final reply arrived.
- 				 */
- 				PJDLOG_ASSERT(!memsyncack);
- 				break;
- 			case 1:
- 				if (memsyncack) {
- 					/*
- 					 * Local request already finished, so we
- 					 * can complete the write.
- 					 */
  					if (hio->hio_errors[0] == 0)
  						write_complete(res, hio);
- 					/*
- 					 * We still need to wait for final
- 					 * remote reply.
- 					 */
  					pjdlog_debug(2,
- 					    "remote_recv: (%p) Moving request back to the recv queue.",
- 					    hio);
  					mtx_lock(&hio_recv_list_lock[ncomp]);
  					TAILQ_INSERT_TAIL(&hio_recv_list[ncomp],
  					    hio, hio_next[ncomp]);
  					hio_recv_list_size[ncomp]++;
  					mtx_unlock(&hio_recv_list_lock[ncomp]);
- 				} else {
- 					/*
- 					 * Remote final reply arrived before
- 					 * local write finished.
- 					 * Nothing to do in such case.
- 					 */
  				}
- 				continue;
- 			case 2:
- 				/*
- 				 * We received remote memsync reply even before
- 				 * local write finished.
- 				 */
- 				PJDLOG_ASSERT(memsyncack);
- 
- 				pjdlog_debug(2,
- 				    "remote_recv: (%p) Moving request back to the recv queue.",
- 				    hio);
- 				mtx_lock(&hio_recv_list_lock[ncomp]);
- 				TAILQ_INSERT_TAIL(&hio_recv_list[ncomp], hio,
- 				    hio_next[ncomp]);
- 				hio_recv_list_size[ncomp]++;
- 				mtx_unlock(&hio_recv_list_lock[ncomp]);
- 				continue;
- 			default:
- 				PJDLOG_ABORT("Invalid hio_countdown.");
  			}
  		}
  		if (ISSYNCREQ(hio)) {
  			mtx_lock(&sync_lock);
  			SYNCREQDONE(hio);
--- 1792,1825 ----
  		hio->hio_errors[ncomp] = 0;
  		nv_free(nv);
  done_queue:
+ 		if (ISMEMSYNCWRITE(hio)) {
+ 			if (!hio->hio_memsyncacked) {
+ 				PJDLOG_ASSERT(memsyncack ||
+ 				    hio->hio_errors[ncomp] != 0);
+ 				/* Remote ack arrived. */
+ 				if (refcnt_release(&hio->hio_writecount) == 0) {
  					if (hio->hio_errors[0] == 0)
  						write_complete(res, hio);
+ 				}
+ 				hio->hio_memsyncacked = true;
+ 				if (hio->hio_errors[ncomp] == 0) {
  					pjdlog_debug(2,
+ 					    "remote_recv: (%p) Moving request "
+ 					    "back to the recv queue.", hio);
  					mtx_lock(&hio_recv_list_lock[ncomp]);
  					TAILQ_INSERT_TAIL(&hio_recv_list[ncomp],
  					    hio, hio_next[ncomp]);
  					hio_recv_list_size[ncomp]++;
  					mtx_unlock(&hio_recv_list_lock[ncomp]);
+ 					continue;
  				}
+ 			} else {
+ 				PJDLOG_ASSERT(!memsyncack);
+ 				/* Remote final reply arrived. */
  			}
  		}
+ 		if (refcnt_release(&hio->hio_countdown) > 0)
+ 			continue;
  		if (ISSYNCREQ(hio)) {
  			mtx_lock(&sync_lock);
  			SYNCREQDONE(hio);