svn commit: r223651 - in stable/8/sbin: hastctl hastd

Mikolaj Golub trociny at FreeBSD.org
Tue Jun 28 19:27:34 UTC 2011


Author: trociny
Date: Tue Jun 28 19:27:34 2011
New Revision: 223651
URL: http://svn.freebsd.org/changeset/base/223651

Log:
  MFC r222164, r222228, r222467, r223181:
  
  r222164 (pjd):
  
  Recognize HIO_FLUSH requests.
  
  r222228 (pjd):
  
  Keep statistics on number of BIO_READ, BIO_WRITE, BIO_DELETE and BIO_FLUSH
  requests as well as number of activemap updates.
  
  Number of BIO_WRITEs and activemap updates are especially interesting, because
  if those two are too close to each other, it means that your workload needs
  bigger number of dirty extents. Activemap should be updated as rarely as
  possible.
  
  r222467:
  
  If READ from the local node failed we send the request to the remote
  node. There is no use in doing this for synchronization requests.
  
  r223181:
  
  In HAST we use two sockets - one for only sending the data and one for
  only receiving the data. In r220271 the unused directions were
  disabled using shutdown(2).
  
  Unfortunately, this broke automatic receive buffer sizing, which
  currently works only for connections in ETASBLISHED state. It was a
  root cause of the issue reported by users, when connection between
  primary and secondary could get stuck.
  
  Disable the code introduced in r220271 until the issue with automatic
  buffer sizing is not resolved.
  
  Reported by:	Daniel Kalchev <daniel at digsys.bg>, danger, sobomax
  Tested by:	Daniel Kalchev <daniel at digsys.bg>, danger
  
  Approved by:	pjd (mentor)

Modified:
  stable/8/sbin/hastctl/hastctl.c
  stable/8/sbin/hastd/control.c
  stable/8/sbin/hastd/hast.h
  stable/8/sbin/hastd/primary.c
  stable/8/sbin/hastd/secondary.c
Directory Properties:
  stable/8/sbin/hastctl/   (props changed)
  stable/8/sbin/hastd/   (props changed)

Modified: stable/8/sbin/hastctl/hastctl.c
==============================================================================
--- stable/8/sbin/hastctl/hastctl.c	Tue Jun 28 17:51:47 2011	(r223650)
+++ stable/8/sbin/hastctl/hastctl.c	Tue Jun 28 19:27:34 2011	(r223651)
@@ -341,6 +341,17 @@ control_status(struct nv *nv)
 		printf("  dirty: %ju (%NB)\n",
 		    (uintmax_t)nv_get_uint64(nv, "dirty%u", ii),
 		    (intmax_t)nv_get_uint64(nv, "dirty%u", ii));
+		printf("  statistics:\n");
+		printf("    reads: %ju\n",
+		    (uint64_t)nv_get_uint64(nv, "stat_read%u", ii));
+		printf("    writes: %ju\n",
+		    (uint64_t)nv_get_uint64(nv, "stat_write%u", ii));
+		printf("    deletes: %ju\n",
+		    (uint64_t)nv_get_uint64(nv, "stat_delete%u", ii));
+		printf("    flushes: %ju\n",
+		    (uint64_t)nv_get_uint64(nv, "stat_flush%u", ii));
+		printf("    activemap updates: %ju\n",
+		    (uint64_t)nv_get_uint64(nv, "stat_activemap_update%u", ii));
 	}
 	return (ret);
 }

Modified: stable/8/sbin/hastd/control.c
==============================================================================
--- stable/8/sbin/hastd/control.c	Tue Jun 28 17:51:47 2011	(r223650)
+++ stable/8/sbin/hastd/control.c	Tue Jun 28 19:27:34 2011	(r223651)
@@ -199,6 +199,16 @@ control_status_worker(struct hast_resour
 	    "extentsize%u", no);
 	nv_add_uint32(nvout, nv_get_uint32(cnvin, "keepdirty"),
 	    "keepdirty%u", no);
+	nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_read"),
+	    "stat_read%u", no);
+	nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_write"),
+	    "stat_write%u", no);
+	nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_delete"),
+	    "stat_delete%u", no);
+	nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_flush"),
+	    "stat_flush%u", no);
+	nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_activemap_update"),
+	    "stat_activemap_update%u", no);
 end:
 	if (cnvin != NULL)
 		nv_free(cnvin);
@@ -446,6 +456,13 @@ ctrl_thread(void *arg)
 				nv_add_uint32(nvout, (uint32_t)0, "keepdirty");
 				nv_add_uint64(nvout, (uint64_t)0, "dirty");
 			}
+			nv_add_uint64(nvout, res->hr_stat_read, "stat_read");
+			nv_add_uint64(nvout, res->hr_stat_write, "stat_write");
+			nv_add_uint64(nvout, res->hr_stat_delete,
+			    "stat_delete");
+			nv_add_uint64(nvout, res->hr_stat_flush, "stat_flush");
+			nv_add_uint64(nvout, res->hr_stat_activemap_update,
+			    "stat_activemap_update");
 			nv_add_int16(nvout, 0, "error");
 			break;
 		case CONTROL_RELOAD:

Modified: stable/8/sbin/hastd/hast.h
==============================================================================
--- stable/8/sbin/hastd/hast.h	Tue Jun 28 17:51:47 2011	(r223650)
+++ stable/8/sbin/hastd/hast.h	Tue Jun 28 19:27:34 2011	(r223651)
@@ -218,6 +218,17 @@ struct hast_resource {
 	/* Locked used to synchronize access to hr_amp. */
 	pthread_mutex_t hr_amp_lock;
 
+	/* Number of BIO_READ requests. */
+	uint64_t	hr_stat_read;
+	/* Number of BIO_WRITE requests. */
+	uint64_t	hr_stat_write;
+	/* Number of BIO_DELETE requests. */
+	uint64_t	hr_stat_delete;
+	/* Number of BIO_FLUSH requests. */
+	uint64_t	hr_stat_flush;
+	/* Number of activemap updates. */
+	uint64_t	hr_stat_activemap_update;
+
 	/* Next resource. */
 	TAILQ_ENTRY(hast_resource) hr_next;
 };

Modified: stable/8/sbin/hastd/primary.c
==============================================================================
--- stable/8/sbin/hastd/primary.c	Tue Jun 28 17:51:47 2011	(r223650)
+++ stable/8/sbin/hastd/primary.c	Tue Jun 28 19:27:34 2011	(r223651)
@@ -726,11 +726,13 @@ init_remote(struct hast_resource *res, s
 		(void)hast_activemap_flush(res);
 	}
 	nv_free(nvin);
+#ifdef notyet
 	/* Setup directions. */
 	if (proto_send(out, NULL, 0) == -1)
 		pjdlog_errno(LOG_WARNING, "Unable to set connection direction");
 	if (proto_recv(in, NULL, 0) == -1)
 		pjdlog_errno(LOG_WARNING, "Unable to set connection direction");
+#endif
 	pjdlog_info("Connected to %s.", res->hr_remoteaddr);
 	if (inp != NULL && outp != NULL) {
 		*inp = in;
@@ -1117,6 +1119,7 @@ ggate_recv_thread(void *arg)
 		 */
 		switch (ggio->gctl_cmd) {
 		case BIO_READ:
+			res->hr_stat_read++;
 			pjdlog_debug(2,
 			    "ggate_recv: (%p) Moving request to the send queue.",
 			    hio);
@@ -1145,6 +1148,7 @@ ggate_recv_thread(void *arg)
 			QUEUE_INSERT1(hio, send, ncomp);
 			break;
 		case BIO_WRITE:
+			res->hr_stat_write++;
 			if (res->hr_resuid == 0) {
 				/*
 				 * This is first write, initialize localcnt and
@@ -1183,12 +1187,21 @@ ggate_recv_thread(void *arg)
 			mtx_lock(&res->hr_amp_lock);
 			if (activemap_write_start(res->hr_amp,
 			    ggio->gctl_offset, ggio->gctl_length)) {
+				res->hr_stat_activemap_update++;
 				(void)hast_activemap_flush(res);
 			}
 			mtx_unlock(&res->hr_amp_lock);
 			/* FALLTHROUGH */
 		case BIO_DELETE:
 		case BIO_FLUSH:
+			switch (ggio->gctl_cmd) {
+			case BIO_DELETE:
+				res->hr_stat_delete++;
+				break;
+			case BIO_FLUSH:
+				res->hr_stat_flush++;
+				break;
+			}
 			pjdlog_debug(2,
 			    "ggate_recv: (%p) Moving request to the send queues.",
 			    hio);
@@ -1232,7 +1245,7 @@ local_send_thread(void *arg)
 			    ggio->gctl_offset + res->hr_localoff);
 			if (ret == ggio->gctl_length)
 				hio->hio_errors[ncomp] = 0;
-			else {
+			else if (!ISSYNCREQ(hio)) {
 				/*
 				 * If READ failed, try to read from remote node.
 				 */

Modified: stable/8/sbin/hastd/secondary.c
==============================================================================
--- stable/8/sbin/hastd/secondary.c	Tue Jun 28 17:51:47 2011	(r223650)
+++ stable/8/sbin/hastd/secondary.c	Tue Jun 28 19:27:34 2011	(r223651)
@@ -183,9 +183,11 @@ init_remote(struct hast_resource *res, s
 	unsigned char *map;
 	size_t mapsize;
 
+#ifdef notyet
 	/* Setup direction. */
 	if (proto_send(res->hr_remoteout, NULL, 0) == -1)
 		pjdlog_errno(LOG_WARNING, "Unable to set connection direction");
+#endif
 
 	map = NULL;
 	mapsize = 0;
@@ -351,9 +353,11 @@ init_remote(struct hast_resource *res, s
 	if (map != NULL)
 		free(map);
 	nv_free(nvout);
+#ifdef notyet
 	/* Setup direction. */
 	if (proto_recv(res->hr_remotein, NULL, 0) == -1)
 		pjdlog_errno(LOG_WARNING, "Unable to set connection direction");
+#endif
 	if (res->hr_secondary_localcnt > res->hr_primary_remotecnt &&
 	     res->hr_primary_localcnt > res->hr_secondary_remotecnt) {
 		/* Exit on split-brain. */
@@ -514,6 +518,7 @@ requnpack(struct hast_resource *res, str
 		goto end;
 	}
 	switch (hio->hio_cmd) {
+	case HIO_FLUSH:
 	case HIO_KEEPALIVE:
 		break;
 	case HIO_READ:
@@ -611,6 +616,20 @@ recv_thread(void *arg)
 			QUEUE_INSERT(send, hio);
 			continue;
 		}
+		switch (hio->hio_cmd) {
+		case HIO_READ:
+			res->hr_stat_read++;
+			break;
+		case HIO_WRITE:
+			res->hr_stat_write++;
+			break;
+		case HIO_DELETE:
+			res->hr_stat_delete++;
+			break;
+		case HIO_FLUSH:
+			res->hr_stat_flush++;
+			break;
+		}
 		reqlog(LOG_DEBUG, 2, -1, hio,
 		    "recv: (%p) Got request header: ", hio);
 		if (hio->hio_cmd == HIO_KEEPALIVE) {


More information about the svn-src-all mailing list