svn commit: r298504 - head/usr.bin/mkuzip
Maxim Sobolev
sobomax at FreeBSD.org
Sat Apr 23 07:23:45 UTC 2016
Author: sobomax
Date: Sat Apr 23 07:23:43 2016
New Revision: 298504
URL: https://svnweb.freebsd.org/changeset/base/298504
Log:
Improve performance in a few key areas:
o Split the compression across several worker threads. By default, "several"
matches number of CPUs, capped at 24 for sanity when running on a very big
hardwares. Provide option to set that number manually;
o Fix bug inherited from the mkulzma (R.I.P) which degraded already slow LZMA
compression even further by calling function to release compression state
after processing each block.
It is neither documented as required nor actually required by the LZMA
library. This caused spree of system calls to release memory and then map
it again for every block. LZMA compression is more than 2x faster after this
change alone;
o Record time it takes to do compression and report throughput achieved.
o Add simple first-level 256 entry hash table for de-dup code, so it's not
becoming a bottleneck at big files.
Added:
head/usr.bin/mkuzip/mkuz_blk.c (contents, props changed)
head/usr.bin/mkuzip/mkuz_blk.h (contents, props changed)
head/usr.bin/mkuzip/mkuz_blk_chain.h (contents, props changed)
head/usr.bin/mkuzip/mkuz_cfg.h (contents, props changed)
head/usr.bin/mkuzip/mkuz_conveyor.c (contents, props changed)
head/usr.bin/mkuzip/mkuz_conveyor.h (contents, props changed)
head/usr.bin/mkuzip/mkuz_format.h (contents, props changed)
head/usr.bin/mkuzip/mkuz_fqueue.c (contents, props changed)
head/usr.bin/mkuzip/mkuz_fqueue.h (contents, props changed)
head/usr.bin/mkuzip/mkuz_time.c (contents, props changed)
head/usr.bin/mkuzip/mkuz_time.h (contents, props changed)
Modified:
head/usr.bin/mkuzip/Makefile
head/usr.bin/mkuzip/mkuz_blockcache.c
head/usr.bin/mkuzip/mkuz_blockcache.h
head/usr.bin/mkuzip/mkuz_lzma.c
head/usr.bin/mkuzip/mkuz_lzma.h
head/usr.bin/mkuzip/mkuz_zlib.c
head/usr.bin/mkuzip/mkuz_zlib.h
head/usr.bin/mkuzip/mkuzip.8
head/usr.bin/mkuzip/mkuzip.c
head/usr.bin/mkuzip/mkuzip.h
Modified: head/usr.bin/mkuzip/Makefile
==============================================================================
--- head/usr.bin/mkuzip/Makefile Sat Apr 23 07:09:23 2016 (r298503)
+++ head/usr.bin/mkuzip/Makefile Sat Apr 23 07:23:43 2016 (r298504)
@@ -2,8 +2,11 @@
PROG= mkuzip
MAN= mkuzip.8
-SRCS= mkuzip.c mkuz_blockcache.c mkuz_lzma.c mkuz_zlib.c
+SRCS= mkuzip.c mkuz_blockcache.c mkuz_lzma.c mkuz_zlib.c mkuz_conveyor.c \
+ mkuz_blk.c mkuz_fqueue.c mkuz_time.c
-LIBADD= z md lzma
+#CFLAGS+= -DMKUZ_DEBUG
+
+LIBADD= z md lzma pthread
.include <bsd.prog.mk>
Added: head/usr.bin/mkuzip/mkuz_blk.c
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/usr.bin/mkuzip/mkuz_blk.c Sat Apr 23 07:23:43 2016 (r298504)
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2004-2016 Maxim Sobolev <sobomax at FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#include "mkuzip.h"
+#include "mkuz_blk.h"
+
+struct mkuz_blk *
+mkuz_blk_ctor(size_t blen)
+{
+ struct mkuz_blk *rval;
+
+ rval = mkuz_safe_zmalloc(sizeof(struct mkuz_blk) + blen);
+ rval->alen = blen;
+ rval->br_offset = OFFSET_UNDEF;
+ return (rval);
+}
Added: head/usr.bin/mkuzip/mkuz_blk.h
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/usr.bin/mkuzip/mkuz_blk.h Sat Apr 23 07:23:43 2016 (r298504)
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2004-2016 Maxim Sobolev <sobomax at FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#define OFFSET_UNDEF UINT64_MAX
+
+struct mkuz_blk_info {
+ uint64_t offset;
+ size_t len;
+ uint32_t blkno;
+ unsigned char digest[16];
+};
+
+#define MKUZ_BLK_EOF (void *)0x1
+#define MKUZ_BLK_MORE (void *)0x2
+
+struct mkuz_blk {
+ struct mkuz_blk_info info;
+ size_t alen;
+ uint64_t br_offset;
+ unsigned char data[];
+};
+
+struct mkuz_blk *mkuz_blk_ctor(size_t);
Added: head/usr.bin/mkuzip/mkuz_blk_chain.h
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/usr.bin/mkuzip/mkuz_blk_chain.h Sat Apr 23 07:23:43 2016 (r298504)
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2016 Maxim Sobolev <sobomax at FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+struct mkuz_blk;
+struct mkuz_bchain_link;
+
+struct mkuz_bchain_link {
+ struct mkuz_blk *this;
+ struct mkuz_bchain_link *prev;
+};
Modified: head/usr.bin/mkuzip/mkuz_blockcache.c
==============================================================================
--- head/usr.bin/mkuzip/mkuz_blockcache.c Sat Apr 23 07:09:23 2016 (r298503)
+++ head/usr.bin/mkuzip/mkuz_blockcache.c Sat Apr 23 07:23:43 2016 (r298504)
@@ -22,97 +22,107 @@
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
- *
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
-#include <sys/types.h>
#include <err.h>
-#include <md5.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#if defined(MKUZ_DEBUG)
+# include <assert.h>
# include <stdio.h>
#endif
#include "mkuz_blockcache.h"
+#include "mkuz_blk.h"
-struct mkuz_blkcache {
- struct mkuz_blkcache_hit hit;
- off_t data_offset;
- unsigned char digest[16];
- struct mkuz_blkcache *next;
+struct mkuz_blkcache_itm {
+ struct mkuz_blk_info hit;
+ struct mkuz_blkcache_itm *next;
};
-static struct mkuz_blkcache blkcache;
+static struct mkuz_blkcache {
+ struct mkuz_blkcache_itm first[256];
+} blkcache;
static int
-verify_match(int fd, off_t data_offset, void *data, ssize_t len,
- struct mkuz_blkcache *bcep)
+verify_match(int fd, const struct mkuz_blk *cbp, struct mkuz_blkcache_itm *bcep)
{
void *vbuf;
ssize_t rlen;
int rval;
rval = -1;
- vbuf = malloc(len);
+ vbuf = malloc(cbp->info.len);
if (vbuf == NULL) {
goto e0;
}
- if (lseek(fd, bcep->data_offset, SEEK_SET) < 0) {
+ if (lseek(fd, bcep->hit.offset, SEEK_SET) < 0) {
goto e1;
}
- rlen = read(fd, vbuf, len);
- if (rlen != len) {
+ rlen = read(fd, vbuf, cbp->info.len);
+ if (rlen < 0 || (unsigned)rlen != cbp->info.len) {
goto e2;
}
- rval = (memcmp(data, vbuf, len) == 0) ? 1 : 0;
+ rval = (memcmp(cbp->data, vbuf, cbp->info.len) == 0) ? 1 : 0;
e2:
- lseek(fd, data_offset, SEEK_SET);
+ lseek(fd, cbp->info.offset, SEEK_SET);
e1:
free(vbuf);
e0:
return (rval);
}
-struct mkuz_blkcache_hit *
-mkuz_blkcache_regblock(int fd, uint32_t blkno, off_t offset, ssize_t len,
- void *data)
+#define I2J(x) ((intmax_t)(x))
+#define U2J(x) ((uintmax_t)(x))
+
+static unsigned char
+digest_fold(const unsigned char *mdigest)
{
- struct mkuz_blkcache *bcep;
- MD5_CTX mcontext;
- off_t data_offset;
- unsigned char mdigest[16];
- int rval;
+ int i;
+ unsigned char rval;
- data_offset = lseek(fd, 0, SEEK_CUR);
- if (data_offset < 0) {
- return (NULL);
+ rval = mdigest[0];
+ for (i = 1; i < 16; i++) {
+ rval = rval ^ mdigest[i];
}
- MD5Init(&mcontext);
- MD5Update(&mcontext, data, len);
- MD5Final(mdigest, &mcontext);
- if (blkcache.hit.len == 0) {
- bcep = &blkcache;
+ return (rval);
+}
+
+struct mkuz_blk_info *
+mkuz_blkcache_regblock(int fd, const struct mkuz_blk *bp)
+{
+ struct mkuz_blkcache_itm *bcep;
+ int rval;
+ unsigned char h;
+
+#if defined(MKUZ_DEBUG)
+ assert((unsigned)lseek(fd, 0, SEEK_CUR) == bp->info.offset);
+#endif
+ h = digest_fold(bp->info.digest);
+ if (blkcache.first[h].hit.len == 0) {
+ bcep = &blkcache.first[h];
} else {
- for (bcep = &blkcache; bcep != NULL; bcep = bcep->next) {
- if (bcep->hit.len != len)
+ for (bcep = &blkcache.first[h]; bcep != NULL; bcep = bcep->next) {
+ if (bcep->hit.len != bp->info.len)
continue;
- if (memcmp(mdigest, bcep->digest, sizeof(mdigest)) == 0) {
+ if (memcmp(bp->info.digest, bcep->hit.digest,
+ sizeof(bp->info.digest)) == 0) {
break;
}
}
if (bcep != NULL) {
- rval = verify_match(fd, data_offset, data, len, bcep);
+ rval = verify_match(fd, bp, bcep);
if (rval == 1) {
#if defined(MKUZ_DEBUG)
- fprintf(stderr, "cache hit %d, %d, %d\n",
- (int)bcep->hit.offset, (int)data_offset, (int)len);
+ fprintf(stderr, "cache hit %jd, %jd, %jd, %jd\n",
+ I2J(bcep->hit.blkno), I2J(bcep->hit.offset),
+ I2J(bp->info.offset), I2J(bp->info.len));
#endif
return (&bcep->hit);
}
@@ -126,17 +136,13 @@ mkuz_blkcache_regblock(int fd, uint32_t
warn("verify_match");
return (NULL);
}
- bcep = malloc(sizeof(struct mkuz_blkcache));
+ bcep = malloc(sizeof(struct mkuz_blkcache_itm));
if (bcep == NULL)
return (NULL);
- memset(bcep, '\0', sizeof(struct mkuz_blkcache));
- bcep->next = blkcache.next;
- blkcache.next = bcep;
+ memset(bcep, '\0', sizeof(struct mkuz_blkcache_itm));
+ bcep->next = blkcache.first[h].next;
+ blkcache.first[h].next = bcep;
}
- memcpy(bcep->digest, mdigest, sizeof(mdigest));
- bcep->data_offset = data_offset;
- bcep->hit.offset = offset;
- bcep->hit.len = len;
- bcep->hit.blkno = blkno;
+ bcep->hit = bp->info;
return (NULL);
}
Modified: head/usr.bin/mkuzip/mkuz_blockcache.h
==============================================================================
--- head/usr.bin/mkuzip/mkuz_blockcache.h Sat Apr 23 07:09:23 2016 (r298503)
+++ head/usr.bin/mkuzip/mkuz_blockcache.h Sat Apr 23 07:23:43 2016 (r298504)
@@ -26,11 +26,6 @@
* $FreeBSD$
*/
-struct mkuz_blkcache_hit {
- uint64_t offset;
- ssize_t len;
- uint32_t blkno;
-};
+struct mkuz_blk;
-struct mkuz_blkcache_hit *mkuz_blkcache_regblock(int, uint32_t, off_t, ssize_t,
- void *);
+struct mkuz_blk_info *mkuz_blkcache_regblock(int, const struct mkuz_blk *);
Added: head/usr.bin/mkuzip/mkuz_cfg.h
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/usr.bin/mkuzip/mkuz_cfg.h Sat Apr 23 07:23:43 2016 (r298504)
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2016 Maxim Sobolev <sobomax at FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+struct mkuz_conveyor;
+
+struct mkuz_cfg {
+ int fdr;
+ int fdw;
+ int verbose;
+ int no_zcomp;
+ int en_dedup;
+ int nworkers;
+ int blksz;
+ const struct mkuz_format *handler;
+};
Added: head/usr.bin/mkuzip/mkuz_conveyor.c
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/usr.bin/mkuzip/mkuz_conveyor.c Sat Apr 23 07:23:43 2016 (r298504)
@@ -0,0 +1,129 @@
+/*
+ * Copyright (c) 2004-2016 Maxim Sobolev <sobomax at FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <err.h>
+#include <inttypes.h>
+#include <md5.h>
+#include <pthread.h>
+#include <stdlib.h>
+#include <strings.h>
+
+#if defined(MKUZ_DEBUG)
+# include <stdio.h>
+#endif
+
+#include "mkuz_conveyor.h"
+#include "mkuz_cfg.h"
+#include "mkuzip.h"
+#include "mkuz_format.h"
+#include "mkuz_blk.h"
+#include "mkuz_fqueue.h"
+#include "mkuz_blk_chain.h"
+
+static void compute_digest(struct mkuz_blk *);
+
+struct cw_args {
+ struct mkuz_conveyor *cvp;
+ struct mkuz_cfg *cfp;
+};
+
+static void *
+cworker(void *p)
+{
+ struct cw_args *cwp;
+ struct mkuz_cfg *cfp;
+ struct mkuz_blk *oblk, *iblk;
+ struct mkuz_conveyor *cvp;
+ void *c_ctx;
+
+ cwp = (struct cw_args *)p;
+ cfp = cwp->cfp;
+ cvp = cwp->cvp;
+ free(cwp);
+ c_ctx = cfp->handler->f_init(cfp->blksz);
+ for (;;) {
+ iblk = mkuz_fqueue_deq(cvp->wrk_queue);
+ if (iblk == MKUZ_BLK_EOF) {
+ /* Let other threads to see the EOF block */
+ mkuz_fqueue_enq(cvp->wrk_queue, iblk);
+ break;
+ }
+ if (cfp->no_zcomp == 0 &&
+ mkuz_memvcmp(iblk->data, '\0', iblk->info.len) != 0) {
+ /* All zeroes block */
+ oblk = mkuz_blk_ctor(0);
+ } else {
+ oblk = cfp->handler->f_compress(c_ctx, iblk);
+ if (cfp->en_dedup != 0) {
+ compute_digest(oblk);
+ }
+ }
+ oblk->info.blkno = iblk->info.blkno;
+ mkuz_fqueue_enq(cvp->results, oblk);
+ free(iblk);
+ }
+ return (NULL);
+}
+
+static void
+compute_digest(struct mkuz_blk *bp)
+{
+ MD5_CTX mcontext;
+
+ MD5Init(&mcontext);
+ MD5Update(&mcontext, bp->data, bp->info.len);
+ MD5Final(bp->info.digest, &mcontext);
+}
+
+struct mkuz_conveyor *
+mkuz_conveyor_ctor(struct mkuz_cfg *cfp)
+{
+ struct mkuz_conveyor *cp;
+ struct cw_args *cwp;
+ int i, r;
+
+ cp = mkuz_safe_zmalloc(sizeof(struct mkuz_conveyor) +
+ (sizeof(pthread_t) * cfp->nworkers));
+
+ cp->wrk_queue = mkuz_fqueue_ctor(1);
+ cp->results = mkuz_fqueue_ctor(1);
+
+ for (i = 0; i < cfp->nworkers; i++) {
+ cwp = mkuz_safe_zmalloc(sizeof(struct cw_args));
+ cwp->cfp = cfp;
+ cwp->cvp = cp;
+ r = pthread_create(&cp->wthreads[i], NULL, cworker, (void *)cwp);
+ if (r != 0) {
+ errx(1, "mkuz_conveyor_ctor: pthread_create() failed");
+ /* Not reached */
+ }
+ }
+ return (cp);
+}
Added: head/usr.bin/mkuzip/mkuz_conveyor.h
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/usr.bin/mkuzip/mkuz_conveyor.h Sat Apr 23 07:23:43 2016 (r298504)
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2016 Maxim Sobolev <sobomax at FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+struct mkuz_fifo_queue;
+
+#define ITEMS_PER_WORKER 4
+
+#define MAX_WORKERS_AUTO 24
+
+struct mkuz_conveyor {
+ /*
+ * Work items are places in here, and picked up by workers in a FIFO
+ * fashion.
+ */
+ struct mkuz_fifo_queue *wrk_queue;
+ /*
+ * Results are dropped into this FIFO and consumer is buzzed to pick them
+ * up
+ */
+ struct mkuz_fifo_queue *results;
+
+ pthread_t wthreads[];
+};
+
+struct mkuz_cfg;
+
+struct mkuz_conveyor *mkuz_conveyor_ctor(struct mkuz_cfg *);
Added: head/usr.bin/mkuzip/mkuz_format.h
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/usr.bin/mkuzip/mkuz_format.h Sat Apr 23 07:23:43 2016 (r298504)
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2016 Maxim Sobolev <sobomax at FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#define DEFINE_RAW_METHOD(func, rval, args...) typedef rval (*func##_t)(args)
+
+DEFINE_RAW_METHOD(f_init, void *, uint32_t);
+DEFINE_RAW_METHOD(f_compress, struct mkuz_blk *, void *, const struct mkuz_blk *);
+
+struct mkuz_format {
+ const char *magic;
+ const char *default_sufx;
+ f_init_t f_init;
+ f_compress_t f_compress;
+};
Added: head/usr.bin/mkuzip/mkuz_fqueue.c
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/usr.bin/mkuzip/mkuz_fqueue.c Sat Apr 23 07:23:43 2016 (r298504)
@@ -0,0 +1,214 @@
+/*
+ * Copyright (c) 2004-2016 Maxim Sobolev <sobomax at FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <err.h>
+#include <pthread.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#if defined(MKUZ_DEBUG)
+# include <assert.h>
+#endif
+
+#include "mkuzip.h"
+#include "mkuz_fqueue.h"
+#include "mkuz_conveyor.h"
+#include "mkuz_blk.h"
+#include "mkuz_blk_chain.h"
+
+struct mkuz_fifo_queue *
+mkuz_fqueue_ctor(int wakeup_len)
+{
+ struct mkuz_fifo_queue *fqp;
+
+ fqp = mkuz_safe_zmalloc(sizeof(struct mkuz_fifo_queue));
+ fqp->wakeup_len = wakeup_len;
+ if (pthread_mutex_init(&fqp->mtx, NULL) != 0) {
+ errx(1, "pthread_mutex_init() failed");
+ }
+ if (pthread_cond_init(&fqp->cvar, NULL) != 0) {
+ errx(1, "pthread_cond_init() failed");
+ }
+ return (fqp);
+}
+
+void
+mkuz_fqueue_enq(struct mkuz_fifo_queue *fqp, struct mkuz_blk *bp)
+{
+ struct mkuz_bchain_link *ip;
+
+ ip = mkuz_safe_zmalloc(sizeof(struct mkuz_bchain_link));
+ ip->this = bp;
+
+ pthread_mutex_lock(&fqp->mtx);
+ if (fqp->first != NULL) {
+ fqp->first->prev = ip;
+ } else {
+ fqp->last = ip;
+ }
+ fqp->first = ip;
+ fqp->length += 1;
+ if (fqp->length >= fqp->wakeup_len) {
+ pthread_cond_signal(&fqp->cvar);
+ }
+ pthread_mutex_unlock(&fqp->mtx);
+}
+
+#if defined(NOTYET)
+int
+mkuz_fqueue_enq_all(struct mkuz_fifo_queue *fqp, struct mkuz_bchain_link *cip_f,
+ struct mkuz_bchain_link *cip_l, int clen)
+{
+ int rval;
+
+ pthread_mutex_lock(&fqp->mtx);
+ if (fqp->first != NULL) {
+ fqp->first->prev = cip_l;
+ } else {
+ fqp->last = cip_l;
+ }
+ fqp->first = cip_f;
+ fqp->length += clen;
+ rval = fqp->length;
+ if (fqp->length >= fqp->wakeup_len) {
+ pthread_cond_signal(&fqp->cvar);
+ }
+ pthread_mutex_unlock(&fqp->mtx);
+ return (rval);
+}
+#endif
+
+static int
+mkuz_fqueue_check(struct mkuz_fifo_queue *fqp, cmp_cb_t cmp_cb, void *cap)
+{
+ struct mkuz_bchain_link *ip;
+
+ for (ip = fqp->last; ip != NULL; ip = ip->prev) {
+ if (cmp_cb(ip->this, cap)) {
+ return (1);
+ }
+ }
+ return (0);
+}
+
+struct mkuz_blk *
+mkuz_fqueue_deq_when(struct mkuz_fifo_queue *fqp, cmp_cb_t cmp_cb, void *cap)
+{
+ struct mkuz_bchain_link *ip, *newlast, *newfirst, *mip;
+ struct mkuz_blk *bp;
+
+ pthread_mutex_lock(&fqp->mtx);
+ while (fqp->last == NULL || !mkuz_fqueue_check(fqp, cmp_cb, cap)) {
+ pthread_cond_wait(&fqp->cvar, &fqp->mtx);
+ }
+ if (cmp_cb(fqp->last->this, cap)) {
+ mip = fqp->last;
+ fqp->last = mip->prev;
+ if (fqp->last == NULL) {
+#if defined(MKUZ_DEBUG)
+ assert(fqp->length == 1);
+#endif
+ fqp->first = NULL;
+ }
+ } else {
+#if defined(MKUZ_DEBUG)
+ assert(fqp->length > 1);
+#endif
+ newfirst = newlast = fqp->last;
+ mip = NULL;
+ for (ip = fqp->last->prev; ip != NULL; ip = ip->prev) {
+ if (cmp_cb(ip->this, cap)) {
+ mip = ip;
+ continue;
+ }
+ newfirst->prev = ip;
+ newfirst = ip;
+ }
+ newfirst->prev = NULL;
+ fqp->first = newfirst;
+ fqp->last = newlast;
+ }
+ fqp->length -= 1;
+ pthread_mutex_unlock(&fqp->mtx);
+ bp = mip->this;
+ free(mip);
+
+ return bp;
+}
+
+struct mkuz_blk *
+mkuz_fqueue_deq(struct mkuz_fifo_queue *fqp)
+{
+ struct mkuz_bchain_link *ip;
+ struct mkuz_blk *bp;
+
+ pthread_mutex_lock(&fqp->mtx);
+ while (fqp->last == NULL) {
+ pthread_cond_wait(&fqp->cvar, &fqp->mtx);
+ }
+#if defined(MKUZ_DEBUG)
+ assert(fqp->length > 0);
+#endif
+ ip = fqp->last;
+ fqp->last = ip->prev;
+ if (fqp->last == NULL) {
+#if defined(MKUZ_DEBUG)
+ assert(fqp->length == 1);
+#endif
+ fqp->first = NULL;
+ }
+ fqp->length -= 1;
+ pthread_mutex_unlock(&fqp->mtx);
+ bp = ip->this;
+ free(ip);
+
+ return bp;
+}
+
+#if defined(NOTYET)
+struct mkuz_bchain_link *
+mkuz_fqueue_deq_all(struct mkuz_fifo_queue *fqp, int *rclen)
+{
+ struct mkuz_bchain_link *rchain;
+
+ pthread_mutex_lock(&fqp->mtx);
+ while (fqp->last == NULL) {
+ pthread_cond_wait(&fqp->cvar, &fqp->mtx);
+ }
+#if defined(MKUZ_DEBUG)
+ assert(fqp->length > 0);
+#endif
+ rchain = fqp->last;
+ fqp->first = fqp->last = NULL;
+ *rclen = fqp->length;
+ fqp->length = 0;
+ pthread_mutex_unlock(&fqp->mtx);
+ return (rchain);
+}
+#endif
Added: head/usr.bin/mkuzip/mkuz_fqueue.h
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/usr.bin/mkuzip/mkuz_fqueue.h Sat Apr 23 07:23:43 2016 (r298504)
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2004-2016 Maxim Sobolev <sobomax at FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+struct mkuz_fifo_queue {
+ pthread_mutex_t mtx;
+ pthread_cond_t cvar;
+ struct mkuz_bchain_link *first;
+ struct mkuz_bchain_link *last;
+ int length;
+ int wakeup_len;
+};
+
+struct mkuz_blk;
+struct mkuz_bchain_link;
+
+DEFINE_RAW_METHOD(cmp_cb, int, const struct mkuz_blk *, void *);
+
+struct mkuz_fifo_queue *mkuz_fqueue_ctor(int);
+void mkuz_fqueue_enq(struct mkuz_fifo_queue *, struct mkuz_blk *);
+struct mkuz_blk *mkuz_fqueue_deq(struct mkuz_fifo_queue *);
+struct mkuz_blk *mkuz_fqueue_deq_when(struct mkuz_fifo_queue *, cmp_cb_t, void *);
+#if defined(NOTYET)
+struct mkuz_bchain_link *mkuz_fqueue_deq_all(struct mkuz_fifo_queue *, int *);
+int mkuz_fqueue_enq_all(struct mkuz_fifo_queue *, struct mkuz_bchain_link *,
+ struct mkuz_bchain_link *, int);
+#endif
Modified: head/usr.bin/mkuzip/mkuz_lzma.c
==============================================================================
--- head/usr.bin/mkuzip/mkuz_lzma.c Sat Apr 23 07:09:23 2016 (r298503)
+++ head/usr.bin/mkuzip/mkuz_lzma.c Sat Apr 23 07:23:43 2016 (r298504)
@@ -23,7 +23,6 @@
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
- *
*/
#include <sys/cdefs.h>
@@ -37,6 +36,7 @@ __FBSDID("$FreeBSD$");
#include "mkuzip.h"
#include "mkuz_lzma.h"
+#include "mkuz_blk.h"
#define USED_BLOCKSIZE DEV_BSIZE
@@ -44,15 +44,16 @@ struct mkuz_lzma {
lzma_filter filters[2];
lzma_options_lzma opt_lzma;
lzma_stream strm;
- char *obuf;
uint32_t blksz;
};
-static struct mkuz_lzma ulzma = {.strm = LZMA_STREAM_INIT};
+static const lzma_stream lzma_stream_init = LZMA_STREAM_INIT;
void *
mkuz_lzma_init(uint32_t blksz)
{
+ struct mkuz_lzma *ulp;
+
if (blksz % USED_BLOCKSIZE != 0) {
errx(1, "cluster size should be multiple of %d",
USED_BLOCKSIZE);
@@ -62,27 +63,34 @@ mkuz_lzma_init(uint32_t blksz)
errx(1, "cluster size is too large");
/* Not reached */
}
- ulzma.obuf = mkuz_safe_malloc(blksz * 2);
+ ulp = mkuz_safe_zmalloc(sizeof(struct mkuz_lzma));
/* Init lzma encoder */
- if (lzma_lzma_preset(&ulzma.opt_lzma, LZMA_PRESET_DEFAULT))
+ ulp->strm = lzma_stream_init;
+ if (lzma_lzma_preset(&ulp->opt_lzma, LZMA_PRESET_DEFAULT))
errx(1, "Error loading LZMA preset");
- ulzma.filters[0].id = LZMA_FILTER_LZMA2;
- ulzma.filters[0].options = &ulzma.opt_lzma;
- ulzma.filters[1].id = LZMA_VLI_UNKNOWN;
+ ulp->filters[0].id = LZMA_FILTER_LZMA2;
+ ulp->filters[0].options = &ulp->opt_lzma;
+ ulp->filters[1].id = LZMA_VLI_UNKNOWN;
- ulzma.blksz = blksz;
+ ulp->blksz = blksz;
- return (ulzma.obuf);
+ return (void *)ulp;
}
-void
-mkuz_lzma_compress(const char *ibuf, uint32_t *destlen)
+struct mkuz_blk *
+mkuz_lzma_compress(void *p, const struct mkuz_blk *iblk)
{
lzma_ret ret;
+ struct mkuz_blk *rval;
+ struct mkuz_lzma *ulp;
+
+ ulp = (struct mkuz_lzma *)p;
+
+ rval = mkuz_blk_ctor(ulp->blksz * 2);
- ret = lzma_stream_encoder(&ulzma.strm, ulzma.filters, LZMA_CHECK_CRC32);
+ ret = lzma_stream_encoder(&ulp->strm, ulp->filters, LZMA_CHECK_CRC32);
if (ret != LZMA_OK) {
if (ret == LZMA_MEMLIMIT_ERROR)
errx(1, "can't compress data: LZMA_MEMLIMIT_ERROR");
@@ -90,21 +98,24 @@ mkuz_lzma_compress(const char *ibuf, uin
errx(1, "can't compress data: LZMA compressor ERROR");
}
- ulzma.strm.next_in = ibuf;
- ulzma.strm.avail_in = ulzma.blksz;
- ulzma.strm.next_out = ulzma.obuf;
- ulzma.strm.avail_out = ulzma.blksz * 2;
+ ulp->strm.next_in = iblk->data;
+ ulp->strm.avail_in = ulp->blksz;
+ ulp->strm.next_out = rval->data;
+ ulp->strm.avail_out = rval->alen;
- ret = lzma_code(&ulzma.strm, LZMA_FINISH);
+ ret = lzma_code(&ulp->strm, LZMA_FINISH);
if (ret != LZMA_STREAM_END) {
/* Error */
errx(1, "lzma_code FINISH failed, code=%d, pos(in=%zd, "
- "out=%zd)", ret, (ulzma.blksz - ulzma.strm.avail_in),
- (ulzma.blksz * 2 - ulzma.strm.avail_out));
+ "out=%zd)", ret, (ulp->blksz - ulp->strm.avail_in),
+ (ulp->blksz * 2 - ulp->strm.avail_out));
}
- lzma_end(&ulzma.strm);
+#if 0
+ lzma_end(&ulp->strm);
+#endif
- *destlen = (ulzma.blksz * 2) - ulzma.strm.avail_out;
+ rval->info.len = rval->alen - ulp->strm.avail_out;
+ return (rval);
}
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-all
mailing list