svn commit: r340835 - in head/lib/libc: regex tests/regex

[Yuri Pankov]

Author: yuripv
Date: Fri Nov 23 15:49:18 2018
New Revision: 340835
URL: https://svnweb.freebsd.org/changeset/base/340835

Log:
  regexec: fix processing multibyte strings.
  
  Matcher function incorrectly assumed that moffset that we get from
  findmust is in bytes. Fix this by introducing a stepback function,
  taking short path if MB_CUR_MAX is 1, and going back byte-by-byte,
  checking if we have a legal character sequence otherwise.
  
  PR:		153502
  Reviewed by:	pfg, kevans
  Approved by:	kib (mentor, implicit)
  Differential revision:	https://reviews.freebsd.org/D18297

Added:
  head/lib/libc/tests/regex/multibyte.sh   (contents, props changed)
Modified:
  head/lib/libc/regex/engine.c
  head/lib/libc/tests/regex/Makefile

Modified: head/lib/libc/regex/engine.c
==============================================================================
--- head/lib/libc/regex/engine.c	Fri Nov 23 13:50:18 2018	(r340834)
+++ head/lib/libc/regex/engine.c	Fri Nov 23 15:49:18 2018	(r340835)
@@ -48,6 +48,7 @@ __FBSDID("$FreeBSD$");
  */
 
 #ifdef SNAMES
+#define	stepback sstepback
 #define	matcher	smatcher
 #define	walk	swalk
 #define	dissect	sdissect
@@ -58,6 +59,7 @@ __FBSDID("$FreeBSD$");
 #define	match	smat
 #endif
 #ifdef LNAMES
+#define	stepback lstepback
 #define	matcher	lmatcher
 #define	walk	lwalk
 #define	dissect	ldissect
@@ -68,6 +70,7 @@ __FBSDID("$FreeBSD$");
 #define	match	lmat
 #endif
 #ifdef MNAMES
+#define	stepback mstepback
 #define	matcher	mmatcher
 #define	walk	mwalk
 #define	dissect	mdissect
@@ -142,6 +145,39 @@ static const char *pchar(int ch);
 #endif
 
 /*
+ * Given a multibyte string pointed to by start, step back nchar characters
+ * from current position pointed to by cur.
+ */
+static const char *
+stepback(const char *start, const char *cur, int nchar)
+{
+	const char *ret;
+	int wc, mbc;
+	mbstate_t mbs;
+	size_t clen;
+
+	if (MB_CUR_MAX == 1)
+		return ((cur - nchar) > start ? cur - nchar : NULL);
+
+	ret = cur;
+	for (wc = nchar; wc > 0; wc--) {
+		for (mbc = 1; mbc <= MB_CUR_MAX; mbc++) {
+			if ((ret - mbc) < start)
+				return (NULL);
+			memset(&mbs, 0, sizeof(mbs));
+			clen = mbrtowc(NULL, ret - mbc, mbc, &mbs);
+			if (clen != (size_t)-1 && clen != (size_t)-2)
+				break;
+		}
+		if (mbc > MB_CUR_MAX)
+			return (NULL);
+		ret -= mbc;
+	}
+
+	return (ret);
+}
+
+/*
  - matcher - the actual matching engine
  == static int matcher(struct re_guts *g, const char *string, \
  ==	size_t nmatch, regmatch_t pmatch[], int eflags);
@@ -244,9 +280,14 @@ matcher(struct re_guts *g,
 	ZAPSTATE(&m->mbs);
 
 	/* Adjust start according to moffset, to speed things up */
-	if (dp != NULL && g->moffset > -1)
-		start = ((dp - g->moffset) < start) ? start : dp - g->moffset;
+	if (dp != NULL && g->moffset > -1) {
+		const char *nstart;
 
+		nstart = stepback(start, dp, g->moffset);
+		if (nstart != NULL)
+			start = nstart;
+	}
+
 	SP("mloop", m->st, *start);
 
 	/* this loop does only one repetition except for backrefs */
@@ -1083,6 +1124,7 @@ pchar(int ch)
 #endif
 #endif
 
+#undef	stepback
 #undef	matcher
 #undef	walk
 #undef	dissect

Modified: head/lib/libc/tests/regex/Makefile
==============================================================================
--- head/lib/libc/tests/regex/Makefile	Fri Nov 23 13:50:18 2018	(r340834)
+++ head/lib/libc/tests/regex/Makefile	Fri Nov 23 15:49:18 2018	(r340835)
@@ -2,6 +2,9 @@
 
 PACKAGE=	tests
 
+# local test cases
+ATF_TESTS_SH+=	multibyte
+
 .include "Makefile.inc"
 .include "${.CURDIR:H}/Makefile.netbsd-tests"
 .include <bsd.test.mk>

Added: head/lib/libc/tests/regex/multibyte.sh
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ head/lib/libc/tests/regex/multibyte.sh	Fri Nov 23 15:49:18 2018	(r340835)
@@ -0,0 +1,35 @@
+# $FreeBSD$
+
+atf_test_case multibyte
+multibyte_head()
+{
+	atf_set "descr" "Check matching multibyte characters (PR153502)"
+}
+multibyte_body()
+{
+	export LC_CTYPE="C.UTF-8"
+
+	printf 'é' | atf_check -o "inline:é" \
+	    sed -ne '/^.$/p'
+	printf 'éé' | atf_check -o "inline:éé" \
+	    sed -ne '/^..$/p'
+	printf 'aéa' | atf_check -o "inline:aéa" \
+	    sed -ne '/a.a/p'
+	printf 'aéa'| atf_check -o "inline:aéa" \
+	    sed -ne '/a.*a/p'
+	printf 'aaéaa' | atf_check -o "inline:aaéaa" \
+	    sed -ne '/aa.aa/p'
+	printf 'aéaéa' | atf_check -o "inline:aéaéa" \
+	    sed -ne '/a.a.a/p'
+	printf 'éa' | atf_check -o "inline:éa" \
+	    sed -ne '/.a/p'
+	printf 'aéaa' | atf_check -o "inline:aéaa" \
+	    sed -ne '/a.aa/p'
+	printf 'éaé' | atf_check -o "inline:éaé" \
+	    sed -ne '/.a./p'
+}
+
+atf_init_test_cases()
+{
+	atf_add_test_case multibyte
+}