git: 6259f7882a8d - main - textproc/py-tokenizer: Add new port

The branch main has been updated by otis:

URL: https://cgit.FreeBSD.org/ports/commit/?id=6259f7882a8dc19deaf79bbbb7d28c2fce54aa34

commit 6259f7882a8dc19deaf79bbbb7d28c2fce54aa34
Author:     Juraj Lutter <otis@FreeBSD.org>
AuthorDate: 2025-08-27 11:13:40 +0000
Commit:     Juraj Lutter <otis@FreeBSD.org>
CommitDate: 2025-08-27 11:13:40 +0000

    textproc/py-tokenizer: Add new port
    
    Add textproc/py-tokenizer, a tokenizer for Icelandic text.
---
 textproc/Makefile               |  1 +
 textproc/py-tokenizer/Makefile  | 21 +++++++++++++++++++++
 textproc/py-tokenizer/distinfo  |  3 +++
 textproc/py-tokenizer/pkg-descr | 11 +++++++++++
 4 files changed, 36 insertions(+)

diff --git a/textproc/Makefile b/textproc/Makefile
index 52ce156f9036..07aaabe37937 100644
--- a/textproc/Makefile
+++ b/textproc/Makefile
@@ -1690,6 +1690,7 @@
     SUBDIR += py-tiktoken
     SUBDIR += py-tinycss
     SUBDIR += py-tinycss2
+    SUBDIR += py-tokenizer
     SUBDIR += py-tokenizers
     SUBDIR += py-toml
     SUBDIR += py-tomli
diff --git a/textproc/py-tokenizer/Makefile b/textproc/py-tokenizer/Makefile
new file mode 100644
index 000000000000..4f8afff7b8be
--- /dev/null
+++ b/textproc/py-tokenizer/Makefile
@@ -0,0 +1,21 @@
+PORTNAME=	tokenizer
+PORTVERSION=	3.5.0
+CATEGORIES=	textproc python
+MASTER_SITES=	PYPI
+PKGNAMEPREFIX=	${PYTHON_PKGNAMEPREFIX}
+
+MAINTAINER=	otis@FreeBSD.org
+COMMENT=	Tokenizer for Icelandic text
+WWW=		https://github.com/mideind/Tokenizer
+
+LICENSE=	MIT
+
+BUILD_DEPENDS=	${PY_SETUPTOOLS} \
+		${PYTHON_PKGNAMEPREFIX}wheel>0:devel/py-wheel@${PY_FLAVOR}
+
+USES=		python
+USE_PYTHON=	autoplist concurrent pep517
+
+NO_ARCH=	yes
+
+.include <bsd.port.mk>
diff --git a/textproc/py-tokenizer/distinfo b/textproc/py-tokenizer/distinfo
new file mode 100644
index 000000000000..5002d345dfbd
--- /dev/null
+++ b/textproc/py-tokenizer/distinfo
@@ -0,0 +1,3 @@
+TIMESTAMP = 1756293103
+SHA256 (tokenizer-3.5.0.tar.gz) = f26694d6be85815d23db167ee9b1c9e2ed7b672cfc8b9baa607ba3aba6070727
+SIZE (tokenizer-3.5.0.tar.gz) = 127323
diff --git a/textproc/py-tokenizer/pkg-descr b/textproc/py-tokenizer/pkg-descr
new file mode 100644
index 000000000000..665fa0186f94
--- /dev/null
+++ b/textproc/py-tokenizer/pkg-descr
@@ -0,0 +1,11 @@
+This python utility package helps to create lazy modules. A lazy module defers
+loading (some of) its attributes until these attributes are first accessed. The
+module's lazy attributes in turn are attributes of other modules. These other
+modules will be imported/loaded only when (and if) associated attributes are
+used. A lazy import strategy can drastically reduce runtime and memory
+consumption.
+
+Additionally, this package provides a utility for optional imports with which
+one can import a module globally while triggering associated import errors only
+at use-sites (when and if a dependency is actually required, for example in the
+context of a specific functionality).