# Copyright 2025 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2

EAPI=8

CRATES="
	ahash@0.8.12
	aho-corasick@1.1.4
	anstream@1.0.0
	anstyle-parse@1.0.0
	anstyle-query@1.1.5
	anstyle-wincon@3.0.11
	anstyle@1.0.14
	anyhow@1.0.102
	autocfg@1.5.0
	base64@0.13.1
	bitflags@2.11.1
	bumpalo@3.20.2
	castaway@0.2.4
	cc@1.2.61
	cfg-if@1.0.4
	colorchoice@1.0.5
	compact_str@0.9.0
	console@0.16.3
	crossbeam-deque@0.8.6
	crossbeam-epoch@0.9.18
	crossbeam-utils@0.8.21
	daachorse@1.0.1
	darling@0.20.11
	darling_core@0.20.11
	darling_macro@0.20.11
	dary_heap@0.3.9
	derive_builder@0.20.2
	derive_builder_core@0.20.2
	derive_builder_macro@0.20.2
	either@1.15.0
	encode_unicode@1.0.0
	env_filter@1.0.1
	env_logger@0.11.10
	equivalent@1.0.2
	errno@0.3.14
	esaxx-rs@0.1.10
	fastrand@2.4.1
	find-msvc-tools@0.1.9
	fnv@1.0.7
	foldhash@0.1.5
	futures-channel@0.3.32
	futures-core@0.3.32
	futures-macro@0.3.32
	futures-task@0.3.32
	futures-util@0.3.32
	getrandom@0.3.4
	getrandom@0.4.2
	hashbrown@0.15.5
	hashbrown@0.17.0
	heck@0.5.0
	id-arena@2.3.0
	ident_case@1.0.1
	indexmap@2.14.0
	indicatif@0.18.4
	is_terminal_polyfill@1.70.2
	itertools@0.14.0
	itoa@1.0.18
	jiff-static@0.2.24
	jiff@0.2.24
	js-sys@0.3.95
	leb128fmt@0.1.0
	libc@0.2.186
	linux-raw-sys@0.12.1
	log@0.4.29
	macro_rules_attribute-proc_macro@0.2.2
	macro_rules_attribute@0.2.2
	matrixmultiply@0.3.10
	memchr@2.8.0
	minimal-lexical@0.2.1
	mio@1.2.0
	monostate-impl@0.1.18
	monostate@0.1.18
	ndarray@0.16.1
	ndarray@0.17.2
	nom@7.1.3
	num-complex@0.4.6
	num-integer@0.1.46
	num-traits@0.2.19
	numpy@0.28.0
	once_cell@1.21.4
	once_cell_polyfill@1.70.2
	onig@6.5.3
	onig_sys@69.9.3
	paste@1.0.15
	pin-project-lite@0.2.17
	pkg-config@0.3.33
	portable-atomic-util@0.2.7
	portable-atomic@1.13.1
	ppv-lite86@0.2.21
	prettyplease@0.2.37
	proc-macro2@1.0.106
	pyo3-async-runtimes@0.28.0
	pyo3-build-config@0.28.2
	pyo3-ffi@0.28.2
	pyo3-macros-backend@0.28.2
	pyo3-macros@0.28.2
	pyo3@0.28.2
	quote@1.0.45
	r-efi@5.3.0
	r-efi@6.0.0
	rand@0.9.4
	rand_chacha@0.9.0
	rand_core@0.9.5
	rawpointer@0.2.1
	rayon-cond@0.4.0
	rayon-core@1.13.0
	rayon@1.12.0
	regex-automata@0.4.14
	regex-syntax@0.8.10
	regex@1.12.3
	rustc-hash@2.1.2
	rustix@1.1.4
	rustversion@1.0.22
	ryu@1.0.23
	semver@1.0.28
	serde@1.0.228
	serde_core@1.0.228
	serde_derive@1.0.228
	serde_json@1.0.149
	shlex@1.3.0
	signal-hook-registry@1.4.8
	slab@0.4.12
	smallvec@1.15.1
	spm_precompiled@0.1.4
	static_assertions@1.1.0
	strsim@0.11.1
	syn@2.0.117
	target-lexicon@0.13.5
	tempfile@3.27.0
	thiserror-impl@2.0.18
	thiserror@2.0.18
	tokio-macros@2.7.0
	tokio@1.52.1
	unicode-ident@1.0.24
	unicode-normalization-alignments@0.1.12
	unicode-segmentation@1.13.2
	unicode-width@0.2.2
	unicode-xid@0.2.6
	unicode_categories@0.1.1
	unit-prefix@0.5.2
	utf8parse@0.2.2
	version_check@0.9.5
	wasi@0.11.1+wasi-snapshot-preview1
	wasip2@1.0.3+wasi-0.2.9
	wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06
	wasm-bindgen-macro-support@0.2.118
	wasm-bindgen-macro@0.2.118
	wasm-bindgen-shared@0.2.118
	wasm-bindgen@0.2.118
	wasm-encoder@0.244.0
	wasm-metadata@0.244.0
	wasmparser@0.244.0
	web-time@1.1.0
	windows-link@0.2.1
	windows-sys@0.61.2
	wit-bindgen-core@0.51.0
	wit-bindgen-rust-macro@0.51.0
	wit-bindgen-rust@0.51.0
	wit-bindgen@0.51.0
	wit-bindgen@0.57.1
	wit-component@0.244.0
	wit-parser@0.244.0
	zerocopy-derive@0.8.48
	zerocopy@0.8.48
	zmij@1.0.21
"

DISTUTILS_EXT=1
DISTUTILS_USE_PEP517=maturin
PYTHON_COMPAT=( python3_{11..13} )

inherit cargo distutils-r1

DESCRIPTION="Fast State-of-the-Art Tokenizers optimized for Research and Production"
HOMEPAGE="https://github.com/huggingface/tokenizers"
SRC_URI="
	https://files.pythonhosted.org/packages/source/t/tokenizers/tokenizers-${PV}.tar.gz
	${CARGO_CRATE_URIS}
"

LICENSE="Apache-2.0"
LICENSE+=" Apache-2.0 MIT Unicode-3.0"
SLOT="0"
KEYWORDS="~amd64"

RDEPEND="
	>=dev-python/huggingface-hub-0.16.4[${PYTHON_USEDEP}]
"

DEPEND="${RDEPEND}"

QA_FLAGS_IGNORED="usr/lib.*/py.*/site-packages/tokenizers/.*\.so"