Datasetpublic

codekingpro/portable-devtools

sourceHugging Faceupdated 5mo agoView on Hugging Face

1likes14kdownloads

rootpythonuser_packagesPython313site-packagestokenizersdecoders__init__.py

__init__.py16 linesDownload Raw Back to decoders

1from .. import decoders
2
3
4Decoder = decoders.Decoder
5ByteLevel = decoders.ByteLevel
6Replace = decoders.Replace
7WordPiece = decoders.WordPiece
8ByteFallback = decoders.ByteFallback
9Fuse = decoders.Fuse
10Strip = decoders.Strip
11Metaspace = decoders.Metaspace
12BPEDecoder = decoders.BPEDecoder
13CTC = decoders.CTC
14Sequence = decoders.Sequence
15DecodeStream = decoders.DecodeStream
16