Commits · xu-song/tokenizer-arena

add more tokenizer

0bba168

xu-song commited on Aug 18

update

83a2e6b

xu-song commited on Jul 28

add more tokenizers

4cbb545

xu-song commited on Jul 28

update

299dc31

xu-song commited on Jul 18

fix typo

efd894c

xu-song commited on Jul 18

update

b0c41e0

xu-song commited on Jul 15

add doc

70c093d

xu-song commited on Jul 13

add doc

47e1616

xu-song commited on Jul 13

update

0d55475

xu-song commited on Jul 13

update gradio to 4.38.1

8167cc2

xu-song commited on Jul 13

add compression_rate details

a4208a2

xu-song commited on Jul 13

remove git lfs

e411220

xu-song commited on Jul 13

update

7c73423

xu-song commited on Jul 13

fix lru_cache and fix qwen_vocab

9d1b27e

xu-song commited on May 23

launch with queue

baf4d1e

xu-song commited on May 23

add access token

f1b4ae2

xu-song commited on May 23

remove lru_cache: has some bugs

f652c69

xu-song commited on May 21

update

97354e0

xu-song commited on May 21

update

d3c1316

xu-song commited on May 21

update

5a684a6

xu-song commited on May 21

update

42127b1

xu-song commited on May 21

remove vocabs; update compression_app; add character_app;

2bd606a

xu-song commited on May 21

update

f331792

xu-song commited on May 1

update

9558ae0

xu-song commited on May 1

add compression leaderboard

1b7fc74

xu-song commited on May 1

update compress rate

367a536

xu-song commited on Apr 23

update compress rate

988921c

xu-song commited on Apr 23

update

7d2062e

xu-song commited on Apr 23

add grok mixtral

480ae5d

xu-song commited on Apr 23

add compress rate

814ee6b

xu-song commited on Apr 22

add zephyr

a6aee1d

xu-song commited on Mar 15

requi

6b70021

xu-song commited on Mar 13

requi

510279b

xu-song commited on Mar 13

requi

7b522e7

xu-song commited on Mar 13

config python version

11379e2

xu-song commited on Mar 13

add xlm-roberta

057bc67

xu-song commited on Mar 13

add amber and crystal_coder

5db13e0

xu-song commited on Mar 8

add character glm

f0f84b2

xu-song commited on Mar 5

update

f02dd94

xu-song commited on Mar 4

fix PyO3PanicException

2461705

xu-song commited on Mar 4

fix unicode error: 'unicodeescape' codec can't decode bytes in position 602-608: unknown Unicode character name

bce41d0

xu-song commited on Mar 4

fix fastchat_t5_3b

c766a08

xu-song commited on Mar 4

fix tiktoken special tokens

adcfb97

xu-song commited on Mar 1

add aya

44c3329

xu-song commited on Feb 28

fix olmo

2442c83

xu-song commited on Feb 28

add olmo tokenizer

bbefe94

xu-song commited on Feb 28

update

24b4aa5

xu-song commited on Feb 26

update

1f833af

xu-song commited on Feb 26

fix tiktoken

a6c67ec

xu-song commited on Feb 25

fix gemma_7b

7011963

xu-song commited on Feb 22

Commit History

add more tokenizer 0bba168

update 83a2e6b

add more tokenizers 4cbb545

update 299dc31

fix typo efd894c

update b0c41e0

add doc 70c093d

add doc 47e1616

update 0d55475

update gradio to 4.38.1 8167cc2

add compression_rate details a4208a2

remove git lfs e411220

update 7c73423

fix lru_cache and fix qwen_vocab 9d1b27e

launch with queue baf4d1e

add access token f1b4ae2

remove lru_cache: has some bugs f652c69

update 97354e0

update d3c1316

update 5a684a6

update 42127b1

remove vocabs; update compression_app; add character_app; 2bd606a

update f331792

update 9558ae0

add compression leaderboard 1b7fc74

update compress rate 367a536

update compress rate 988921c

update 7d2062e

add grok mixtral 480ae5d

add compress rate 814ee6b

add zephyr a6aee1d

requi 6b70021

requi 510279b

requi 7b522e7

config python version 11379e2

add xlm-roberta 057bc67

add amber and crystal_coder 5db13e0

add character glm f0f84b2

update f02dd94

fix PyO3PanicException 2461705

fix unicode error: 'unicodeescape' codec can't decode bytes in position 602-608: unknown Unicode character name bce41d0

fix fastchat_t5_3b c766a08

fix tiktoken special tokens adcfb97

add aya 44c3329

fix olmo 2442c83

add olmo tokenizer bbefe94

update 24b4aa5

update 1f833af

fix tiktoken a6c67ec

fix gemma_7b 7011963

add more tokenizer

0bba168

update

83a2e6b

add more tokenizers

4cbb545

update

299dc31

fix typo

efd894c

update

b0c41e0

add doc

70c093d

add doc

47e1616

update

0d55475

update gradio to 4.38.1

8167cc2

add compression_rate details

a4208a2

remove git lfs

e411220

update

7c73423

fix lru_cache and fix qwen_vocab

9d1b27e

launch with queue

baf4d1e

add access token

f1b4ae2

remove lru_cache: has some bugs

f652c69

update

97354e0

update

d3c1316

update

5a684a6

update

42127b1

remove vocabs; update compression_app; add character_app;

2bd606a

update

f331792

update

9558ae0

add compression leaderboard

1b7fc74

update compress rate

367a536

update compress rate

988921c

update

7d2062e

add grok mixtral

480ae5d

add compress rate

814ee6b

add zephyr

a6aee1d

requi

6b70021

requi

510279b

requi

7b522e7

config python version

11379e2

add xlm-roberta

057bc67

add amber and crystal_coder

5db13e0

add character glm

f0f84b2

update

f02dd94

fix PyO3PanicException

2461705

fix unicode error: 'unicodeescape' codec can't decode bytes in position 602-608: unknown Unicode character name

bce41d0

fix fastchat_t5_3b

c766a08

fix tiktoken special tokens

adcfb97

add aya

44c3329

fix olmo

2442c83

add olmo tokenizer

bbefe94

update

24b4aa5

update

1f833af

fix tiktoken

a6c67ec

fix gemma_7b

7011963