{
  "repository": "https://github.com/slayerlabs/tokenizer",
  "commit": "1b25efca5c1a3891edcfdc47a8945b55e531223c",
  "checked_at": "2026-10-09",
  "published_tokenization_artifact_files": 54,
  "author_directories": 12,
  "max_observed_vocabulary_size": 512000,
  "counting_note": "Serialized tokenizer vocabularies and merge artifacts. Excludes metadata, metrics and three experiment-result JSON reports. File count, not a claim of 54 unique algorithms.",
  "files": [
    {
      "path": "Arek/1024/lektury-fast.json",
      "observed_vocab_entries": 1024
    },
    {
      "path": "Arek/1024/lektury-naive.json",
      "observed_vocab_entries": 1024
    },
    {
      "path": "Arek/115200/slayer-v2.json",
      "observed_vocab_entries": 115200
    },
    {
      "path": "Arek/128000/slayer-v2.json",
      "observed_vocab_entries": 128000
    },
    {
      "path": "Arek/15000/lektury-fast.json",
      "observed_vocab_entries": 15000
    },
    {
      "path": "Arek/2048/lektury-fast.json",
      "observed_vocab_entries": 2048
    },
    {
      "path": "Arek/2048/lektury-naive.json",
      "observed_vocab_entries": 2048
    },
    {
      "path": "Arek/256000/slayer-v2.json",
      "observed_vocab_entries": 256000
    },
    {
      "path": "Arek/32000/slayer-v1.json",
      "observed_vocab_entries": 32000
    },
    {
      "path": "Arek/32000/slayer-v2.json",
      "observed_vocab_entries": 32000
    },
    {
      "path": "Arek/4096/lektury-fast.json",
      "observed_vocab_entries": 4096
    },
    {
      "path": "Arek/512/lektury-fast.json",
      "observed_vocab_entries": 512
    },
    {
      "path": "Arek/512/lektury-naive.json",
      "observed_vocab_entries": 512
    },
    {
      "path": "Arek/512000/slayer-v2.json",
      "observed_vocab_entries": 512000
    },
    {
      "path": "Arek/64000/slayer-v1.json",
      "observed_vocab_entries": 64000
    },
    {
      "path": "Arek/64000/slayer-v2.json",
      "observed_vocab_entries": 64000
    },
    {
      "path": "Arek/8192/lektury-fast.json",
      "observed_vocab_entries": 8192
    },
    {
      "path": "Jakub K./tokenizer_merges.json",
      "observed_vocab_entries": null
    },
    {
      "path": "Janek/tokenizer.json",
      "observed_vocab_entries": 8192
    },
    {
      "path": "KasiaMP/wyniki/hf_bez_regex.json",
      "observed_vocab_entries": 6756
    },
    {
      "path": "KasiaMP/wyniki/hf_norm_filtrowany.json",
      "observed_vocab_entries": 6756
    },
    {
      "path": "KasiaMP/wyniki/hf_normalizowany.json",
      "observed_vocab_entries": 6756
    },
    {
      "path": "KasiaMP/wyniki/hf_z_regex.json",
      "observed_vocab_entries": 6756
    },
    {
      "path": "KasiaMP/wyniki/tokenizer32k.json",
      "observed_vocab_entries": 32568
    },
    {
      "path": "Konrad/tokenizer.json",
      "observed_vocab_entries": 456
    },
    {
      "path": "Kuba/tokenizer.json",
      "observed_vocab_entries": 14000
    },
    {
      "path": "dawidm/PollockBPE/tokenizer.json",
      "observed_vocab_entries": 12285
    },
    {
      "path": "dawidm/vocabs/dynaword-100-novels-pretok_1500.tokenizer.json",
      "observed_vocab_entries": 1500
    },
    {
      "path": "dawidm/vocabs/syzyfowe-prace-pretok_5000.tokenizer.json",
      "observed_vocab_entries": 5000
    },
    {
      "path": "dawidm/vocabs/syzyfowe-prace_16000.tokenizer.json",
      "observed_vocab_entries": 16000
    },
    {
      "path": "dawidm/vocabs/syzyfowe-prace_5000.tokenizer.json",
      "observed_vocab_entries": 5000
    },
    {
      "path": "dawidm/vocabs/wikinews-pretok_16000.tokenizer.json",
      "observed_vocab_entries": 16000
    },
    {
      "path": "dawidm/vocabs/wikinews_16000.tokenizer.json",
      "observed_vocab_entries": 16000
    },
    {
      "path": "dawidm/vocabs/wikinews_5000.tokenizer.json",
      "observed_vocab_entries": 5000
    },
    {
      "path": "maciej/tokenizers/hplt-pl-16k-pl.json",
      "observed_vocab_entries": 15999
    },
    {
      "path": "maciej/tokenizers/hplt-pl-32k-gpt2.json",
      "observed_vocab_entries": 31999
    },
    {
      "path": "maciej/tokenizers/hplt-pl-32k-pl.json",
      "observed_vocab_entries": 31999
    },
    {
      "path": "maciej/tokenizers/hplt-pl-64k-pl.json",
      "observed_vocab_entries": 63999
    },
    {
      "path": "maciej/tokenizers/hplt-pl-8k-pl.json",
      "observed_vocab_entries": 7999
    },
    {
      "path": "ola/diverse_tokenizer.json",
      "observed_vocab_entries": 512
    },
    {
      "path": "ola/polish_bpe_8k.json",
      "observed_vocab_entries": 8000
    },
    {
      "path": "ola/quo_vadis_tokenizer.json",
      "observed_vocab_entries": 512
    },
    {
      "path": "ola/quo_vadis_tokenizer_4k.json",
      "observed_vocab_entries": 4096
    },
    {
      "path": "patryk/polish_bpe_vocab.json",
      "observed_vocab_entries": 16000
    },
    {
      "path": "ppuzio/tokenizers/dynaword-10k/tokenizer.json",
      "observed_vocab_entries": 10000
    },
    {
      "path": "ppuzio/tokenizers/dynaword-128k-g220mb-l50mb/tokenizer.json",
      "observed_vocab_entries": 128000
    },
    {
      "path": "ppuzio/tokenizers/dynaword-128k-g500mb-l50mb/tokenizer.json",
      "observed_vocab_entries": 128000
    },
    {
      "path": "ppuzio/tokenizers/dynaword-128k-gfull-l50mb/tokenizer.json",
      "observed_vocab_entries": 128000
    },
    {
      "path": "ppuzio/tokenizers/dynaword-14k-220mb/tokenizer.json",
      "observed_vocab_entries": 14000
    },
    {
      "path": "ppuzio/tokenizers/dynaword-14k/tokenizer.json",
      "observed_vocab_entries": 14000
    },
    {
      "path": "ppuzio/tokenizers/dynaword-32k-220mb/tokenizer.json",
      "observed_vocab_entries": 32000
    },
    {
      "path": "ppuzio/tokenizers/dynaword-32k/tokenizer.json",
      "observed_vocab_entries": 32000
    },
    {
      "path": "ppuzio/tokenizers/dynaword-64k-220mb/tokenizer.json",
      "observed_vocab_entries": 64000
    },
    {
      "path": "ppuzio/tokenizers/v1-ogniem-10k/tokenizer.json",
      "observed_vocab_entries": null
    }
  ]
}