{
  "checked_at": "2026-10-09",
  "commit": "1b25efca5c1a3891edcfdc47a8945b55e531223c",
  "groups": [
    {
      "id": "dynaword",
      "label": "DynaWord",
      "docs": 48354,
      "description": "48 354 dokumenty · wydzielone końcówki 12 źródeł DynaWord",
      "source": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/ppuzio/tokenizers/fertility_comparison.json",
      "rows": [
        {
          "name": "dynaword-128k-g220mb-l50mb",
          "author": "Paweł Puzio",
          "vocab": 128000,
          "fertility": 1.4988,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/ppuzio/tokenizers/dynaword-128k-g220mb-l50mb/tokenizer.json"
        },
        {
          "name": "dynaword-128k-g500mb-l50mb",
          "author": "Paweł Puzio",
          "vocab": 128000,
          "fertility": 1.5001,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/ppuzio/tokenizers/dynaword-128k-g500mb-l50mb/tokenizer.json"
        },
        {
          "name": "dynaword-128k-gfull-l50mb",
          "author": "Paweł Puzio",
          "vocab": 128000,
          "fertility": 1.5089,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/ppuzio/tokenizers/dynaword-128k-gfull-l50mb/tokenizer.json"
        },
        {
          "name": "dynaword-64k-220mb",
          "author": "Paweł Puzio",
          "vocab": 64000,
          "fertility": 1.5745,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/ppuzio/tokenizers/dynaword-64k-220mb/tokenizer.json"
        },
        {
          "name": "dynaword-32k-220mb",
          "author": "Paweł Puzio",
          "vocab": 32000,
          "fertility": 1.6967,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/ppuzio/tokenizers/dynaword-32k-220mb/tokenizer.json"
        },
        {
          "name": "dynaword-32k",
          "author": "Paweł Puzio",
          "vocab": 32000,
          "fertility": 1.7675,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/ppuzio/tokenizers/dynaword-32k/tokenizer.json"
        },
        {
          "name": "dynaword-14k-220mb",
          "author": "Paweł Puzio",
          "vocab": 14000,
          "fertility": 1.8962,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/ppuzio/tokenizers/dynaword-14k-220mb/tokenizer.json"
        },
        {
          "name": "dynaword-14k",
          "author": "Paweł Puzio",
          "vocab": 14000,
          "fertility": 1.9673,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/ppuzio/tokenizers/dynaword-14k/tokenizer.json"
        },
        {
          "name": "dynaword-10k",
          "author": "Paweł Puzio",
          "vocab": 10000,
          "fertility": 2.0668,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/ppuzio/tokenizers/dynaword-10k/tokenizer.json"
        },
        {
          "name": "kuba-14k",
          "author": "Kuba",
          "vocab": 14000,
          "fertility": 2.1498,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/Kuba/tokenizer.json"
        }
      ]
    },
    {
      "id": "hplt",
      "label": "HPLT",
      "docs": 4773,
      "description": "4 773 dokumenty · 10,5 MB polskiego held-outu HPLT",
      "source": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/maciej/results/metrics.json",
      "rows": [
        {
          "name": "hplt-pl-64k-pl",
          "author": "Maciej",
          "vocab": 64000,
          "fertility": 1.517,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/maciej/tokenizers/hplt-pl-64k-pl.json"
        },
        {
          "name": "hplt-pl-32k-gpt2",
          "author": "Maciej",
          "vocab": 32000,
          "fertility": 1.6472,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/maciej/tokenizers/hplt-pl-32k-gpt2.json"
        },
        {
          "name": "hplt-pl-32k-pl",
          "author": "Maciej",
          "vocab": 32000,
          "fertility": 1.6516,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/maciej/tokenizers/hplt-pl-32k-pl.json"
        },
        {
          "name": "hplt-pl-16k-pl",
          "author": "Maciej",
          "vocab": 16000,
          "fertility": 1.8315,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/maciej/tokenizers/hplt-pl-16k-pl.json"
        },
        {
          "name": "hplt-pl-8k-pl",
          "author": "Maciej",
          "vocab": 8000,
          "fertility": 2.0617,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/maciej/tokenizers/hplt-pl-8k-pl.json"
        },
        {
          "name": "o200k_base (GPT-4o)",
          "author": "Punkt odniesienia",
          "vocab": 200019,
          "fertility": 2.3159,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/maciej/results/metrics.json"
        },
        {
          "name": "cl100k_base (GPT-4)",
          "author": "Punkt odniesienia",
          "vocab": 100277,
          "fertility": 2.6983,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/maciej/results/metrics.json"
        },
        {
          "name": "gpt2",
          "author": "Punkt odniesienia",
          "vocab": 50257,
          "fertility": 3.7073,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/maciej/results/metrics.json"
        }
      ]
    },
    {
      "id": "catalog",
      "label": "Katalog artefaktów",
      "docs": null,
      "description": "Jeden projekt na autora. Wybrane artefakty prowadzą do prac uczestników; pozostałe warianty są w repozytorium. Rozmiary słowników odczytano z zapisanych plików.",
      "source": "https://github.com/slayerlabs/tokenizer/tree/1b25efca5c1a3891edcfdc47a8945b55e531223c",
      "rows": [
        {
          "name": "Arek/512000/slayer-v2.json",
          "author": "Arek",
          "vocab": 512000,
          "fertility": null,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/Arek/512000/slayer-v2.json"
        },
        {
          "name": "Jakub K./tokenizer_merges.json",
          "author": "Jakub K.",
          "vocab": null,
          "fertility": null,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/Jakub%20K./tokenizer_merges.json"
        },
        {
          "name": "Janek/tokenizer.json",
          "author": "Janek",
          "vocab": 8192,
          "fertility": null,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/Janek/tokenizer.json"
        },
        {
          "name": "KasiaMP/wyniki/tokenizer32k.json",
          "author": "Kasia",
          "vocab": 32568,
          "fertility": null,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/KasiaMP/wyniki/tokenizer32k.json"
        },
        {
          "name": "Konrad/tokenizer.json",
          "author": "Konrad",
          "vocab": 456,
          "fertility": null,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/Konrad/tokenizer.json"
        },
        {
          "name": "Kuba/tokenizer.json",
          "author": "Kuba",
          "vocab": 14000,
          "fertility": null,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/Kuba/tokenizer.json"
        },
        {
          "name": "dawidm/vocabs/syzyfowe-prace_16000.tokenizer.json",
          "author": "Dawid",
          "vocab": 16000,
          "fertility": null,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/dawidm/vocabs/syzyfowe-prace_16000.tokenizer.json"
        },
        {
          "name": "maciej/tokenizers/hplt-pl-64k-pl.json",
          "author": "Maciej",
          "vocab": 63999,
          "fertility": null,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/maciej/tokenizers/hplt-pl-64k-pl.json"
        },
        {
          "name": "ola/polish_bpe_8k.json",
          "author": "Ola",
          "vocab": 8000,
          "fertility": null,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/ola/polish_bpe_8k.json"
        },
        {
          "name": "patryk/polish_bpe_vocab.json",
          "author": "Patryk",
          "vocab": 16000,
          "fertility": null,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/patryk/polish_bpe_vocab.json"
        },
        {
          "name": "ppuzio/tokenizers/dynaword-128k-g220mb-l50mb/tokenizer.json",
          "author": "Paweł Puzio",
          "vocab": 128000,
          "fertility": null,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/ppuzio/tokenizers/dynaword-128k-g220mb-l50mb/tokenizer.json"
        },
        {
          "name": "Tokenizer UTF-8 · eksperymenty i raport",
          "author": "Piotr",
          "vocab": null,
          "fertility": null,
          "url": "https://github.com/slayerlabs/tokenizer/blob/1b25efca5c1a3891edcfdc47a8945b55e531223c/piotr/v2/opracowanie.md"
        }
      ]
    }
  ]
}